Merge branch 'stats' into stats_histo

145 files changed, 7109 insertions, 3475 deletions

diff --git a/src/core/ext/census/grpc_filter.c b/src/core/ext/census/grpc_filter.c
index 13fe2e6b1c..3e10f61154 100644
--- a/src/core/ext/census/grpc_filter.c
+++ b/src/core/ext/census/grpc_filter.c
@@ -179,7 +179,6 @@ const grpc_channel_filter grpc_client_census_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "census-client"};
 
@@ -193,6 +192,5 @@ const grpc_channel_filter grpc_server_census_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "census-server"};
diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c
index 543a73c5ad..823c681abf 100644
--- a/src/core/ext/census/tracing.c
+++ b/src/core/ext/census/tracing.c
@@ -21,7 +21,6 @@
 #include <grpc/census.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <openssl/rand.h>
 #include "src/core/ext/census/mlog.h"
 
 void trace_start_span(const trace_span_context *span_ctxt,
diff --git a/src/core/ext/filters/client_channel/channel_connectivity.c b/src/core/ext/filters/client_channel/channel_connectivity.c
index c3dca14305..b83c95275f 100644
--- a/src/core/ext/filters/client_channel/channel_connectivity.c
+++ b/src/core/ext/filters/client_channel/channel_connectivity.c
@@ -208,7 +208,7 @@ void grpc_channel_watch_connectivity_state(
       7, (channel, (int)last_observed_state, deadline.tv_sec, deadline.tv_nsec,
           (int)deadline.clock_type, cq, tag));
 
-  grpc_cq_begin_op(cq, tag);
+  GPR_ASSERT(grpc_cq_begin_op(cq, tag));
 
   gpr_mu_init(&w->mu);
   GRPC_CLOSURE_INIT(&w->on_complete, watch_complete, w,
diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c
index 58e31d7b45..e6822ce801 100644
--- a/src/core/ext/filters/client_channel/client_channel.c
+++ b/src/core/ext/filters/client_channel/client_channel.c
@@ -796,7 +796,8 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 //   send_message
 //   recv_trailing_metadata
 //   send_trailing_metadata
-#define MAX_WAITING_BATCHES 6
+// We also add room for a single cancel_stream batch.
+#define MAX_WAITING_BATCHES 7
 
 /** Call data.  Holds a pointer to grpc_subchannel_call and the
     associated machinery to create such a pointer.
@@ -808,23 +809,25 @@ typedef struct client_channel_call_data {
   // The code in deadline_filter.c requires this to be the first field.
   // TODO(roth): This is slightly sub-optimal in that grpc_deadline_state
   // and this struct both independently store a pointer to the call
-  // stack and each has its own mutex.  If/when we have time, find a way
-  // to avoid this without breaking the grpc_deadline_state abstraction.
+  // combiner.  If/when we have time, find a way to avoid this without
+  // breaking the grpc_deadline_state abstraction.
   grpc_deadline_state deadline_state;
 
   grpc_slice path;  // Request path.
   gpr_timespec call_start_time;
   gpr_timespec deadline;
+  gpr_arena *arena;
+  grpc_call_combiner *call_combiner;
+
   grpc_server_retry_throttle_data *retry_throttle_data;
   method_parameters *method_params;
 
-  /** either 0 for no call, a pointer to a grpc_subchannel_call (if the lowest
-      bit is 0), or a pointer to an error (if the lowest bit is 1) */
-  gpr_atm subchannel_call_or_error;
-  gpr_arena *arena;
+  grpc_subchannel_call *subchannel_call;
+  grpc_error *error;
 
   grpc_lb_policy *lb_policy;  // Holds ref while LB pick is pending.
   grpc_closure lb_pick_closure;
+  grpc_closure cancel_closure;
 
   grpc_connected_subchannel *connected_subchannel;
   grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
@@ -832,10 +835,9 @@ typedef struct client_channel_call_data {
 
   grpc_transport_stream_op_batch *waiting_for_pick_batches[MAX_WAITING_BATCHES];
   size_t waiting_for_pick_batches_count;
+  grpc_closure handle_pending_batch_in_call_combiner[MAX_WAITING_BATCHES];
 
-  grpc_transport_stream_op_batch_payload *initial_metadata_payload;
-
-  grpc_call_stack *owning_call;
+  grpc_transport_stream_op_batch *initial_metadata_batch;
 
   grpc_linked_mdelem lb_token_mdelem;
 
@@ -843,55 +845,42 @@ typedef struct client_channel_call_data {
   grpc_closure *original_on_complete;
 } call_data;
 
-typedef struct {
-  grpc_subchannel_call *subchannel_call;
-  grpc_error *error;
-} call_or_error;
-
-static call_or_error get_call_or_error(call_data *p) {
-  gpr_atm c = gpr_atm_acq_load(&p->subchannel_call_or_error);
-  if (c == 0)
-    return (call_or_error){NULL, NULL};
-  else if (c & 1)
-    return (call_or_error){NULL, (grpc_error *)((c) & ~(gpr_atm)1)};
-  else
-    return (call_or_error){(grpc_subchannel_call *)c, NULL};
+grpc_subchannel_call *grpc_client_channel_get_subchannel_call(
+    grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
+  return calld->subchannel_call;
 }
 
-static bool set_call_or_error(call_data *p, call_or_error coe) {
-  // this should always be under a lock
-  call_or_error existing = get_call_or_error(p);
-  if (existing.error != GRPC_ERROR_NONE) {
-    GRPC_ERROR_UNREF(coe.error);
-    return false;
-  }
-  GPR_ASSERT(existing.subchannel_call == NULL);
-  if (coe.error != GRPC_ERROR_NONE) {
-    GPR_ASSERT(coe.subchannel_call == NULL);
-    gpr_atm_rel_store(&p->subchannel_call_or_error, 1 | (gpr_atm)coe.error);
+// This is called via the call combiner, so access to calld is synchronized.
+static void waiting_for_pick_batches_add(
+    call_data *calld, grpc_transport_stream_op_batch *batch) {
+  if (batch->send_initial_metadata) {
+    GPR_ASSERT(calld->initial_metadata_batch == NULL);
+    calld->initial_metadata_batch = batch;
   } else {
-    GPR_ASSERT(coe.subchannel_call != NULL);
-    gpr_atm_rel_store(&p->subchannel_call_or_error,
-                      (gpr_atm)coe.subchannel_call);
+    GPR_ASSERT(calld->waiting_for_pick_batches_count < MAX_WAITING_BATCHES);
+    calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count++] =
+        batch;
   }
-  return true;
 }
 
-grpc_subchannel_call *grpc_client_channel_get_subchannel_call(
-    grpc_call_element *call_elem) {
-  return get_call_or_error(call_elem->call_data).subchannel_call;
-}
-
-static void waiting_for_pick_batches_add_locked(
-    call_data *calld, grpc_transport_stream_op_batch *batch) {
-  GPR_ASSERT(calld->waiting_for_pick_batches_count < MAX_WAITING_BATCHES);
-  calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count++] =
-      batch;
+// This is called via the call combiner, so access to calld is synchronized.
+static void fail_pending_batch_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                                void *arg, grpc_error *error) {
+  call_data *calld = arg;
+  if (calld->waiting_for_pick_batches_count > 0) {
+    --calld->waiting_for_pick_batches_count;
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx,
+        calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count],
+        GRPC_ERROR_REF(error), calld->call_combiner);
+  }
 }
 
-static void waiting_for_pick_batches_fail_locked(grpc_exec_ctx *exec_ctx,
-                                                 grpc_call_element *elem,
-                                                 grpc_error *error) {
+// This is called via the call combiner, so access to calld is synchronized.
+static void waiting_for_pick_batches_fail(grpc_exec_ctx *exec_ctx,
+                                          grpc_call_element *elem,
+                                          grpc_error *error) {
   call_data *calld = elem->call_data;
   if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
     gpr_log(GPR_DEBUG,
@@ -900,34 +889,60 @@ static void waiting_for_pick_batches_fail_locked(grpc_exec_ctx *exec_ctx,
             grpc_error_string(error));
   }
   for (size_t i = 0; i < calld->waiting_for_pick_batches_count; ++i) {
+    GRPC_CLOSURE_INIT(&calld->handle_pending_batch_in_call_combiner[i],
+                      fail_pending_batch_in_call_combiner, calld,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CALL_COMBINER_START(exec_ctx, calld->call_combiner,
+                             &calld->handle_pending_batch_in_call_combiner[i],
+                             GRPC_ERROR_REF(error),
+                             "waiting_for_pick_batches_fail");
+  }
+  if (calld->initial_metadata_batch != NULL) {
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, calld->waiting_for_pick_batches[i], GRPC_ERROR_REF(error));
+        exec_ctx, calld->initial_metadata_batch, GRPC_ERROR_REF(error),
+        calld->call_combiner);
+  } else {
+    GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                            "waiting_for_pick_batches_fail");
   }
-  calld->waiting_for_pick_batches_count = 0;
   GRPC_ERROR_UNREF(error);
 }
 
-static void waiting_for_pick_batches_resume_locked(grpc_exec_ctx *exec_ctx,
-                                                   grpc_call_element *elem) {
-  call_data *calld = elem->call_data;
-  if (calld->waiting_for_pick_batches_count == 0) return;
-  call_or_error coe = get_call_or_error(calld);
-  if (coe.error != GRPC_ERROR_NONE) {
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem,
-                                         GRPC_ERROR_REF(coe.error));
-    return;
+// This is called via the call combiner, so access to calld is synchronized.
+static void run_pending_batch_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                               void *arg, grpc_error *ignored) {
+  call_data *calld = arg;
+  if (calld->waiting_for_pick_batches_count > 0) {
+    --calld->waiting_for_pick_batches_count;
+    grpc_subchannel_call_process_op(
+        exec_ctx, calld->subchannel_call,
+        calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count]);
   }
+}
+
+// This is called via the call combiner, so access to calld is synchronized.
+static void waiting_for_pick_batches_resume(grpc_exec_ctx *exec_ctx,
+                                            grpc_call_element *elem) {
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
   if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: sending %" PRIdPTR
                        " pending batches to subchannel_call=%p",
-            elem->channel_data, calld, calld->waiting_for_pick_batches_count,
-            coe.subchannel_call);
+            chand, calld, calld->waiting_for_pick_batches_count,
+            calld->subchannel_call);
   }
   for (size_t i = 0; i < calld->waiting_for_pick_batches_count; ++i) {
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call,
-                                    calld->waiting_for_pick_batches[i]);
-  }
-  calld->waiting_for_pick_batches_count = 0;
+    GRPC_CLOSURE_INIT(&calld->handle_pending_batch_in_call_combiner[i],
+                      run_pending_batch_in_call_combiner, calld,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CALL_COMBINER_START(exec_ctx, calld->call_combiner,
+                             &calld->handle_pending_batch_in_call_combiner[i],
+                             GRPC_ERROR_NONE,
+                             "waiting_for_pick_batches_resume");
+  }
+  GPR_ASSERT(calld->initial_metadata_batch != NULL);
+  grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call,
+                                  calld->initial_metadata_batch);
 }
 
 // Applies service config to the call.  Must be invoked once we know
@@ -968,29 +983,28 @@ static void apply_service_config_to_call_locked(grpc_exec_ctx *exec_ctx,
 static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx,
                                           grpc_call_element *elem,
                                           grpc_error *error) {
+  channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  grpc_subchannel_call *subchannel_call = NULL;
   const grpc_connected_subchannel_call_args call_args = {
       .pollent = calld->pollent,
       .path = calld->path,
       .start_time = calld->call_start_time,
       .deadline = calld->deadline,
       .arena = calld->arena,
-      .context = calld->subchannel_call_context};
+      .context = calld->subchannel_call_context,
+      .call_combiner = calld->call_combiner};
   grpc_error *new_error = grpc_connected_subchannel_create_call(
-      exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call);
+      exec_ctx, calld->connected_subchannel, &call_args,
+      &calld->subchannel_call);
   if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
-            elem->channel_data, calld, subchannel_call,
-            grpc_error_string(new_error));
+            chand, calld, calld->subchannel_call, grpc_error_string(new_error));
   }
-  GPR_ASSERT(set_call_or_error(
-      calld, (call_or_error){.subchannel_call = subchannel_call}));
   if (new_error != GRPC_ERROR_NONE) {
     new_error = grpc_error_add_child(new_error, error);
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, new_error);
+    waiting_for_pick_batches_fail(exec_ctx, elem, new_error);
   } else {
-    waiting_for_pick_batches_resume_locked(exec_ctx, elem);
+    waiting_for_pick_batches_resume(exec_ctx, elem);
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -1002,60 +1016,27 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
   channel_data *chand = elem->channel_data;
   grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent,
                                            chand->interested_parties);
-  call_or_error coe = get_call_or_error(calld);
   if (calld->connected_subchannel == NULL) {
     // Failed to create subchannel.
-    grpc_error *failure =
-        error == GRPC_ERROR_NONE
-            ? GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                  "Call dropped by load balancing policy")
-            : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                  "Failed to create subchannel", &error, 1);
+    GRPC_ERROR_UNREF(calld->error);
+    calld->error = error == GRPC_ERROR_NONE
+                       ? GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                             "Call dropped by load balancing policy")
+                       : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                             "Failed to create subchannel", &error, 1);
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG,
               "chand=%p calld=%p: failed to create subchannel: error=%s", chand,
-              calld, grpc_error_string(failure));
-    }
-    set_call_or_error(calld, (call_or_error){.error = GRPC_ERROR_REF(failure)});
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, failure);
-  } else if (coe.error != GRPC_ERROR_NONE) {
-    /* already cancelled before subchannel became ready */
-    grpc_error *child_errors[] = {error, coe.error};
-    grpc_error *cancellation_error =
-        GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-            "Cancelled before creating subchannel", child_errors,
-            GPR_ARRAY_SIZE(child_errors));
-    /* if due to deadline, attach the deadline exceeded status to the error */
-    if (gpr_time_cmp(calld->deadline, gpr_now(GPR_CLOCK_MONOTONIC)) < 0) {
-      cancellation_error =
-          grpc_error_set_int(cancellation_error, GRPC_ERROR_INT_GRPC_STATUS,
-                             GRPC_STATUS_DEADLINE_EXCEEDED);
+              calld, grpc_error_string(calld->error));
     }
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG,
-              "chand=%p calld=%p: cancelled before subchannel became ready: %s",
-              chand, calld, grpc_error_string(cancellation_error));
-    }
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, cancellation_error);
+    waiting_for_pick_batches_fail(exec_ctx, elem, GRPC_ERROR_REF(calld->error));
   } else {
     /* Create call on subchannel. */
     create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
   }
-  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
   GRPC_ERROR_UNREF(error);
 }
 
-static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  call_data *calld = elem->call_data;
-  grpc_subchannel_call *subchannel_call =
-      get_call_or_error(calld).subchannel_call;
-  if (subchannel_call == NULL) {
-    return NULL;
-  } else {
-    return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
-  }
-}
-
 /** Return true if subchannel is available immediately (in which case
     subchannel_ready_locked() should not be called), or false otherwise (in
     which case subchannel_ready_locked() should be called when the subchannel
@@ -1069,6 +1050,44 @@ typedef struct {
   grpc_closure closure;
 } pick_after_resolver_result_args;
 
+// Note: This runs under the client_channel combiner, but will NOT be
+// holding the call combiner.
+static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
+                                                     void *arg,
+                                                     grpc_error *error) {
+  grpc_call_element *elem = arg;
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  // If we don't yet have a resolver result, then a closure for
+  // pick_after_resolver_result_done_locked() will have been added to
+  // chand->waiting_for_resolver_result_closures, and it may not be invoked
+  // until after this call has been destroyed.  We mark the operation as
+  // cancelled, so that when pick_after_resolver_result_done_locked()
+  // is called, it will be a no-op.  We also immediately invoke
+  // subchannel_ready_locked() to propagate the error back to the caller.
+  for (grpc_closure *closure = chand->waiting_for_resolver_result_closures.head;
+       closure != NULL; closure = closure->next_data.next) {
+    pick_after_resolver_result_args *args = closure->cb_arg;
+    if (!args->cancelled && args->elem == elem) {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG,
+                "chand=%p calld=%p: "
+                "cancelling pick waiting for resolver result",
+                chand, calld);
+      }
+      args->cancelled = true;
+      // Note: Although we are not in the call combiner here, we are
+      // basically stealing the call combiner from the pending pick, so
+      // it's safe to call subchannel_ready_locked() here -- we are
+      // essentially calling it here instead of calling it in
+      // pick_after_resolver_result_done_locked().
+      subchannel_ready_locked(exec_ctx, elem,
+                              GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                                  "Pick cancelled", &error, 1));
+    }
+  }
+}
+
 static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
                                                    void *arg,
                                                    grpc_error *error) {
@@ -1079,21 +1098,24 @@ static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
       gpr_log(GPR_DEBUG, "call cancelled before resolver result");
     }
   } else {
-    channel_data *chand = args->elem->channel_data;
-    call_data *calld = args->elem->call_data;
+    grpc_call_element *elem = args->elem;
+    channel_data *chand = elem->channel_data;
+    call_data *calld = elem->call_data;
+    grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
+                                            NULL);
     if (error != GRPC_ERROR_NONE) {
       if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
         gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data",
                 chand, calld);
       }
-      subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_REF(error));
+      subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
     } else {
       if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
         gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick",
                 chand, calld);
       }
-      if (pick_subchannel_locked(exec_ctx, args->elem)) {
-        subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_NONE);
+      if (pick_subchannel_locked(exec_ctx, elem)) {
+        subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE);
       }
     }
   }
@@ -1116,41 +1138,33 @@ static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
                     args, grpc_combiner_scheduler(chand->combiner));
   grpc_closure_list_append(&chand->waiting_for_resolver_result_closures,
                            &args->closure, GRPC_ERROR_NONE);
+  grpc_call_combiner_set_notify_on_cancel(
+      exec_ctx, calld->call_combiner,
+      GRPC_CLOSURE_INIT(&calld->cancel_closure,
+                        pick_after_resolver_result_cancel_locked, elem,
+                        grpc_combiner_scheduler(chand->combiner)));
 }
 
-static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
-                                                     grpc_call_element *elem,
-                                                     grpc_error *error) {
+// Note: This runs under the client_channel combiner, but will NOT be
+// holding the call combiner.
+static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  grpc_call_element *elem = arg;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  // If we don't yet have a resolver result, then a closure for
-  // pick_after_resolver_result_done_locked() will have been added to
-  // chand->waiting_for_resolver_result_closures, and it may not be invoked
-  // until after this call has been destroyed.  We mark the operation as
-  // cancelled, so that when pick_after_resolver_result_done_locked()
-  // is called, it will be a no-op.  We also immediately invoke
-  // subchannel_ready_locked() to propagate the error back to the caller.
-  for (grpc_closure *closure = chand->waiting_for_resolver_result_closures.head;
-       closure != NULL; closure = closure->next_data.next) {
-    pick_after_resolver_result_args *args = closure->cb_arg;
-    if (!args->cancelled && args->elem == elem) {
-      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-        gpr_log(GPR_DEBUG,
-                "chand=%p calld=%p: "
-                "cancelling pick waiting for resolver result",
-                chand, calld);
-      }
-      args->cancelled = true;
-      subchannel_ready_locked(exec_ctx, elem,
-                              GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                                  "Pick cancelled", &error, 1));
+  if (calld->lb_policy != NULL) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
+              chand, calld, calld->lb_policy);
     }
+    grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
+                                      &calld->connected_subchannel,
+                                      GRPC_ERROR_REF(error));
   }
-  GRPC_ERROR_UNREF(error);
 }
 
 // Callback invoked by grpc_lb_policy_pick_locked() for async picks.
-// Unrefs the LB policy after invoking subchannel_ready_locked().
+// Unrefs the LB policy and invokes subchannel_ready_locked().
 static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                       grpc_error *error) {
   grpc_call_element *elem = arg;
@@ -1160,6 +1174,7 @@ static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
             chand, calld);
   }
+  grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL);
   GPR_ASSERT(calld->lb_policy != NULL);
   GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
   calld->lb_policy = NULL;
@@ -1194,24 +1209,15 @@ static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx,
     }
     GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
     calld->lb_policy = NULL;
+  } else {
+    grpc_call_combiner_set_notify_on_cancel(
+        exec_ctx, calld->call_combiner,
+        GRPC_CLOSURE_INIT(&calld->cancel_closure, pick_callback_cancel_locked,
+                          elem, grpc_combiner_scheduler(chand->combiner)));
   }
   return pick_done;
 }
 
-static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_element *elem,
-                                        grpc_error *error) {
-  channel_data *chand = elem->channel_data;
-  call_data *calld = elem->call_data;
-  GPR_ASSERT(calld->lb_policy != NULL);
-  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-    gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-            chand, calld, calld->lb_policy);
-  }
-  grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
-                                    &calld->connected_subchannel, error);
-}
-
 static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem) {
   GPR_TIMER_BEGIN("pick_subchannel", 0);
@@ -1224,7 +1230,7 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
     // Otherwise, if the service config specified a value for this
     // method, use that.
     uint32_t initial_metadata_flags =
-        calld->initial_metadata_payload->send_initial_metadata
+        calld->initial_metadata_batch->payload->send_initial_metadata
             .send_initial_metadata_flags;
     const bool wait_for_ready_set_from_api =
         initial_metadata_flags &
@@ -1241,7 +1247,7 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
       }
     }
     const grpc_lb_policy_pick_args inputs = {
-        calld->initial_metadata_payload->send_initial_metadata
+        calld->initial_metadata_batch->payload->send_initial_metadata
             .send_initial_metadata,
         initial_metadata_flags, &calld->lb_token_mdelem};
     pick_done = pick_callback_start_locked(exec_ctx, elem, &inputs);
@@ -1258,91 +1264,33 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
   return pick_done;
 }
 
-static void start_transport_stream_op_batch_locked(grpc_exec_ctx *exec_ctx,
-                                                   void *arg,
-                                                   grpc_error *error_ignored) {
-  GPR_TIMER_BEGIN("start_transport_stream_op_batch_locked", 0);
-  grpc_transport_stream_op_batch *batch = arg;
-  grpc_call_element *elem = batch->handler_private.extra_arg;
-  call_data *calld = elem->call_data;
-  channel_data *chand = elem->channel_data;
-  /* need to recheck that another thread hasn't set the call */
-  call_or_error coe = get_call_or_error(calld);
-  if (coe.error != GRPC_ERROR_NONE) {
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
-              chand, calld, grpc_error_string(coe.error));
-    }
-    grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, batch, GRPC_ERROR_REF(coe.error));
-    goto done;
-  }
-  if (coe.subchannel_call != NULL) {
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG,
-              "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
-              calld, coe.subchannel_call);
-    }
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, batch);
-    goto done;
-  }
-  // Add to waiting-for-pick list.  If we succeed in getting a
-  // subchannel call below, we'll handle this batch (along with any
-  // other waiting batches) in waiting_for_pick_batches_resume_locked().
-  waiting_for_pick_batches_add_locked(calld, batch);
-  // If this is a cancellation, cancel the pending pick (if any) and
-  // fail any pending batches.
-  if (batch->cancel_stream) {
-    grpc_error *error = batch->payload->cancel_stream.cancel_error;
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: recording cancel_error=%s", chand,
-              calld, grpc_error_string(error));
-    }
-    /* Stash a copy of cancel_error in our call data, so that we can use
-       it for subsequent operations.  This ensures that if the call is
-       cancelled before any batches are passed down (e.g., if the deadline
-       is in the past when the call starts), we can return the right
-       error to the caller when the first batch does get passed down. */
-    set_call_or_error(calld, (call_or_error){.error = GRPC_ERROR_REF(error)});
-    if (calld->lb_policy != NULL) {
-      pick_callback_cancel_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
+static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error_ignored) {
+  GPR_TIMER_BEGIN("start_pick_locked", 0);
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  GPR_ASSERT(calld->connected_subchannel == NULL);
+  if (pick_subchannel_locked(exec_ctx, elem)) {
+    // Pick was returned synchronously.
+    if (calld->connected_subchannel == NULL) {
+      GRPC_ERROR_UNREF(calld->error);
+      calld->error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "Call dropped by load balancing policy");
+      waiting_for_pick_batches_fail(exec_ctx, elem,
+                                    GRPC_ERROR_REF(calld->error));
     } else {
-      pick_after_resolver_result_cancel_locked(exec_ctx, elem,
-                                               GRPC_ERROR_REF(error));
-    }
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
-    goto done;
-  }
-  /* if we don't have a subchannel, try to get one */
-  if (batch->send_initial_metadata) {
-    GPR_ASSERT(calld->connected_subchannel == NULL);
-    calld->initial_metadata_payload = batch->payload;
-    GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel");
-    /* If a subchannel is not available immediately, the polling entity from
-       call_data should be provided to channel_data's interested_parties, so
-       that IO of the lb_policy and resolver could be done under it. */
-    if (pick_subchannel_locked(exec_ctx, elem)) {
-      // Pick was returned synchronously.
-      GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
-      if (calld->connected_subchannel == NULL) {
-        grpc_error *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-            "Call dropped by load balancing policy");
-        set_call_or_error(calld,
-                          (call_or_error){.error = GRPC_ERROR_REF(error)});
-        waiting_for_pick_batches_fail_locked(exec_ctx, elem, error);
-      } else {
-        // Create subchannel call.
-        create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE);
-      }
-    } else {
-      grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent,
-                                             chand->interested_parties);
+      // Create subchannel call.
+      create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE);
     }
+  } else {
+    // Pick will be done asynchronously.  Add the call's polling entity to
+    // the channel's interested_parties, so that I/O for the resolver
+    // and LB policy can be done under it.
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent,
+                                           chand->interested_parties);
   }
-done:
-  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call,
-                        "start_transport_stream_op_batch");
-  GPR_TIMER_END("start_transport_stream_op_batch_locked", 0);
+  GPR_TIMER_END("start_pick_locked", 0);
 }
 
 static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
@@ -1365,27 +1313,49 @@ static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
                    GRPC_ERROR_REF(error));
 }
 
-/* The logic here is fairly complicated, due to (a) the fact that we
-   need to handle the case where we receive the send op before the
-   initial metadata op, and (b) the need for efficiency, especially in
-   the streaming case.
-
-   We use double-checked locking to initially see if initialization has been
-   performed. If it has not, we acquire the combiner and perform initialization.
-   If it has, we proceed on the fast path. */
 static void cc_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  if (GRPC_TRACER_ON(grpc_client_channel_trace) ||
-      GRPC_TRACER_ON(grpc_trace_channel)) {
-    grpc_call_log_op(GPR_INFO, elem, batch);
-  }
   if (chand->deadline_checking_enabled) {
     grpc_deadline_state_client_start_transport_stream_op_batch(exec_ctx, elem,
                                                                batch);
   }
+  GPR_TIMER_BEGIN("cc_start_transport_stream_op_batch", 0);
+  // If we've previously been cancelled, immediately fail any new batches.
+  if (calld->error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
+              chand, calld, grpc_error_string(calld->error));
+    }
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, batch, GRPC_ERROR_REF(calld->error), calld->call_combiner);
+    goto done;
+  }
+  if (batch->cancel_stream) {
+    // Stash a copy of cancel_error in our call data, so that we can use
+    // it for subsequent operations.  This ensures that if the call is
+    // cancelled before any batches are passed down (e.g., if the deadline
+    // is in the past when the call starts), we can return the right
+    // error to the caller when the first batch does get passed down.
+    GRPC_ERROR_UNREF(calld->error);
+    calld->error = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: recording cancel_error=%s", chand,
+              calld, grpc_error_string(calld->error));
+    }
+    // If we have a subchannel call, send the cancellation batch down.
+    // Otherwise, fail all pending batches.
+    if (calld->subchannel_call != NULL) {
+      grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, batch);
+    } else {
+      waiting_for_pick_batches_add(calld, batch);
+      waiting_for_pick_batches_fail(exec_ctx, elem,
+                                    GRPC_ERROR_REF(calld->error));
+    }
+    goto done;
+  }
   // Intercept on_complete for recv_trailing_metadata so that we can
   // check retry throttle status.
   if (batch->recv_trailing_metadata) {
@@ -1395,38 +1365,43 @@ static void cc_start_transport_stream_op_batch(
                       grpc_schedule_on_exec_ctx);
     batch->on_complete = &calld->on_complete;
   }
-  /* try to (atomically) get the call */
-  call_or_error coe = get_call_or_error(calld);
-  GPR_TIMER_BEGIN("cc_start_transport_stream_op_batch", 0);
-  if (coe.error != GRPC_ERROR_NONE) {
+  // Check if we've already gotten a subchannel call.
+  // Note that once we have completed the pick, we do not need to enter
+  // the channel combiner, which is more efficient (especially for
+  // streaming calls).
+  if (calld->subchannel_call != NULL) {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
-              chand, calld, grpc_error_string(coe.error));
+      gpr_log(GPR_DEBUG,
+              "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
+              calld, calld->subchannel_call);
     }
-    grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, batch, GRPC_ERROR_REF(coe.error));
+    grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, batch);
     goto done;
   }
-  if (coe.subchannel_call != NULL) {
+  // We do not yet have a subchannel call.
+  // Add the batch to the waiting-for-pick list.
+  waiting_for_pick_batches_add(calld, batch);
+  // For batches containing a send_initial_metadata op, enter the channel
+  // combiner to start a pick.
+  if (batch->send_initial_metadata) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld);
+    }
+    GRPC_CLOSURE_SCHED(
+        exec_ctx,
+        GRPC_CLOSURE_INIT(&batch->handler_private.closure, start_pick_locked,
+                          elem, grpc_combiner_scheduler(chand->combiner)),
+        GRPC_ERROR_NONE);
+  } else {
+    // For all other batches, release the call combiner.
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG,
-              "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
-              calld, coe.subchannel_call);
+              "chand=%p calld=%p: saved batch, yeilding call combiner", chand,
+              calld);
     }
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, batch);
-    goto done;
-  }
-  /* we failed; lock and figure out what to do */
-  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-    gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld);
+    GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                            "batch does not include send_initial_metadata");
   }
-  GRPC_CALL_STACK_REF(calld->owning_call, "start_transport_stream_op_batch");
-  batch->handler_private.extra_arg = elem;
-  GRPC_CLOSURE_SCHED(
-      exec_ctx, GRPC_CLOSURE_INIT(&batch->handler_private.closure,
-                                  start_transport_stream_op_batch_locked, batch,
-                                  grpc_combiner_scheduler(chand->combiner)),
-      GRPC_ERROR_NONE);
 done:
   GPR_TIMER_END("cc_start_transport_stream_op_batch", 0);
 }
@@ -1441,10 +1416,11 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx,
   calld->path = grpc_slice_ref_internal(args->path);
   calld->call_start_time = args->start_time;
   calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC);
-  calld->owning_call = args->call_stack;
   calld->arena = args->arena;
+  calld->call_combiner = args->call_combiner;
   if (chand->deadline_checking_enabled) {
-    grpc_deadline_state_init(exec_ctx, elem, args->call_stack, calld->deadline);
+    grpc_deadline_state_init(exec_ctx, elem, args->call_stack,
+                             args->call_combiner, calld->deadline);
   }
   return GRPC_ERROR_NONE;
 }
@@ -1463,13 +1439,12 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
   if (calld->method_params != NULL) {
     method_parameters_unref(calld->method_params);
   }
-  call_or_error coe = get_call_or_error(calld);
-  GRPC_ERROR_UNREF(coe.error);
-  if (coe.subchannel_call != NULL) {
-    grpc_subchannel_call_set_cleanup_closure(coe.subchannel_call,
+  GRPC_ERROR_UNREF(calld->error);
+  if (calld->subchannel_call != NULL) {
+    grpc_subchannel_call_set_cleanup_closure(calld->subchannel_call,
                                              then_schedule_closure);
     then_schedule_closure = NULL;
-    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, coe.subchannel_call,
+    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, calld->subchannel_call,
                                "client_channel_destroy_call");
   }
   GPR_ASSERT(calld->lb_policy == NULL);
@@ -1508,7 +1483,6 @@ const grpc_channel_filter grpc_client_channel_filter = {
     sizeof(channel_data),
     cc_init_channel_elem,
     cc_destroy_channel_elem,
-    cc_get_peer,
     cc_get_channel_info,
     "client-channel",
 };
diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c
index aa3f61c991..ef3512ed83 100644
--- a/src/core/ext/filters/client_channel/http_proxy.c
+++ b/src/core/ext/filters/client_channel/http_proxy.c
@@ -30,15 +30,23 @@
 #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
 #include "src/core/ext/filters/client_channel/uri_parser.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/slice/b64.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 
-static char* grpc_get_http_proxy_server(grpc_exec_ctx* exec_ctx) {
+/**
+ * Parses the 'http_proxy' env var and returns the proxy hostname to resolve or
+ * NULL on error. Also sets 'user_cred' to user credentials if present in the
+ * 'http_proxy' env var, otherwise leaves it unchanged. It is caller's
+ * responsibility to gpr_free user_cred.
+ */
+static char* get_http_proxy_server(grpc_exec_ctx* exec_ctx, char** user_cred) {
+  GPR_ASSERT(user_cred != NULL);
+  char* proxy_name = NULL;
   char* uri_str = gpr_getenv("http_proxy");
   if (uri_str == NULL) return NULL;
   grpc_uri* uri =
       grpc_uri_parse(exec_ctx, uri_str, false /* suppress_errors */);
-  char* proxy_name = NULL;
   if (uri == NULL || uri->authority == NULL) {
     gpr_log(GPR_ERROR, "cannot parse value of 'http_proxy' env var");
     goto done;
@@ -47,11 +55,27 @@ static char* grpc_get_http_proxy_server(grpc_exec_ctx* exec_ctx) {
     gpr_log(GPR_ERROR, "'%s' scheme not supported in proxy URI", uri->scheme);
     goto done;
   }
-  if (strchr(uri->authority, '@') != NULL) {
-    gpr_log(GPR_ERROR, "userinfo not supported in proxy URI");
-    goto done;
+  /* Split on '@' to separate user credentials from host */
+  char** authority_strs = NULL;
+  size_t authority_nstrs;
+  gpr_string_split(uri->authority, "@", &authority_strs, &authority_nstrs);
+  GPR_ASSERT(authority_nstrs != 0); /* should have at least 1 string */
+  if (authority_nstrs == 1) {
+    /* User cred not present in authority */
+    proxy_name = authority_strs[0];
+  } else if (authority_nstrs == 2) {
+    /* User cred found */
+    *user_cred = authority_strs[0];
+    proxy_name = authority_strs[1];
+    gpr_log(GPR_DEBUG, "userinfo found in proxy URI");
+  } else {
+    /* Bad authority */
+    for (size_t i = 0; i < authority_nstrs; i++) {
+      gpr_free(authority_strs[i]);
+    }
+    proxy_name = NULL;
   }
-  proxy_name = gpr_strdup(uri->authority);
+  gpr_free(authority_strs);
 done:
   gpr_free(uri_str);
   grpc_uri_destroy(uri);
@@ -64,7 +88,8 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
                                   const grpc_channel_args* args,
                                   char** name_to_resolve,
                                   grpc_channel_args** new_args) {
-  *name_to_resolve = grpc_get_http_proxy_server(exec_ctx);
+  char* user_cred = NULL;
+  *name_to_resolve = get_http_proxy_server(exec_ctx, &user_cred);
   if (*name_to_resolve == NULL) return false;
   grpc_uri* uri =
       grpc_uri_parse(exec_ctx, server_uri, false /* suppress_errors */);
@@ -73,12 +98,16 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
             "'http_proxy' environment variable set, but cannot "
             "parse server URI '%s' -- not using proxy",
             server_uri);
-    if (uri != NULL) grpc_uri_destroy(uri);
+    if (uri != NULL) {
+      gpr_free(user_cred);
+      grpc_uri_destroy(uri);
+    }
     return false;
   }
   if (strcmp(uri->scheme, "unix") == 0) {
     gpr_log(GPR_INFO, "not using proxy for Unix domain socket '%s'",
             server_uri);
+    gpr_free(user_cred);
     grpc_uri_destroy(uri);
     return false;
   }
@@ -126,10 +155,25 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
       }
     }
   }
-  grpc_arg new_arg = grpc_channel_arg_string_create(
+  grpc_arg args_to_add[2];
+  args_to_add[0] = grpc_channel_arg_string_create(
       GRPC_ARG_HTTP_CONNECT_SERVER,
       uri->path[0] == '/' ? uri->path + 1 : uri->path);
-  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
+  if (user_cred != NULL) {
+    /* Use base64 encoding for user credentials as stated in RFC 7617 */
+    char* encoded_user_cred =
+        grpc_base64_encode(user_cred, strlen(user_cred), 0, 0);
+    char* header;
+    gpr_asprintf(&header, "Proxy-Authorization:Basic %s", encoded_user_cred);
+    gpr_free(encoded_user_cred);
+    args_to_add[1] =
+        grpc_channel_arg_string_create(GRPC_ARG_HTTP_CONNECT_HEADERS, header);
+    *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 2);
+    gpr_free(header);
+  } else {
+    *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 1);
+  }
+  gpr_free(user_cred);
   grpc_uri_destroy(uri);
   return true;
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy.c b/src/core/ext/filters/client_channel/lb_policy.c
index 10b032214b..dd95a135cf 100644
--- a/src/core/ext/filters/client_channel/lb_policy.c
+++ b/src/core/ext/filters/client_channel/lb_policy.c
@@ -54,7 +54,7 @@ static gpr_atm ref_mutate(grpc_lb_policy *c, gpr_atm delta,
 #ifndef NDEBUG
   if (GRPC_TRACER_ON(grpc_trace_lb_policy_refcount)) {
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "LB_POLICY: 0x%p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
+            "LB_POLICY: %p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
             purpose, old_val, old_val + delta, reason);
   }
 #endif
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c
index 52c6e38c87..299f26b4de 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c
@@ -88,7 +88,6 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   // Record call finished, optionally setting client_failed_to_send and
   // received.
   grpc_grpclb_client_stats_add_call_finished(
-      false /* drop_for_rate_limiting */, false /* drop_for_load_balancing */,
       !calld->send_initial_metadata_succeeded /* client_failed_to_send */,
       calld->recv_initial_metadata_succeeded /* known_received */,
       calld->client_stats);
@@ -133,6 +132,5 @@ const grpc_channel_filter grpc_client_load_reporting_filter = {
     0,  // sizeof(channel_data)
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "client_load_reporting"};
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
index fdb18f687f..087b4076e2 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
@@ -416,9 +416,7 @@ struct rr_connectivity_data {
 
 static bool is_server_valid(const grpc_grpclb_server *server, size_t idx,
                             bool log) {
-  if (server->drop_for_rate_limiting || server->drop_for_load_balancing) {
-    return false;
-  }
+  if (server->drop) return false;
   const grpc_grpclb_ip_address *ip = &server->ip_address;
   if (server->port >> 16 != 0) {
     if (log) {
@@ -462,7 +460,7 @@ static const grpc_lb_user_data_vtable lb_token_vtable = {
 static void parse_server(const grpc_grpclb_server *server,
                          grpc_resolved_address *addr) {
   memset(addr, 0, sizeof(*addr));
-  if (server->drop_for_rate_limiting || server->drop_for_load_balancing) return;
+  if (server->drop) return;
   const uint16_t netorder_port = htons((uint16_t)server->port);
   /* the addresses are given in binary format (a in(6)_addr struct) in
    * server->ip_address.bytes. */
@@ -491,11 +489,8 @@ static grpc_lb_addresses *process_serverlist_locked(
   for (size_t i = 0; i < serverlist->num_servers; ++i) {
     if (is_server_valid(serverlist->servers[i], i, true)) ++num_valid;
   }
-  if (num_valid == 0) return NULL;
-
   grpc_lb_addresses *lb_addresses =
       grpc_lb_addresses_create(num_valid, &lb_token_vtable);
-
   /* second pass: actually populate the addresses and LB tokens (aka user data
    * to the outside world) to be read by the RR policy during its creation.
    * Given that the validity tests are very cheap, they are performed again
@@ -503,14 +498,12 @@ static grpc_lb_addresses *process_serverlist_locked(
    * incurr in an allocation due to the arbitrary number of server */
   size_t addr_idx = 0;
   for (size_t sl_idx = 0; sl_idx < serverlist->num_servers; ++sl_idx) {
-    GPR_ASSERT(addr_idx < num_valid);
     const grpc_grpclb_server *server = serverlist->servers[sl_idx];
     if (!is_server_valid(serverlist->servers[sl_idx], sl_idx, false)) continue;
-
+    GPR_ASSERT(addr_idx < num_valid);
     /* address processing */
     grpc_resolved_address addr;
     parse_server(server, &addr);
-
     /* lb token processing */
     void *user_data;
     if (server->has_load_balance_token) {
@@ -596,7 +589,7 @@ static void update_lb_connectivity_status_locked(
         grpc_connectivity_state_name(rr_state), (void *)glb_policy->rr_policy);
   }
   grpc_connectivity_state_set(exec_ctx, &glb_policy->state_tracker, rr_state,
-                              GRPC_ERROR_REF(rr_state_error),
+                              rr_state_error,
                               "update_lb_connectivity_status_locked");
 }
 
@@ -615,7 +608,7 @@ static bool pick_from_internal_rr_locked(
   if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) {
     glb_policy->serverlist_index = 0;  // Wrap-around.
   }
-  if (server->drop_for_rate_limiting || server->drop_for_load_balancing) {
+  if (server->drop) {
     // Not using the RR policy, so unref it.
     if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")",
@@ -627,11 +620,8 @@ static bool pick_from_internal_rr_locked(
     // the client_load_reporting filter, because we do not create a
     // subchannel call (and therefore no client_load_reporting filter)
     // for dropped calls.
-    grpc_grpclb_client_stats_add_call_started(wc_arg->client_stats);
-    grpc_grpclb_client_stats_add_call_finished(
-        server->drop_for_rate_limiting, server->drop_for_load_balancing,
-        false /* failed_to_send */, false /* known_received */,
-        wc_arg->client_stats);
+    grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token,
+                                                     wc_arg->client_stats);
     grpc_grpclb_client_stats_unref(wc_arg->client_stats);
     if (force_async) {
       GPR_ASSERT(wc_arg->wrapped_closure != NULL);
@@ -678,11 +668,12 @@ static bool pick_from_internal_rr_locked(
 
 static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx,
                                                   glb_lb_policy *glb_policy) {
+  grpc_lb_addresses *addresses =
+      process_serverlist_locked(exec_ctx, glb_policy->serverlist);
+  GPR_ASSERT(addresses != NULL);
   grpc_lb_policy_args *args = gpr_zalloc(sizeof(*args));
   args->client_channel_factory = glb_policy->cc_factory;
   args->combiner = glb_policy->base.combiner;
-  grpc_lb_addresses *addresses =
-      process_serverlist_locked(exec_ctx, glb_policy->serverlist);
   // Replace the LB addresses in the channel args that we pass down to
   // the subchannel.
   static const char *keys_to_remove[] = {GRPC_ARG_LB_ADDRESSES};
@@ -719,7 +710,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
     return;
   }
   glb_policy->rr_policy = new_rr_policy;
-
   grpc_error *rr_state_error = NULL;
   const grpc_connectivity_state rr_state =
       grpc_lb_policy_check_connectivity_locked(exec_ctx, glb_policy->rr_policy,
@@ -727,7 +717,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
   /* Connectivity state is a function of the RR policy updated/created */
   update_lb_connectivity_status_locked(exec_ctx, glb_policy, rr_state,
                                        rr_state_error);
-
   /* Add the gRPC LB's interested_parties pollset_set to that of the newly
    * created RR policy. This will make the RR policy progress upon activity on
    * gRPC LB, which in turn is tied to the application's call */
@@ -746,7 +735,7 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
   rr_connectivity->state = rr_state;
 
   /* Subscribe to changes to the connectivity of the new RR */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "rr_connectivity_sched");
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "glb_rr_connectivity_cb");
   grpc_lb_policy_notify_on_state_change_locked(exec_ctx, glb_policy->rr_policy,
                                                &rr_connectivity->state,
                                                &rr_connectivity->on_change);
@@ -761,8 +750,8 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
     pp->wrapped_on_complete_arg.client_stats =
         grpc_grpclb_client_stats_ref(glb_policy->client_stats);
     if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
-      gpr_log(GPR_INFO, "Pending pick about to PICK from 0x%" PRIxPTR "",
-              (intptr_t)glb_policy->rr_policy);
+      gpr_log(GPR_INFO, "Pending pick about to (async) PICK from %p",
+              (void *)glb_policy->rr_policy);
     }
     pick_from_internal_rr_locked(exec_ctx, glb_policy, &pp->pick_args,
                                  true /* force_async */, pp->target,
@@ -788,10 +777,9 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx,
                                glb_lb_policy *glb_policy) {
   GPR_ASSERT(glb_policy->serverlist != NULL &&
              glb_policy->serverlist->num_servers > 0);
-
   if (glb_policy->shutting_down) return;
-
   grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy);
+  GPR_ASSERT(args != NULL);
   if (glb_policy->rr_policy != NULL) {
     if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
       gpr_log(GPR_DEBUG, "Updating Round Robin policy (%p)",
@@ -812,32 +800,31 @@ static void glb_rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx,
                                                void *arg, grpc_error *error) {
   rr_connectivity_data *rr_connectivity = arg;
   glb_lb_policy *glb_policy = rr_connectivity->glb_policy;
-
-  const bool shutting_down = glb_policy->shutting_down;
-  bool unref_needed = false;
-  GRPC_ERROR_REF(error);
-
-  if (rr_connectivity->state == GRPC_CHANNEL_SHUTDOWN || shutting_down) {
-    /* RR policy shutting down. Don't renew subscription and free the arg of
-     * this callback. In addition  we need to stash away the current policy to
-     * be UNREF'd after releasing the lock. Otherwise, if the UNREF is the last
-     * one, the policy would be destroyed, alongside the lock, which would
-     * result in a use-after-free */
-    unref_needed = true;
+  if (glb_policy->shutting_down) {
+    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
+                              "glb_rr_connectivity_cb");
     gpr_free(rr_connectivity);
-  } else { /* rr state != SHUTDOWN && !shutting down: biz as usual */
-    update_lb_connectivity_status_locked(exec_ctx, glb_policy,
-                                         rr_connectivity->state, error);
-    /* Resubscribe. Reuse the "rr_connectivity_cb" weak ref. */
-    grpc_lb_policy_notify_on_state_change_locked(
-        exec_ctx, glb_policy->rr_policy, &rr_connectivity->state,
-        &rr_connectivity->on_change);
+    return;
   }
-  if (unref_needed) {
+  if (rr_connectivity->state == GRPC_CHANNEL_SHUTDOWN) {
+    /* An RR policy that has transitioned into the SHUTDOWN connectivity state
+     * should not be considered for picks or updates: the SHUTDOWN state is a
+     * sink, policies can't transition back from it. .*/
+    GRPC_LB_POLICY_UNREF(exec_ctx, glb_policy->rr_policy,
+                         "rr_connectivity_shutdown");
+    glb_policy->rr_policy = NULL;
     GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
-                              "rr_connectivity_cb");
+                              "glb_rr_connectivity_cb");
+    gpr_free(rr_connectivity);
+    return;
   }
-  GRPC_ERROR_UNREF(error);
+  /* rr state != SHUTDOWN && !glb_policy->shutting down: biz as usual */
+  update_lb_connectivity_status_locked(
+      exec_ctx, glb_policy, rr_connectivity->state, GRPC_ERROR_REF(error));
+  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" weak ref. */
+  grpc_lb_policy_notify_on_state_change_locked(exec_ctx, glb_policy->rr_policy,
+                                               &rr_connectivity->state,
+                                               &rr_connectivity->on_change);
 }
 
 static void destroy_balancer_name(grpc_exec_ctx *exec_ctx,
@@ -1001,7 +988,6 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
     gpr_free(glb_policy);
     return NULL;
   }
-
   GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed,
                     glb_lb_channel_on_connectivity_changed_cb, glb_policy,
                     grpc_combiner_scheduler(args->combiner));
@@ -1058,7 +1044,7 @@ static void glb_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   glb_policy->pending_picks = NULL;
   pending_ping *pping = glb_policy->pending_pings;
   glb_policy->pending_pings = NULL;
-  if (glb_policy->rr_policy) {
+  if (glb_policy->rr_policy != NULL) {
     GRPC_LB_POLICY_UNREF(exec_ctx, glb_policy->rr_policy, "glb_shutdown");
   }
   // We destroy the LB channel here because
@@ -1089,6 +1075,16 @@ static void glb_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   }
 }
 
+// Cancel a specific pending pick.
+//
+// A grpclb pick progresses as follows:
+// - If there's a Round Robin policy (glb_policy->rr_policy) available, it'll be
+//   handed over to the RR policy (in create_rr_locked()). From that point
+//   onwards, it'll be RR's responsibility. For cancellations, that implies the
+//   pick needs also be cancelled by the RR instance.
+// - Otherwise, without an RR instance, picks stay pending at this policy's
+//   level (grpclb), inside the glb_policy->pending_picks list. To cancel these,
+//   we invoke the completion closure and set *target to NULL right here.
 static void glb_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
                                    grpc_connected_subchannel **target,
                                    grpc_error *error) {
@@ -1108,9 +1104,23 @@ static void glb_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     }
     pp = next;
   }
+  if (glb_policy->rr_policy != NULL) {
+    grpc_lb_policy_cancel_pick_locked(exec_ctx, glb_policy->rr_policy, target,
+                                      GRPC_ERROR_REF(error));
+  }
   GRPC_ERROR_UNREF(error);
 }
 
+// Cancel all pending picks.
+//
+// A grpclb pick progresses as follows:
+// - If there's a Round Robin policy (glb_policy->rr_policy) available, it'll be
+//   handed over to the RR policy (in create_rr_locked()). From that point
+//   onwards, it'll be RR's responsibility. For cancellations, that implies the
+//   pick needs also be cancelled by the RR instance.
+// - Otherwise, without an RR instance, picks stay pending at this policy's
+//   level (grpclb), inside the glb_policy->pending_picks list. To cancel these,
+//   we invoke the completion closure and set *target to NULL right here.
 static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx,
                                     grpc_lb_policy *pol,
                                     uint32_t initial_metadata_flags_mask,
@@ -1132,6 +1142,11 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx,
     }
     pp = next;
   }
+  if (glb_policy->rr_policy != NULL) {
+    grpc_lb_policy_cancel_picks_locked(
+        exec_ctx, glb_policy->rr_policy, initial_metadata_flags_mask,
+        initial_metadata_flags_eq, GRPC_ERROR_REF(error));
+  }
   GRPC_ERROR_UNREF(error);
 }
 
@@ -1286,15 +1301,14 @@ static void do_send_client_load_report_locked(grpc_exec_ctx *exec_ctx,
 }
 
 static bool load_report_counters_are_zero(grpc_grpclb_request *request) {
+  grpc_grpclb_dropped_call_counts *drop_entries =
+      request->client_stats.calls_finished_with_drop.arg;
   return request->client_stats.num_calls_started == 0 &&
          request->client_stats.num_calls_finished == 0 &&
-         request->client_stats.num_calls_finished_with_drop_for_rate_limiting ==
-             0 &&
-         request->client_stats
-                 .num_calls_finished_with_drop_for_load_balancing == 0 &&
          request->client_stats.num_calls_finished_with_client_failed_to_send ==
              0 &&
-         request->client_stats.num_calls_finished_known_received == 0;
+         request->client_stats.num_calls_finished_known_received == 0 &&
+         (drop_entries == NULL || drop_entries->num_entries == 0);
 }
 
 static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg,
@@ -1309,7 +1323,7 @@ static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg,
   // Construct message payload.
   GPR_ASSERT(glb_policy->client_load_report_payload == NULL);
   grpc_grpclb_request *request =
-      grpc_grpclb_load_report_request_create(glb_policy->client_stats);
+      grpc_grpclb_load_report_request_create_locked(glb_policy->client_stats);
   // Skip client load report if the counters were all zero in the last
   // report and they are still zero in this one.
   if (load_report_counters_are_zero(request)) {
@@ -1463,7 +1477,8 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
   op++;
   /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
    * count goes to zero) to be unref'd in lb_on_sent_initial_request_locked() */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_server_status_received");
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
+                          "lb_on_sent_initial_request_locked");
   call_error = grpc_call_start_batch_and_execute(
       exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_sent_initial_request);
@@ -1480,8 +1495,9 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
   op->reserved = NULL;
   op++;
   /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
-   * count goes to zero) to be unref'd in lb_on_server_status_received */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_server_status_received");
+   * count goes to zero) to be unref'd in lb_on_server_status_received_locked */
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
+                          "lb_on_server_status_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_server_status_received);
@@ -1493,8 +1509,9 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
   op->flags = 0;
   op->reserved = NULL;
   op++;
-  /* take another weak ref to be unref'd in lb_on_response_received */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_response_received");
+  /* take another weak ref to be unref'd/reused in
+   * lb_on_response_received_locked */
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_response_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_response_received);
@@ -1511,13 +1528,12 @@ static void lb_on_sent_initial_request_locked(grpc_exec_ctx *exec_ctx,
     do_send_client_load_report_locked(exec_ctx, glb_policy);
   }
   GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
-                            "lb_on_response_received_locked");
+                            "lb_on_sent_initial_request_locked");
 }
 
 static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                            grpc_error *error) {
   glb_lb_policy *glb_policy = arg;
-
   grpc_op ops[2];
   memset(ops, 0, sizeof(ops));
   grpc_op *op = ops;
@@ -1528,6 +1544,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_byte_buffer_reader bbr;
     grpc_byte_buffer_reader_init(&bbr, glb_policy->lb_response_payload);
     grpc_slice response_slice = grpc_byte_buffer_reader_readall(&bbr);
+    grpc_byte_buffer_reader_destroy(&bbr);
     grpc_byte_buffer_destroy(glb_policy->lb_response_payload);
 
     grpc_grpclb_initial_response *response = NULL;
@@ -1548,7 +1565,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
         }
         /* take a weak ref (won't prevent calling of \a glb_shutdown() if the
          * strong ref count goes to zero) to be unref'd in
-         * send_client_load_report() */
+         * send_client_load_report_locked() */
         glb_policy->client_load_report_timer_pending = true;
         GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report");
         schedule_next_client_load_report(exec_ctx, glb_policy);
@@ -1576,7 +1593,6 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
             gpr_free(ipport);
           }
         }
-
         /* update serverlist */
         if (serverlist->num_servers > 0) {
           if (grpc_grpclb_serverlist_equals(glb_policy->serverlist,
@@ -1611,9 +1627,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
                 grpc_dump_slice(response_slice, GPR_DUMP_ASCII | GPR_DUMP_HEX));
       }
     }
-
     grpc_slice_unref_internal(exec_ctx, response_slice);
-
     if (!glb_policy->shutting_down) {
       /* keep listening for serverlist updates */
       op->op = GRPC_OP_RECV_MESSAGE;
@@ -1621,7 +1635,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
       op->flags = 0;
       op->reserved = NULL;
       op++;
-      /* reuse the "lb_on_response_received" weak ref taken in
+      /* reuse the "lb_on_response_received_locked" weak ref taken in
        * query_for_backends_locked() */
       const grpc_call_error call_error = grpc_call_start_batch_and_execute(
           exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops),
@@ -1629,10 +1643,10 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
       GPR_ASSERT(GRPC_CALL_OK == call_error);
     }
   } else { /* empty payload: call cancelled. */
-           /* dispose of the "lb_on_response_received" weak ref taken in
+           /* dispose of the "lb_on_response_received_locked" weak ref taken in
             * query_for_backends_locked() and reused in every reception loop */
     GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
-                              "lb_on_response_received_empty_payload");
+                              "lb_on_response_received_locked_empty_payload");
   }
 }
 
@@ -1699,7 +1713,7 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx,
                     &glb_policy->lb_on_call_retry, now);
   }
   GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
-                            "lb_on_server_status_received");
+                            "lb_on_server_status_received_locked");
 }
 
 static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
@@ -1756,7 +1770,8 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
 
   if (!glb_policy->watching_lb_channel) {
     // Watch the LB channel connectivity for connection.
-    glb_policy->lb_channel_connectivity = GRPC_CHANNEL_INIT;
+    glb_policy->lb_channel_connectivity = grpc_channel_check_connectivity_state(
+        glb_policy->lb_channel, true /* try to connect */);
     grpc_channel_element *client_channel_elem = grpc_channel_stack_last_element(
         grpc_channel_get_channel_stack(glb_policy->lb_channel));
     GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c
index c762443b7c..5b62623145 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c
@@ -18,8 +18,11 @@
 
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h"
 
+#include <string.h>
+
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 
@@ -29,10 +32,11 @@
 
 struct grpc_grpclb_client_stats {
   gpr_refcount refs;
+  // This field must only be accessed via *_locked() methods.
+  grpc_grpclb_dropped_call_counts* drop_token_counts;
+  // These fields may be accessed from multiple threads at a time.
   gpr_atm num_calls_started;
   gpr_atm num_calls_finished;
-  gpr_atm num_calls_finished_with_drop_for_rate_limiting;
-  gpr_atm num_calls_finished_with_drop_for_load_balancing;
   gpr_atm num_calls_finished_with_client_failed_to_send;
   gpr_atm num_calls_finished_known_received;
 };
@@ -51,6 +55,7 @@ grpc_grpclb_client_stats* grpc_grpclb_client_stats_ref(
 
 void grpc_grpclb_client_stats_unref(grpc_grpclb_client_stats* client_stats) {
   if (gpr_unref(&client_stats->refs)) {
+    grpc_grpclb_dropped_call_counts_destroy(client_stats->drop_token_counts);
     gpr_free(client_stats);
   }
 }
@@ -61,21 +66,9 @@ void grpc_grpclb_client_stats_add_call_started(
 }
 
 void grpc_grpclb_client_stats_add_call_finished(
-    bool finished_with_drop_for_rate_limiting,
-    bool finished_with_drop_for_load_balancing,
     bool finished_with_client_failed_to_send, bool finished_known_received,
     grpc_grpclb_client_stats* client_stats) {
   gpr_atm_full_fetch_add(&client_stats->num_calls_finished, (gpr_atm)1);
-  if (finished_with_drop_for_rate_limiting) {
-    gpr_atm_full_fetch_add(
-        &client_stats->num_calls_finished_with_drop_for_rate_limiting,
-        (gpr_atm)1);
-  }
-  if (finished_with_drop_for_load_balancing) {
-    gpr_atm_full_fetch_add(
-        &client_stats->num_calls_finished_with_drop_for_load_balancing,
-        (gpr_atm)1);
-  }
   if (finished_with_client_failed_to_send) {
     gpr_atm_full_fetch_add(
         &client_stats->num_calls_finished_with_client_failed_to_send,
@@ -87,32 +80,70 @@ void grpc_grpclb_client_stats_add_call_finished(
   }
 }
 
+void grpc_grpclb_client_stats_add_call_dropped_locked(
+    char* token, grpc_grpclb_client_stats* client_stats) {
+  // Increment num_calls_started and num_calls_finished.
+  gpr_atm_full_fetch_add(&client_stats->num_calls_started, (gpr_atm)1);
+  gpr_atm_full_fetch_add(&client_stats->num_calls_finished, (gpr_atm)1);
+  // Record the drop.
+  if (client_stats->drop_token_counts == NULL) {
+    client_stats->drop_token_counts =
+        gpr_zalloc(sizeof(grpc_grpclb_dropped_call_counts));
+  }
+  grpc_grpclb_dropped_call_counts* drop_token_counts =
+      client_stats->drop_token_counts;
+  for (size_t i = 0; i < drop_token_counts->num_entries; ++i) {
+    if (strcmp(drop_token_counts->token_counts[i].token, token) == 0) {
+      ++drop_token_counts->token_counts[i].count;
+      return;
+    }
+  }
+  // Not found, so add a new entry.  We double the size of the array each time.
+  size_t new_num_entries = 2;
+  while (new_num_entries < drop_token_counts->num_entries + 1) {
+    new_num_entries *= 2;
+  }
+  drop_token_counts->token_counts =
+      gpr_realloc(drop_token_counts->token_counts,
+                  new_num_entries * sizeof(grpc_grpclb_drop_token_count));
+  grpc_grpclb_drop_token_count* new_entry =
+      &drop_token_counts->token_counts[drop_token_counts->num_entries++];
+  new_entry->token = gpr_strdup(token);
+  new_entry->count = 1;
+}
+
 static void atomic_get_and_reset_counter(int64_t* value, gpr_atm* counter) {
   *value = (int64_t)gpr_atm_acq_load(counter);
   gpr_atm_full_fetch_add(counter, (gpr_atm)(-*value));
 }
 
-void grpc_grpclb_client_stats_get(
+void grpc_grpclb_client_stats_get_locked(
     grpc_grpclb_client_stats* client_stats, int64_t* num_calls_started,
     int64_t* num_calls_finished,
-    int64_t* num_calls_finished_with_drop_for_rate_limiting,
-    int64_t* num_calls_finished_with_drop_for_load_balancing,
     int64_t* num_calls_finished_with_client_failed_to_send,
-    int64_t* num_calls_finished_known_received) {
+    int64_t* num_calls_finished_known_received,
+    grpc_grpclb_dropped_call_counts** drop_token_counts) {
   atomic_get_and_reset_counter(num_calls_started,
                                &client_stats->num_calls_started);
   atomic_get_and_reset_counter(num_calls_finished,
                                &client_stats->num_calls_finished);
   atomic_get_and_reset_counter(
-      num_calls_finished_with_drop_for_rate_limiting,
-      &client_stats->num_calls_finished_with_drop_for_rate_limiting);
-  atomic_get_and_reset_counter(
-      num_calls_finished_with_drop_for_load_balancing,
-      &client_stats->num_calls_finished_with_drop_for_load_balancing);
-  atomic_get_and_reset_counter(
       num_calls_finished_with_client_failed_to_send,
       &client_stats->num_calls_finished_with_client_failed_to_send);
   atomic_get_and_reset_counter(
       num_calls_finished_known_received,
       &client_stats->num_calls_finished_known_received);
+  *drop_token_counts = client_stats->drop_token_counts;
+  client_stats->drop_token_counts = NULL;
+}
+
+void grpc_grpclb_dropped_call_counts_destroy(
+    grpc_grpclb_dropped_call_counts* drop_entries) {
+  if (drop_entries != NULL) {
+    for (size_t i = 0; i < drop_entries->num_entries; ++i) {
+      gpr_free(drop_entries->token_counts[i].token);
+    }
+    gpr_free(drop_entries->token_counts);
+    gpr_free(drop_entries);
+  }
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h
index 4bb47d5c5c..c51e2a431a 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h
@@ -25,6 +25,16 @@
 
 typedef struct grpc_grpclb_client_stats grpc_grpclb_client_stats;
 
+typedef struct {
+  char* token;
+  int64_t count;
+} grpc_grpclb_drop_token_count;
+
+typedef struct {
+  grpc_grpclb_drop_token_count* token_counts;
+  size_t num_entries;
+} grpc_grpclb_dropped_call_counts;
+
 grpc_grpclb_client_stats* grpc_grpclb_client_stats_create();
 grpc_grpclb_client_stats* grpc_grpclb_client_stats_ref(
     grpc_grpclb_client_stats* client_stats);
@@ -33,18 +43,23 @@ void grpc_grpclb_client_stats_unref(grpc_grpclb_client_stats* client_stats);
 void grpc_grpclb_client_stats_add_call_started(
     grpc_grpclb_client_stats* client_stats);
 void grpc_grpclb_client_stats_add_call_finished(
-    bool finished_with_drop_for_rate_limiting,
-    bool finished_with_drop_for_load_balancing,
     bool finished_with_client_failed_to_send, bool finished_known_received,
     grpc_grpclb_client_stats* client_stats);
 
-void grpc_grpclb_client_stats_get(
+// This method is not thread-safe; caller must synchronize.
+void grpc_grpclb_client_stats_add_call_dropped_locked(
+    char* token, grpc_grpclb_client_stats* client_stats);
+
+// This method is not thread-safe; caller must synchronize.
+void grpc_grpclb_client_stats_get_locked(
     grpc_grpclb_client_stats* client_stats, int64_t* num_calls_started,
     int64_t* num_calls_finished,
-    int64_t* num_calls_finished_with_drop_for_rate_limiting,
-    int64_t* num_calls_finished_with_drop_for_load_balancing,
     int64_t* num_calls_finished_with_client_failed_to_send,
-    int64_t* num_calls_finished_known_received);
+    int64_t* num_calls_finished_known_received,
+    grpc_grpclb_dropped_call_counts** drop_token_counts);
+
+void grpc_grpclb_dropped_call_counts_destroy(
+    grpc_grpclb_dropped_call_counts* drop_entries);
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_CLIENT_STATS_H \
           */
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c
index bec7c97a78..6fa29f326e 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c
@@ -76,7 +76,33 @@ static void populate_timestamp(gpr_timespec timestamp,
   timestamp_pb->nanos = timestamp.tv_nsec;
 }
 
-grpc_grpclb_request *grpc_grpclb_load_report_request_create(
+static bool encode_string(pb_ostream_t *stream, const pb_field_t *field,
+                          void *const *arg) {
+  char *str = *arg;
+  if (!pb_encode_tag_for_field(stream, field)) return false;
+  return pb_encode_string(stream, (uint8_t *)str, strlen(str));
+}
+
+static bool encode_drops(pb_ostream_t *stream, const pb_field_t *field,
+                         void *const *arg) {
+  grpc_grpclb_dropped_call_counts *drop_entries = *arg;
+  if (drop_entries == NULL) return true;
+  for (size_t i = 0; i < drop_entries->num_entries; ++i) {
+    if (!pb_encode_tag_for_field(stream, field)) return false;
+    grpc_lb_v1_ClientStatsPerToken drop_message;
+    drop_message.load_balance_token.funcs.encode = encode_string;
+    drop_message.load_balance_token.arg = drop_entries->token_counts[i].token;
+    drop_message.has_num_calls = true;
+    drop_message.num_calls = drop_entries->token_counts[i].count;
+    if (!pb_encode_submessage(stream, grpc_lb_v1_ClientStatsPerToken_fields,
+                              &drop_message)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+grpc_grpclb_request *grpc_grpclb_load_report_request_create_locked(
     grpc_grpclb_client_stats *client_stats) {
   grpc_grpclb_request *req = gpr_zalloc(sizeof(grpc_grpclb_request));
   req->has_client_stats = true;
@@ -84,18 +110,17 @@ grpc_grpclb_request *grpc_grpclb_load_report_request_create(
   populate_timestamp(gpr_now(GPR_CLOCK_REALTIME), &req->client_stats.timestamp);
   req->client_stats.has_num_calls_started = true;
   req->client_stats.has_num_calls_finished = true;
-  req->client_stats.has_num_calls_finished_with_drop_for_rate_limiting = true;
-  req->client_stats.has_num_calls_finished_with_drop_for_load_balancing = true;
   req->client_stats.has_num_calls_finished_with_client_failed_to_send = true;
   req->client_stats.has_num_calls_finished_with_client_failed_to_send = true;
   req->client_stats.has_num_calls_finished_known_received = true;
-  grpc_grpclb_client_stats_get(
+  req->client_stats.calls_finished_with_drop.funcs.encode = encode_drops;
+  grpc_grpclb_client_stats_get_locked(
       client_stats, &req->client_stats.num_calls_started,
       &req->client_stats.num_calls_finished,
-      &req->client_stats.num_calls_finished_with_drop_for_rate_limiting,
-      &req->client_stats.num_calls_finished_with_drop_for_load_balancing,
       &req->client_stats.num_calls_finished_with_client_failed_to_send,
-      &req->client_stats.num_calls_finished_known_received);
+      &req->client_stats.num_calls_finished_known_received,
+      (grpc_grpclb_dropped_call_counts **)&req->client_stats
+          .calls_finished_with_drop.arg);
   return req;
 }
 
@@ -117,6 +142,11 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) {
 }
 
 void grpc_grpclb_request_destroy(grpc_grpclb_request *request) {
+  if (request->has_client_stats) {
+    grpc_grpclb_dropped_call_counts *drop_entries =
+        request->client_stats.calls_finished_with_drop.arg;
+    grpc_grpclb_dropped_call_counts_destroy(drop_entries);
+  }
   gpr_free(request);
 }
 
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h
index ef8d563edc..c4a98492c9 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h
@@ -44,7 +44,7 @@ typedef struct {
 
 /** Create a request for a gRPC LB service under \a lb_service_name */
 grpc_grpclb_request *grpc_grpclb_request_create(const char *lb_service_name);
-grpc_grpclb_request *grpc_grpclb_load_report_request_create(
+grpc_grpclb_request *grpc_grpclb_load_report_request_create_locked(
     grpc_grpclb_client_stats *client_stats);
 
 /** Protocol Buffers v3-encode \a request */
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
index fb119c7fc8..6a5d54c82a 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
@@ -33,14 +33,19 @@ const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2] = {
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v1_ClientStats_fields[8] = {
+const pb_field_t grpc_lb_v1_ClientStatsPerToken_fields[3] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, grpc_lb_v1_ClientStatsPerToken, load_balance_token, load_balance_token, 0),
+    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStatsPerToken, num_calls, load_balance_token, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v1_ClientStats_fields[7] = {
     PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_ClientStats, timestamp, timestamp, &grpc_lb_v1_Timestamp_fields),
     PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_started, timestamp, 0),
     PB_FIELD(  3, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_finished, num_calls_started, 0),
-    PB_FIELD(  4, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_drop_for_rate_limiting, num_calls_finished, 0),
-    PB_FIELD(  5, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_drop_for_load_balancing, num_calls_finished_with_drop_for_rate_limiting, 0),
-    PB_FIELD(  6, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_client_failed_to_send, num_calls_finished_with_drop_for_load_balancing, 0),
+    PB_FIELD(  6, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_client_failed_to_send, num_calls_finished, 0),
     PB_FIELD(  7, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_known_received, num_calls_finished_with_client_failed_to_send, 0),
+    PB_FIELD(  8, MESSAGE , REPEATED, CALLBACK, OTHER, grpc_lb_v1_ClientStats, calls_finished_with_drop, num_calls_finished_known_received, &grpc_lb_v1_ClientStatsPerToken_fields),
     PB_LAST_FIELD
 };
 
@@ -62,12 +67,11 @@ const pb_field_t grpc_lb_v1_ServerList_fields[3] = {
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v1_Server_fields[6] = {
+const pb_field_t grpc_lb_v1_Server_fields[5] = {
     PB_FIELD(  1, BYTES   , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0),
     PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, port, ip_address, 0),
     PB_FIELD(  3, STRING  , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, load_balance_token, port, 0),
-    PB_FIELD(  4, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, drop_for_rate_limiting, load_balance_token, 0),
-    PB_FIELD(  5, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, drop_for_load_balancing, drop_for_rate_limiting, 0),
+    PB_FIELD(  4, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, drop, load_balance_token, 0),
     PB_LAST_FIELD
 };
 
@@ -81,7 +85,7 @@ const pb_field_t grpc_lb_v1_Server_fields[6] = {
  * numbers or field sizes that are larger than what can fit in 8 or 16 bit
  * field descriptors.
  */
-PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v1_ClientStats, timestamp) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v1_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_Timestamp_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server)
+PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v1_ClientStats, timestamp) < 65536 && pb_membersize(grpc_lb_v1_ClientStats, calls_finished_with_drop) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v1_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_Timestamp_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStatsPerToken_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server)
 #endif
 
 #if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
@@ -92,7 +96,7 @@ PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request)
  * numbers or field sizes that are larger than what can fit in the default
  * 8 bit descriptors.
  */
-PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v1_ClientStats, timestamp) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v1_ServerList, servers) < 256 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_Timestamp_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server)
+PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v1_ClientStats, timestamp) < 256 && pb_membersize(grpc_lb_v1_ClientStats, calls_finished_with_drop) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v1_ServerList, servers) < 256 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_Timestamp_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStatsPerToken_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server)
 #endif
 
 
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
index d3ae919ec2..93333d1aed 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
@@ -14,6 +14,13 @@ extern "C" {
 #endif
 
 /* Struct definitions */
+typedef struct _grpc_lb_v1_ClientStatsPerToken {
+    pb_callback_t load_balance_token;
+    bool has_num_calls;
+    int64_t num_calls;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_ClientStatsPerToken) */
+} grpc_lb_v1_ClientStatsPerToken;
+
 typedef struct _grpc_lb_v1_Duration {
     bool has_seconds;
     int64_t seconds;
@@ -36,10 +43,8 @@ typedef struct _grpc_lb_v1_Server {
     int32_t port;
     bool has_load_balance_token;
     char load_balance_token[50];
-    bool has_drop_for_rate_limiting;
-    bool drop_for_rate_limiting;
-    bool has_drop_for_load_balancing;
-    bool drop_for_load_balancing;
+    bool has_drop;
+    bool drop;
 /* @@protoc_insertion_point(struct:grpc_lb_v1_Server) */
 } grpc_lb_v1_Server;
 
@@ -58,14 +63,11 @@ typedef struct _grpc_lb_v1_ClientStats {
     int64_t num_calls_started;
     bool has_num_calls_finished;
     int64_t num_calls_finished;
-    bool has_num_calls_finished_with_drop_for_rate_limiting;
-    int64_t num_calls_finished_with_drop_for_rate_limiting;
-    bool has_num_calls_finished_with_drop_for_load_balancing;
-    int64_t num_calls_finished_with_drop_for_load_balancing;
     bool has_num_calls_finished_with_client_failed_to_send;
     int64_t num_calls_finished_with_client_failed_to_send;
     bool has_num_calls_finished_known_received;
     int64_t num_calls_finished_known_received;
+    pb_callback_t calls_finished_with_drop;
 /* @@protoc_insertion_point(struct:grpc_lb_v1_ClientStats) */
 } grpc_lb_v1_ClientStats;
 
@@ -107,39 +109,41 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse {
 #define grpc_lb_v1_Timestamp_init_default        {false, 0, false, 0}
 #define grpc_lb_v1_LoadBalanceRequest_init_default {false, grpc_lb_v1_InitialLoadBalanceRequest_init_default, false, grpc_lb_v1_ClientStats_init_default}
 #define grpc_lb_v1_InitialLoadBalanceRequest_init_default {false, ""}
-#define grpc_lb_v1_ClientStats_init_default      {false, grpc_lb_v1_Timestamp_init_default, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0}
+#define grpc_lb_v1_ClientStatsPerToken_init_default {{{NULL}, NULL}, false, 0}
+#define grpc_lb_v1_ClientStats_init_default      {false, grpc_lb_v1_Timestamp_init_default, false, 0, false, 0, false, 0, false, 0, {{NULL}, NULL}}
 #define grpc_lb_v1_LoadBalanceResponse_init_default {false, grpc_lb_v1_InitialLoadBalanceResponse_init_default, false, grpc_lb_v1_ServerList_init_default}
 #define grpc_lb_v1_InitialLoadBalanceResponse_init_default {false, "", false, grpc_lb_v1_Duration_init_default}
 #define grpc_lb_v1_ServerList_init_default       {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_default}
-#define grpc_lb_v1_Server_init_default           {false, {0, {0}}, false, 0, false, "", false, 0, false, 0}
+#define grpc_lb_v1_Server_init_default           {false, {0, {0}}, false, 0, false, "", false, 0}
 #define grpc_lb_v1_Duration_init_zero            {false, 0, false, 0}
 #define grpc_lb_v1_Timestamp_init_zero           {false, 0, false, 0}
 #define grpc_lb_v1_LoadBalanceRequest_init_zero  {false, grpc_lb_v1_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v1_ClientStats_init_zero}
 #define grpc_lb_v1_InitialLoadBalanceRequest_init_zero {false, ""}
-#define grpc_lb_v1_ClientStats_init_zero         {false, grpc_lb_v1_Timestamp_init_zero, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0}
+#define grpc_lb_v1_ClientStatsPerToken_init_zero {{{NULL}, NULL}, false, 0}
+#define grpc_lb_v1_ClientStats_init_zero         {false, grpc_lb_v1_Timestamp_init_zero, false, 0, false, 0, false, 0, false, 0, {{NULL}, NULL}}
 #define grpc_lb_v1_LoadBalanceResponse_init_zero {false, grpc_lb_v1_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v1_ServerList_init_zero}
 #define grpc_lb_v1_InitialLoadBalanceResponse_init_zero {false, "", false, grpc_lb_v1_Duration_init_zero}
 #define grpc_lb_v1_ServerList_init_zero          {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_zero}
-#define grpc_lb_v1_Server_init_zero              {false, {0, {0}}, false, 0, false, "", false, 0, false, 0}
+#define grpc_lb_v1_Server_init_zero              {false, {0, {0}}, false, 0, false, "", false, 0}
 
 /* Field tags (for use in manual encoding/decoding) */
+#define grpc_lb_v1_ClientStatsPerToken_load_balance_token_tag 1
+#define grpc_lb_v1_ClientStatsPerToken_num_calls_tag 2
 #define grpc_lb_v1_Duration_seconds_tag          1
 #define grpc_lb_v1_Duration_nanos_tag            2
 #define grpc_lb_v1_InitialLoadBalanceRequest_name_tag 1
 #define grpc_lb_v1_Server_ip_address_tag         1
 #define grpc_lb_v1_Server_port_tag               2
 #define grpc_lb_v1_Server_load_balance_token_tag 3
-#define grpc_lb_v1_Server_drop_for_rate_limiting_tag 4
-#define grpc_lb_v1_Server_drop_for_load_balancing_tag 5
+#define grpc_lb_v1_Server_drop_tag               4
 #define grpc_lb_v1_Timestamp_seconds_tag         1
 #define grpc_lb_v1_Timestamp_nanos_tag           2
 #define grpc_lb_v1_ClientStats_timestamp_tag     1
 #define grpc_lb_v1_ClientStats_num_calls_started_tag 2
 #define grpc_lb_v1_ClientStats_num_calls_finished_tag 3
-#define grpc_lb_v1_ClientStats_num_calls_finished_with_drop_for_rate_limiting_tag 4
-#define grpc_lb_v1_ClientStats_num_calls_finished_with_drop_for_load_balancing_tag 5
 #define grpc_lb_v1_ClientStats_num_calls_finished_with_client_failed_to_send_tag 6
 #define grpc_lb_v1_ClientStats_num_calls_finished_known_received_tag 7
+#define grpc_lb_v1_ClientStats_calls_finished_with_drop_tag 8
 #define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 1
 #define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 2
 #define grpc_lb_v1_ServerList_servers_tag        1
@@ -154,22 +158,24 @@ extern const pb_field_t grpc_lb_v1_Duration_fields[3];
 extern const pb_field_t grpc_lb_v1_Timestamp_fields[3];
 extern const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3];
 extern const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2];
-extern const pb_field_t grpc_lb_v1_ClientStats_fields[8];
+extern const pb_field_t grpc_lb_v1_ClientStatsPerToken_fields[3];
+extern const pb_field_t grpc_lb_v1_ClientStats_fields[7];
 extern const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3];
 extern const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3];
 extern const pb_field_t grpc_lb_v1_ServerList_fields[3];
-extern const pb_field_t grpc_lb_v1_Server_fields[6];
+extern const pb_field_t grpc_lb_v1_Server_fields[5];
 
 /* Maximum encoded size of messages (where known) */
 #define grpc_lb_v1_Duration_size                 22
 #define grpc_lb_v1_Timestamp_size                22
-#define grpc_lb_v1_LoadBalanceRequest_size       226
+#define grpc_lb_v1_LoadBalanceRequest_size       (140 + grpc_lb_v1_ClientStats_size)
 #define grpc_lb_v1_InitialLoadBalanceRequest_size 131
-#define grpc_lb_v1_ClientStats_size              90
+/* grpc_lb_v1_ClientStatsPerToken_size depends on runtime parameters */
+/* grpc_lb_v1_ClientStats_size depends on runtime parameters */
 #define grpc_lb_v1_LoadBalanceResponse_size      (98 + grpc_lb_v1_ServerList_size)
 #define grpc_lb_v1_InitialLoadBalanceResponse_size 90
 /* grpc_lb_v1_ServerList_size depends on runtime parameters */
-#define grpc_lb_v1_Server_size                   85
+#define grpc_lb_v1_Server_size                   83
 
 /* Message IDs (where set with "msgid" option) */
 #ifdef PB_MSGID
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
index 341763a4d7..110a9c8047 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
@@ -420,6 +420,7 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
                           grpc_call_context_element *context, void **user_data,
                           grpc_closure *on_complete) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
+  GPR_ASSERT(!p->shutdown);
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] Trying to pick", (void *)pol);
   }
@@ -532,6 +533,7 @@ static grpc_connectivity_state update_lb_connectivity_status_locked(
     grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                 GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error),
                                 "rr_shutdown");
+    p->shutdown = true;
     new_state = GRPC_CHANNEL_SHUTDOWN;
   } else if (subchannel_list->num_transient_failures ==
              p->subchannel_list->num_subchannels) { /* 4) TRANSIENT_FAILURE */
@@ -584,10 +586,16 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
   // Dispose of outdated subchannel lists.
   if (sd->subchannel_list != p->subchannel_list &&
       sd->subchannel_list != p->latest_pending_subchannel_list) {
-    // sd belongs to an outdated subchannel_list: get rid of it.
-    rr_subchannel_list_shutdown_and_unref(exec_ctx, sd->subchannel_list,
-                                          "sl_outdated");
-    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "sl_outdated");
+    char *reason = NULL;
+    if (sd->subchannel_list->shutting_down) {
+      reason = "sl_outdated_straggler";
+      rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, reason);
+    } else {
+      reason = "sl_outdated";
+      rr_subchannel_list_shutdown_and_unref(exec_ctx, sd->subchannel_list,
+                                            reason);
+    }
+    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, reason);
     return;
   }
   // Now that we're inside the combiner, copy the pending connectivity
@@ -753,6 +761,7 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
   for (size_t i = 0; i < addresses->num_addresses; i++) {
     if (!addresses->addresses[i].is_balancer) ++num_addrs;
   }
+  rr_subchannel_list *subchannel_list = rr_subchannel_list_create(p, num_addrs);
   if (num_addrs == 0) {
     grpc_connectivity_state_set(
         exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
@@ -761,18 +770,16 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     if (p->subchannel_list != NULL) {
       rr_subchannel_list_shutdown_and_unref(exec_ctx, p->subchannel_list,
                                             "sl_shutdown_empty_update");
-      p->subchannel_list = NULL;
     }
+    p->subchannel_list = subchannel_list;  // empty list
     return;
   }
   size_t subchannel_index = 0;
-  rr_subchannel_list *subchannel_list = rr_subchannel_list_create(p, num_addrs);
   if (p->latest_pending_subchannel_list != NULL && p->started_picking) {
     if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
       gpr_log(GPR_DEBUG,
               "[RR %p] Shutting down latest pending subchannel list %p, about "
-              "to be "
-              "replaced by newer latest %p",
+              "to be replaced by newer latest %p",
               (void *)p, (void *)p->latest_pending_subchannel_list,
               (void *)subchannel_list);
     }
@@ -800,19 +807,30 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     sc_args.args = new_args;
     grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel(
         exec_ctx, args->client_channel_factory, &sc_args);
+    grpc_channel_args_destroy(exec_ctx, new_args);
+    grpc_error *error;
+    // Get the connectivity state of the subchannel. Already existing ones may
+    // be in a state other than INIT.
+    const grpc_connectivity_state subchannel_connectivity_state =
+        grpc_subchannel_check_connectivity(subchannel, &error);
+    if (error != GRPC_ERROR_NONE) {
+      // The subchannel is in error (e.g. shutting down). Ignore it.
+      GRPC_SUBCHANNEL_UNREF(exec_ctx, subchannel, "new_sc_connectivity_error");
+      GRPC_ERROR_UNREF(error);
+      continue;
+    }
     if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
       char *address_uri =
           grpc_sockaddr_to_uri(&addresses->addresses[i].address);
       gpr_log(
           GPR_DEBUG,
           "[RR %p] index %lu: Created subchannel %p for address uri %s into "
-          "subchannel_list %p",
+          "subchannel_list %p. Connectivity state %s",
           (void *)p, (unsigned long)subchannel_index, (void *)subchannel,
-          address_uri, (void *)subchannel_list);
+          address_uri, (void *)subchannel_list,
+          grpc_connectivity_state_name(subchannel_connectivity_state));
       gpr_free(address_uri);
     }
-    grpc_channel_args_destroy(exec_ctx, new_args);
-
     subchannel_data *sd = &subchannel_list->subchannels[subchannel_index++];
     sd->subchannel_list = subchannel_list;
     sd->subchannel = subchannel;
@@ -824,7 +842,7 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
      * won't be referring to this value again and it'll be overwritten after
      * the first call to rr_connectivity_changed_locked */
     sd->prev_connectivity_state = GRPC_CHANNEL_INIT;
-    sd->curr_connectivity_state = GRPC_CHANNEL_IDLE;
+    sd->curr_connectivity_state = subchannel_connectivity_state;
     sd->user_data_vtable = addresses->user_data_vtable;
     if (sd->user_data_vtable != NULL) {
       sd->user_data =
@@ -876,10 +894,10 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
                                           grpc_lb_policy_args *args) {
   GPR_ASSERT(args->client_channel_factory != NULL);
   round_robin_lb_policy *p = gpr_zalloc(sizeof(*p));
-  rr_update_locked(exec_ctx, &p->base, args);
   grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable, args->combiner);
   grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
                                "round_robin");
+  rr_update_locked(exec_ctx, &p->base, args);
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_DEBUG, "[RR %p] Created with %lu subchannels", (void *)p,
             (unsigned long)p->subchannel_list->num_subchannels);
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c
index 04a7852323..f1480bb1ae 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c
@@ -19,7 +19,10 @@
 #include <grpc/support/port_platform.h>
 #if GRPC_ARES == 1 && !defined(GRPC_UV)
 
+#include <limits.h>
+#include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
@@ -31,11 +34,14 @@
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/combiner.h"
+#include "src/core/lib/iomgr/gethostname.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/json/json.h"
 #include "src/core/lib/support/backoff.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/service_config.h"
 
 #define GRPC_DNS_MIN_CONNECT_TIMEOUT_SECONDS 1
 #define GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS 1
@@ -54,6 +60,8 @@ typedef struct {
   char *default_port;
   /** channel args. */
   grpc_channel_args *channel_args;
+  /** whether to request the service config */
+  bool request_service_config;
   /** pollset_set to drive the name resolution process */
   grpc_pollset_set *interested_parties;
 
@@ -85,6 +93,8 @@ typedef struct {
 
   /** currently resolving addresses */
   grpc_lb_addresses *lb_addresses;
+  /** currently resolving service config */
+  char *service_config_json;
 } ares_dns_resolver;
 
 static void dns_ares_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
@@ -144,6 +154,77 @@ static void dns_ares_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
   GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer");
 }
 
+static bool value_in_json_array(grpc_json *array, const char *value) {
+  for (grpc_json *entry = array->child; entry != NULL; entry = entry->next) {
+    if (entry->type == GRPC_JSON_STRING && strcmp(entry->value, value) == 0) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static char *choose_service_config(char *service_config_choice_json) {
+  grpc_json *choices_json = grpc_json_parse_string(service_config_choice_json);
+  if (choices_json == NULL || choices_json->type != GRPC_JSON_ARRAY) {
+    gpr_log(GPR_ERROR, "cannot parse service config JSON string");
+    return NULL;
+  }
+  char *service_config = NULL;
+  for (grpc_json *choice = choices_json->child; choice != NULL;
+       choice = choice->next) {
+    if (choice->type != GRPC_JSON_OBJECT) {
+      gpr_log(GPR_ERROR, "cannot parse service config JSON string");
+      break;
+    }
+    grpc_json *service_config_json = NULL;
+    for (grpc_json *field = choice->child; field != NULL; field = field->next) {
+      // Check client language, if specified.
+      if (strcmp(field->key, "clientLanguage") == 0) {
+        if (field->type != GRPC_JSON_ARRAY ||
+            !value_in_json_array(field, "c++")) {
+          service_config_json = NULL;
+          break;
+        }
+      }
+      // Check client hostname, if specified.
+      if (strcmp(field->key, "clientHostname") == 0) {
+        char *hostname = grpc_gethostname();
+        if (hostname == NULL || field->type != GRPC_JSON_ARRAY ||
+            !value_in_json_array(field, hostname)) {
+          service_config_json = NULL;
+          break;
+        }
+      }
+      // Check percentage, if specified.
+      if (strcmp(field->key, "percentage") == 0) {
+        if (field->type != GRPC_JSON_NUMBER) {
+          service_config_json = NULL;
+          break;
+        }
+        int random_pct = rand() % 100;
+        int percentage;
+        if (sscanf(field->value, "%d", &percentage) != 1 ||
+            random_pct > percentage) {
+          service_config_json = NULL;
+          break;
+        }
+      }
+      // Save service config.
+      if (strcmp(field->key, "serviceConfig") == 0) {
+        if (field->type == GRPC_JSON_OBJECT) {
+          service_config_json = field;
+        }
+      }
+    }
+    if (service_config_json != NULL) {
+      service_config = grpc_json_dump_to_string(service_config_json, 0);
+      break;
+    }
+  }
+  grpc_json_destroy(choices_json);
+  return service_config;
+}
+
 static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                         grpc_error *error) {
   ares_dns_resolver *r = arg;
@@ -152,8 +233,40 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg,
   r->resolving = false;
   r->pending_request = NULL;
   if (r->lb_addresses != NULL) {
-    grpc_arg new_arg = grpc_lb_addresses_create_channel_arg(r->lb_addresses);
-    result = grpc_channel_args_copy_and_add(r->channel_args, &new_arg, 1);
+    static const char *args_to_remove[2];
+    size_t num_args_to_remove = 0;
+    grpc_arg new_args[3];
+    size_t num_args_to_add = 0;
+    new_args[num_args_to_add++] =
+        grpc_lb_addresses_create_channel_arg(r->lb_addresses);
+    grpc_service_config *service_config = NULL;
+    char *service_config_string = NULL;
+    if (r->service_config_json != NULL) {
+      service_config_string = choose_service_config(r->service_config_json);
+      gpr_free(r->service_config_json);
+      if (service_config_string != NULL) {
+        gpr_log(GPR_INFO, "selected service config choice: %s",
+                service_config_string);
+        args_to_remove[num_args_to_remove++] = GRPC_ARG_SERVICE_CONFIG;
+        new_args[num_args_to_add++] = grpc_channel_arg_string_create(
+            GRPC_ARG_SERVICE_CONFIG, service_config_string);
+        service_config = grpc_service_config_create(service_config_string);
+        if (service_config != NULL) {
+          const char *lb_policy_name =
+              grpc_service_config_get_lb_policy_name(service_config);
+          if (lb_policy_name != NULL) {
+            args_to_remove[num_args_to_remove++] = GRPC_ARG_LB_POLICY_NAME;
+            new_args[num_args_to_add++] = grpc_channel_arg_string_create(
+                GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name);
+          }
+        }
+      }
+    }
+    result = grpc_channel_args_copy_and_add_and_remove(
+        r->channel_args, args_to_remove, num_args_to_remove, new_args,
+        num_args_to_add);
+    if (service_config != NULL) grpc_service_config_destroy(service_config);
+    gpr_free(service_config_string);
     grpc_lb_addresses_destroy(exec_ctx, r->lb_addresses);
   } else {
     const char *msg = grpc_error_string(error);
@@ -207,10 +320,12 @@ static void dns_ares_start_resolving_locked(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(!r->resolving);
   r->resolving = true;
   r->lb_addresses = NULL;
+  r->service_config_json = NULL;
   r->pending_request = grpc_dns_lookup_ares(
       exec_ctx, r->dns_server, r->name_to_resolve, r->default_port,
       r->interested_parties, &r->dns_ares_on_resolved_locked, &r->lb_addresses,
-      true /* check_grpclb */);
+      true /* check_grpclb */,
+      r->request_service_config ? &r->service_config_json : NULL);
 }
 
 static void dns_ares_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
@@ -256,6 +371,10 @@ static grpc_resolver *dns_ares_create(grpc_exec_ctx *exec_ctx,
   r->name_to_resolve = gpr_strdup(path);
   r->default_port = gpr_strdup(default_port);
   r->channel_args = grpc_channel_args_copy(args->args);
+  const grpc_arg *arg = grpc_channel_args_find(
+      r->channel_args, GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION);
+  r->request_service_config = !grpc_channel_arg_get_integer(
+      arg, (grpc_integer_options){false, false, true});
   r->interested_parties = grpc_pollset_set_create();
   if (args->pollset_set != NULL) {
     grpc_pollset_set_add_pollset_set(exec_ctx, r->interested_parties,
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c
index 1ab8295e9e..b696344eab 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c
@@ -103,10 +103,9 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) {
   grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->grpc_fd);
   /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up
      immediately by another thread, and should not be closed by the following
-     grpc_fd_orphan. To prevent this fd from being closed by grpc_fd_orphan,
-     a fd pointer is provided. */
-  int fd;
-  grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, &fd, "c-ares query finished");
+     grpc_fd_orphan. */
+  grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, NULL, true /* already_closed */,
+                 "c-ares query finished");
   gpr_free(fdn);
 }
 
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
index 9065e33613..e65723a63b 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
@@ -33,13 +33,13 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include <nameser.h>
 
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/nameser.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/support/string.h"
 
@@ -54,6 +54,8 @@ struct grpc_ares_request {
   grpc_closure *on_done;
   /** the pointer to receive the resolved addresses */
   grpc_lb_addresses **lb_addrs_out;
+  /** the pointer to receive the service config in JSON */
+  char **service_config_json_out;
   /** the evernt driver used by this request */
   grpc_ares_ev_driver *ev_driver;
   /** number of ongoing queries */
@@ -266,10 +268,68 @@ static void on_srv_query_done_cb(void *arg, int status, int timeouts,
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
+static const char g_service_config_attribute_prefix[] = "grpc_config=";
+
+static void on_txt_done_cb(void *arg, int status, int timeouts,
+                           unsigned char *buf, int len) {
+  gpr_log(GPR_DEBUG, "on_txt_done_cb");
+  char *error_msg;
+  grpc_ares_request *r = (grpc_ares_request *)arg;
+  gpr_mu_lock(&r->mu);
+  if (status != ARES_SUCCESS) goto fail;
+  struct ares_txt_ext *reply = NULL;
+  status = ares_parse_txt_reply_ext(buf, len, &reply);
+  if (status != ARES_SUCCESS) goto fail;
+  // Find service config in TXT record.
+  const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1;
+  struct ares_txt_ext *result;
+  for (result = reply; result != NULL; result = result->next) {
+    if (result->record_start &&
+        memcmp(result->txt, g_service_config_attribute_prefix, prefix_len) ==
+            0) {
+      break;
+    }
+  }
+  // Found a service config record.
+  if (result != NULL) {
+    size_t service_config_len = result->length - prefix_len;
+    *r->service_config_json_out = gpr_malloc(service_config_len + 1);
+    memcpy(*r->service_config_json_out, result->txt + prefix_len,
+           service_config_len);
+    for (result = result->next; result != NULL && !result->record_start;
+         result = result->next) {
+      *r->service_config_json_out = gpr_realloc(
+          *r->service_config_json_out, service_config_len + result->length + 1);
+      memcpy(*r->service_config_json_out + service_config_len, result->txt,
+             result->length);
+      service_config_len += result->length;
+    }
+    (*r->service_config_json_out)[service_config_len] = '\0';
+    gpr_log(GPR_INFO, "found service config: %s", *r->service_config_json_out);
+  }
+  // Clean up.
+  ares_free_data(reply);
+  goto done;
+fail:
+  gpr_asprintf(&error_msg, "C-ares TXT lookup status is not ARES_SUCCESS: %s",
+               ares_strerror(status));
+  grpc_error *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg);
+  gpr_free(error_msg);
+  if (r->error == GRPC_ERROR_NONE) {
+    r->error = error;
+  } else {
+    r->error = grpc_error_add_child(error, r->error);
+  }
+done:
+  gpr_mu_unlock(&r->mu);
+  grpc_ares_request_unref(NULL, r);
+}
+
 static grpc_ares_request *grpc_dns_lookup_ares_impl(
     grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name,
     const char *default_port, grpc_pollset_set *interested_parties,
-    grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb) {
+    grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb,
+    char **service_config_json) {
   grpc_error *error = GRPC_ERROR_NONE;
   /* TODO(zyc): Enable tracing after #9603 is checked in */
   /* if (grpc_dns_trace) {
@@ -300,11 +360,12 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl(
   error = grpc_ares_ev_driver_create(&ev_driver, interested_parties);
   if (error != GRPC_ERROR_NONE) goto error_cleanup;
 
-  grpc_ares_request *r = gpr_malloc(sizeof(grpc_ares_request));
+  grpc_ares_request *r = gpr_zalloc(sizeof(grpc_ares_request));
   gpr_mu_init(&r->mu);
   r->ev_driver = ev_driver;
   r->on_done = on_done;
   r->lb_addrs_out = addrs;
+  r->service_config_json_out = service_config_json;
   r->success = false;
   r->error = GRPC_ERROR_NONE;
   ares_channel *channel = grpc_ares_ev_driver_get_channel(r->ev_driver);
@@ -315,13 +376,17 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl(
     grpc_resolved_address addr;
     if (grpc_parse_ipv4_hostport(dns_server, &addr, false /* log_errors */)) {
       r->dns_server_addr.family = AF_INET;
-      memcpy(&r->dns_server_addr.addr.addr4, addr.addr, addr.len);
+      struct sockaddr_in *in = (struct sockaddr_in *)addr.addr;
+      memcpy(&r->dns_server_addr.addr.addr4, &in->sin_addr,
+             sizeof(struct in_addr));
       r->dns_server_addr.tcp_port = grpc_sockaddr_get_port(&addr);
       r->dns_server_addr.udp_port = grpc_sockaddr_get_port(&addr);
     } else if (grpc_parse_ipv6_hostport(dns_server, &addr,
                                         false /* log_errors */)) {
       r->dns_server_addr.family = AF_INET6;
-      memcpy(&r->dns_server_addr.addr.addr6, addr.addr, addr.len);
+      struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)addr.addr;
+      memcpy(&r->dns_server_addr.addr.addr6, &in6->sin6_addr,
+             sizeof(struct in6_addr));
       r->dns_server_addr.tcp_port = grpc_sockaddr_get_port(&addr);
       r->dns_server_addr.udp_port = grpc_sockaddr_get_port(&addr);
     } else {
@@ -342,8 +407,6 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl(
       goto error_cleanup;
     }
   }
-  // An extra reference is put here to avoid destroying the request in
-  // on_done_cb before calling grpc_ares_ev_driver_start.
   gpr_ref_init(&r->pending_queries, 1);
   if (grpc_ipv6_loopback_available()) {
     grpc_ares_hostbyname_request *hr = create_hostbyname_request(
@@ -362,6 +425,10 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl(
                r);
     gpr_free(service_name);
   }
+  if (service_config_json != NULL) {
+    grpc_ares_request_ref(r);
+    ares_search(*channel, hr->host, ns_c_in, ns_t_txt, on_txt_done_cb, r);
+  }
   /* TODO(zyc): Handle CNAME records here. */
   grpc_ares_ev_driver_start(exec_ctx, r->ev_driver);
   grpc_ares_request_unref(exec_ctx, r);
@@ -379,8 +446,8 @@ error_cleanup:
 grpc_ares_request *(*grpc_dns_lookup_ares)(
     grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name,
     const char *default_port, grpc_pollset_set *interested_parties,
-    grpc_closure *on_done, grpc_lb_addresses **addrs,
-    bool check_grpclb) = grpc_dns_lookup_ares_impl;
+    grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb,
+    char **service_config_json) = grpc_dns_lookup_ares_impl;
 
 void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) {
   if (grpc_dns_lookup_ares == grpc_dns_lookup_ares_impl) {
@@ -465,7 +532,8 @@ static void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx,
                     grpc_schedule_on_exec_ctx);
   grpc_dns_lookup_ares(exec_ctx, NULL /* dns_server */, name, default_port,
                        interested_parties, &r->on_dns_lookup_done, &r->lb_addrs,
-                       false /* check_grpclb */);
+                       false /* check_grpclb */,
+                       NULL /* service_config_json */);
 }
 
 void (*grpc_resolve_address_ares)(
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
index 5d2d6c993b..108333047d 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
@@ -27,29 +27,30 @@
 
 typedef struct grpc_ares_request grpc_ares_request;
 
-/* Asynchronously resolve addr. Use \a default_port if a port isn't designated
-   in addr, otherwise use the port in addr. grpc_ares_init() must be called at
-   least once before this function. \a on_done may be called directly in this
-   function without being scheduled with \a exec_ctx, it must not try to acquire
-   locks that are being held by the caller. */
+/* Asynchronously resolve \a name. Use \a default_port if a port isn't
+   designated in \a name, otherwise use the port in \a name. grpc_ares_init()
+   must be called at least once before this function. \a on_done may be
+   called directly in this function without being scheduled with \a exec_ctx,
+   so it must not try to acquire locks that are being held by the caller. */
 extern void (*grpc_resolve_address_ares)(grpc_exec_ctx *exec_ctx,
-                                         const char *addr,
+                                         const char *name,
                                          const char *default_port,
                                          grpc_pollset_set *interested_parties,
                                          grpc_closure *on_done,
                                          grpc_resolved_addresses **addresses);
 
-/* Asynchronously resolve addr. It will try to resolve grpclb SRV records in
+/* Asynchronously resolve \a name. It will try to resolve grpclb SRV records in
   addition to the normal address records. For normal address records, it uses
-  \a default_port if a port isn't designated in \a addr, otherwise it uses the
-  port in \a addr. grpc_ares_init() must be called at least once before this
+  \a default_port if a port isn't designated in \a name, otherwise it uses the
+  port in \a name. grpc_ares_init() must be called at least once before this
   function. \a on_done may be called directly in this function without being
-  scheduled with \a exec_ctx, it must not try to acquire locks that are being
-  held by the caller. */
+  scheduled with \a exec_ctx, so it must not try to acquire locks that are
+  being held by the caller. */
 extern grpc_ares_request *(*grpc_dns_lookup_ares)(
-    grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr,
+    grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name,
     const char *default_port, grpc_pollset_set *interested_parties,
-    grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb);
+    grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb,
+    char **service_config_json);
 
 /* Cancel the pending grpc_ares_request \a request */
 void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c
index b67636a3e4..f2587c4520 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c
@@ -28,15 +28,16 @@ struct grpc_ares_request {
 static grpc_ares_request *grpc_dns_lookup_ares_impl(
     grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name,
     const char *default_port, grpc_pollset_set *interested_parties,
-    grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb) {
+    grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb,
+    char **service_config_json) {
   return NULL;
 }
 
 grpc_ares_request *(*grpc_dns_lookup_ares)(
     grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name,
     const char *default_port, grpc_pollset_set *interested_parties,
-    grpc_closure *on_done, grpc_lb_addresses **addrs,
-    bool check_grpclb) = grpc_dns_lookup_ares_impl;
+    grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb,
+    char **service_config_json) = grpc_dns_lookup_ares_impl;
 
 void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) {}
 
diff --git a/src/core/ext/filters/client_channel/retry_throttle.c b/src/core/ext/filters/client_channel/retry_throttle.c
index 3009e21d49..0c7a3ae651 100644
--- a/src/core/ext/filters/client_channel/retry_throttle.c
+++ b/src/core/ext/filters/client_channel/retry_throttle.c
@@ -130,24 +130,28 @@ static grpc_server_retry_throttle_data* grpc_server_retry_throttle_data_create(
 // avl vtable for string -> server_retry_throttle_data map
 //
 
-static void* copy_server_name(void* key) { return gpr_strdup(key); }
+static void* copy_server_name(void* key, void* unused) {
+  return gpr_strdup(key);
+}
 
-static long compare_server_name(void* key1, void* key2) {
+static long compare_server_name(void* key1, void* key2, void* unused) {
   return strcmp(key1, key2);
 }
 
-static void destroy_server_retry_throttle_data(void* value) {
+static void destroy_server_retry_throttle_data(void* value, void* unused) {
   grpc_server_retry_throttle_data* throttle_data = value;
   grpc_server_retry_throttle_data_unref(throttle_data);
 }
 
-static void* copy_server_retry_throttle_data(void* value) {
+static void* copy_server_retry_throttle_data(void* value, void* unused) {
   grpc_server_retry_throttle_data* throttle_data = value;
   return grpc_server_retry_throttle_data_ref(throttle_data);
 }
 
+static void destroy_server_name(void* key, void* unused) { gpr_free(key); }
+
 static const gpr_avl_vtable avl_vtable = {
-    gpr_free /* destroy_key */, copy_server_name, compare_server_name,
+    destroy_server_name, copy_server_name, compare_server_name,
     destroy_server_retry_throttle_data, copy_server_retry_throttle_data};
 
 //
@@ -164,19 +168,19 @@ void grpc_retry_throttle_map_init() {
 
 void grpc_retry_throttle_map_shutdown() {
   gpr_mu_destroy(&g_mu);
-  gpr_avl_unref(g_avl);
+  gpr_avl_unref(g_avl, NULL);
 }
 
 grpc_server_retry_throttle_data* grpc_retry_throttle_map_get_data_for_server(
     const char* server_name, int max_milli_tokens, int milli_token_ratio) {
   gpr_mu_lock(&g_mu);
   grpc_server_retry_throttle_data* throttle_data =
-      gpr_avl_get(g_avl, (char*)server_name);
+      gpr_avl_get(g_avl, (char*)server_name, NULL);
   if (throttle_data == NULL) {
     // Entry not found.  Create a new one.
     throttle_data = grpc_server_retry_throttle_data_create(
         max_milli_tokens, milli_token_ratio, NULL);
-    g_avl = gpr_avl_add(g_avl, (char*)server_name, throttle_data);
+    g_avl = gpr_avl_add(g_avl, (char*)server_name, throttle_data, NULL);
   } else {
     if (throttle_data->max_milli_tokens != max_milli_tokens ||
         throttle_data->milli_token_ratio != milli_token_ratio) {
@@ -184,7 +188,7 @@ grpc_server_retry_throttle_data* grpc_retry_throttle_map_get_data_for_server(
       // the original one.
       throttle_data = grpc_server_retry_throttle_data_create(
           max_milli_tokens, milli_token_ratio, throttle_data);
-      g_avl = gpr_avl_add(g_avl, (char*)server_name, throttle_data);
+      g_avl = gpr_avl_add(g_avl, (char*)server_name, throttle_data, NULL);
     } else {
       // Entry found.  Increase refcount.
       grpc_server_retry_throttle_data_ref(throttle_data);
diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c
index 5788819331..5cc8be7628 100644
--- a/src/core/ext/filters/client_channel/subchannel.c
+++ b/src/core/ext/filters/client_channel/subchannel.c
@@ -724,13 +724,6 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
   GRPC_CALL_STACK_UNREF(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON);
 }
 
-char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx,
-                                    grpc_subchannel_call *call) {
-  grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
-  grpc_call_element *top_elem = grpc_call_stack_element(call_stack, 0);
-  return top_elem->filter->get_peer(exec_ctx, top_elem);
-}
-
 void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
                                      grpc_subchannel_call *call,
                                      grpc_transport_stream_op_batch *op) {
@@ -760,13 +753,15 @@ grpc_error *grpc_connected_subchannel_create_call(
       args->arena, sizeof(grpc_subchannel_call) + chanstk->call_stack_size);
   grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call);
   (*call)->connection = GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call");
-  const grpc_call_element_args call_args = {.call_stack = callstk,
-                                            .server_transport_data = NULL,
-                                            .context = args->context,
-                                            .path = args->path,
-                                            .start_time = args->start_time,
-                                            .deadline = args->deadline,
-                                            .arena = args->arena};
+  const grpc_call_element_args call_args = {
+      .call_stack = callstk,
+      .server_transport_data = NULL,
+      .context = args->context,
+      .path = args->path,
+      .start_time = args->start_time,
+      .deadline = args->deadline,
+      .arena = args->arena,
+      .call_combiner = args->call_combiner};
   grpc_error *error = grpc_call_stack_init(
       exec_ctx, chanstk, 1, subchannel_call_destroy, *call, &call_args);
   if (error != GRPC_ERROR_NONE) {
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index 6d2abb04df..51d712f6a7 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -106,6 +106,7 @@ typedef struct {
   gpr_timespec deadline;
   gpr_arena *arena;
   grpc_call_context_element *context;
+  grpc_call_combiner *call_combiner;
 } grpc_connected_subchannel_call_args;
 
 grpc_error *grpc_connected_subchannel_create_call(
@@ -150,10 +151,6 @@ void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
                                      grpc_subchannel_call *subchannel_call,
                                      grpc_transport_stream_op_batch *op);
 
-/** continue querying for peer */
-char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx,
-                                    grpc_subchannel_call *subchannel_call);
-
 /** Must be called once per call. Sets the 'then_schedule_closure' argument for
     call stack destruction. */
 void grpc_subchannel_call_set_cleanup_closure(
diff --git a/src/core/ext/filters/client_channel/subchannel_index.c b/src/core/ext/filters/client_channel/subchannel_index.c
index a33ab950bf..ababd05d84 100644
--- a/src/core/ext/filters/client_channel/subchannel_index.c
+++ b/src/core/ext/filters/client_channel/subchannel_index.c
@@ -38,26 +38,8 @@ struct grpc_subchannel_key {
   grpc_subchannel_args args;
 };
 
-GPR_TLS_DECL(subchannel_index_exec_ctx);
-
 static bool g_force_creation = false;
 
-static void enter_ctx(grpc_exec_ctx *exec_ctx) {
-  GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == 0);
-  gpr_tls_set(&subchannel_index_exec_ctx, (intptr_t)exec_ctx);
-}
-
-static void leave_ctx(grpc_exec_ctx *exec_ctx) {
-  GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == (intptr_t)exec_ctx);
-  gpr_tls_set(&subchannel_index_exec_ctx, 0);
-}
-
-static grpc_exec_ctx *current_ctx() {
-  grpc_exec_ctx *c = (grpc_exec_ctx *)gpr_tls_get(&subchannel_index_exec_ctx);
-  GPR_ASSERT(c != NULL);
-  return c;
-}
-
 static grpc_subchannel_key *create_key(
     const grpc_subchannel_args *args,
     grpc_channel_args *(*copy_channel_args)(const grpc_channel_args *args)) {
@@ -104,21 +86,25 @@ void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx,
   gpr_free(k);
 }
 
-static void sck_avl_destroy(void *p) {
-  grpc_subchannel_key_destroy(current_ctx(), p);
+static void sck_avl_destroy(void *p, void *user_data) {
+  grpc_exec_ctx *exec_ctx = (grpc_exec_ctx *)user_data;
+  grpc_subchannel_key_destroy(exec_ctx, p);
 }
 
-static void *sck_avl_copy(void *p) { return subchannel_key_copy(p); }
+static void *sck_avl_copy(void *p, void *unused) {
+  return subchannel_key_copy(p);
+}
 
-static long sck_avl_compare(void *a, void *b) {
+static long sck_avl_compare(void *a, void *b, void *unused) {
   return grpc_subchannel_key_compare(a, b);
 }
 
-static void scv_avl_destroy(void *p) {
-  GRPC_SUBCHANNEL_WEAK_UNREF(current_ctx(), p, "subchannel_index");
+static void scv_avl_destroy(void *p, void *user_data) {
+  grpc_exec_ctx *exec_ctx = (grpc_exec_ctx *)user_data;
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, p, "subchannel_index");
 }
 
-static void *scv_avl_copy(void *p) {
+static void *scv_avl_copy(void *p, void *unused) {
   GRPC_SUBCHANNEL_WEAK_REF(p, "subchannel_index");
   return p;
 }
@@ -133,38 +119,33 @@ static const gpr_avl_vtable subchannel_avl_vtable = {
 void grpc_subchannel_index_init(void) {
   g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable);
   gpr_mu_init(&g_mu);
-  gpr_tls_init(&subchannel_index_exec_ctx);
 }
 
 void grpc_subchannel_index_shutdown(void) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   gpr_mu_destroy(&g_mu);
-  gpr_avl_unref(g_subchannel_index);
-  gpr_tls_destroy(&subchannel_index_exec_ctx);
+  gpr_avl_unref(g_subchannel_index, &exec_ctx);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx,
                                             grpc_subchannel_key *key) {
-  enter_ctx(exec_ctx);
-
   // Lock, and take a reference to the subchannel index.
   // We don't need to do the search under a lock as avl's are immutable.
   gpr_mu_lock(&g_mu);
-  gpr_avl index = gpr_avl_ref(g_subchannel_index);
+  gpr_avl index = gpr_avl_ref(g_subchannel_index, exec_ctx);
   gpr_mu_unlock(&g_mu);
 
-  grpc_subchannel *c =
-      GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(gpr_avl_get(index, key), "index_find");
-  gpr_avl_unref(index);
+  grpc_subchannel *c = GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(
+      gpr_avl_get(index, key, exec_ctx), "index_find");
+  gpr_avl_unref(index, exec_ctx);
 
-  leave_ctx(exec_ctx);
   return c;
 }
 
 grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx,
                                                 grpc_subchannel_key *key,
                                                 grpc_subchannel *constructed) {
-  enter_ctx(exec_ctx);
-
   grpc_subchannel *c = NULL;
   bool need_to_unref_constructed;
 
@@ -174,11 +155,11 @@ grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx,
     // Compare and swap loop:
     // - take a reference to the current index
     gpr_mu_lock(&g_mu);
-    gpr_avl index = gpr_avl_ref(g_subchannel_index);
+    gpr_avl index = gpr_avl_ref(g_subchannel_index, exec_ctx);
     gpr_mu_unlock(&g_mu);
 
     // - Check to see if a subchannel already exists
-    c = gpr_avl_get(index, key);
+    c = gpr_avl_get(index, key, exec_ctx);
     if (c != NULL) {
       c = GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(c, "index_register");
     }
@@ -187,9 +168,9 @@ grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx,
       need_to_unref_constructed = true;
     } else {
       // no -> update the avl and compare/swap
-      gpr_avl updated =
-          gpr_avl_add(gpr_avl_ref(index), subchannel_key_copy(key),
-                      GRPC_SUBCHANNEL_WEAK_REF(constructed, "index_register"));
+      gpr_avl updated = gpr_avl_add(
+          gpr_avl_ref(index, exec_ctx), subchannel_key_copy(key),
+          GRPC_SUBCHANNEL_WEAK_REF(constructed, "index_register"), exec_ctx);
 
       // it may happen (but it's expected to be unlikely)
       // that some other thread has changed the index:
@@ -201,13 +182,11 @@ grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx,
       }
       gpr_mu_unlock(&g_mu);
 
-      gpr_avl_unref(updated);
+      gpr_avl_unref(updated, exec_ctx);
     }
-    gpr_avl_unref(index);
+    gpr_avl_unref(index, exec_ctx);
   }
 
-  leave_ctx(exec_ctx);
-
   if (need_to_unref_constructed) {
     GRPC_SUBCHANNEL_UNREF(exec_ctx, constructed, "index_register");
   }
@@ -218,27 +197,26 @@ grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx,
 void grpc_subchannel_index_unregister(grpc_exec_ctx *exec_ctx,
                                       grpc_subchannel_key *key,
                                       grpc_subchannel *constructed) {
-  enter_ctx(exec_ctx);
-
   bool done = false;
   while (!done) {
     // Compare and swap loop:
     // - take a reference to the current index
     gpr_mu_lock(&g_mu);
-    gpr_avl index = gpr_avl_ref(g_subchannel_index);
+    gpr_avl index = gpr_avl_ref(g_subchannel_index, exec_ctx);
     gpr_mu_unlock(&g_mu);
 
     // Check to see if this key still refers to the previously
     // registered subchannel
-    grpc_subchannel *c = gpr_avl_get(index, key);
+    grpc_subchannel *c = gpr_avl_get(index, key, exec_ctx);
     if (c != constructed) {
-      gpr_avl_unref(index);
+      gpr_avl_unref(index, exec_ctx);
       break;
     }
 
     // compare and swap the update (some other thread may have
     // mutated the index behind us)
-    gpr_avl updated = gpr_avl_remove(gpr_avl_ref(index), key);
+    gpr_avl updated =
+        gpr_avl_remove(gpr_avl_ref(index, exec_ctx), key, exec_ctx);
 
     gpr_mu_lock(&g_mu);
     if (index.root == g_subchannel_index.root) {
@@ -247,11 +225,9 @@ void grpc_subchannel_index_unregister(grpc_exec_ctx *exec_ctx,
     }
     gpr_mu_unlock(&g_mu);
 
-    gpr_avl_unref(updated);
-    gpr_avl_unref(index);
+    gpr_avl_unref(updated, exec_ctx);
+    gpr_avl_unref(index, exec_ctx);
   }
-
-  leave_ctx(exec_ctx);
 }
 
 void grpc_subchannel_index_test_only_set_force_creation(bool force_creation) {
diff --git a/src/core/ext/filters/deadline/deadline_filter.c b/src/core/ext/filters/deadline/deadline_filter.c
index 6789903c95..565b0679dc 100644
--- a/src/core/ext/filters/deadline/deadline_filter.c
+++ b/src/core/ext/filters/deadline/deadline_filter.c
@@ -34,22 +34,56 @@
 // grpc_deadline_state
 //
 
+// The on_complete callback used when sending a cancel_error batch down the
+// filter stack.  Yields the call combiner when the batch returns.
+static void yield_call_combiner(grpc_exec_ctx* exec_ctx, void* arg,
+                                grpc_error* ignored) {
+  grpc_deadline_state* deadline_state = arg;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, deadline_state->call_combiner,
+                          "got on_complete from cancel_stream batch");
+  GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack, "deadline_timer");
+}
+
+// This is called via the call combiner, so access to deadline_state is
+// synchronized.
+static void send_cancel_op_in_call_combiner(grpc_exec_ctx* exec_ctx, void* arg,
+                                            grpc_error* error) {
+  grpc_call_element* elem = arg;
+  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_transport_stream_op_batch* batch = grpc_make_transport_stream_op(
+      GRPC_CLOSURE_INIT(&deadline_state->timer_callback, yield_call_combiner,
+                        deadline_state, grpc_schedule_on_exec_ctx));
+  batch->cancel_stream = true;
+  batch->payload->cancel_stream.cancel_error = GRPC_ERROR_REF(error);
+  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, batch);
+}
+
 // Timer callback.
 static void timer_callback(grpc_exec_ctx* exec_ctx, void* arg,
                            grpc_error* error) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   if (error != GRPC_ERROR_CANCELLED) {
-    grpc_call_element_signal_error(
-        exec_ctx, elem,
-        grpc_error_set_int(
-            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
-            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED));
+    error = grpc_error_set_int(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
+        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED);
+    grpc_call_combiner_cancel(exec_ctx, deadline_state->call_combiner,
+                              GRPC_ERROR_REF(error));
+    GRPC_CLOSURE_INIT(&deadline_state->timer_callback,
+                      send_cancel_op_in_call_combiner, elem,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CALL_COMBINER_START(exec_ctx, deadline_state->call_combiner,
+                             &deadline_state->timer_callback, error,
+                             "deadline exceeded -- sending cancel_stream op");
+  } else {
+    GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack,
+                          "deadline_timer");
   }
-  GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack, "deadline_timer");
 }
 
 // Starts the deadline timer.
+// This is called via the call combiner, so access to deadline_state is
+// synchronized.
 static void start_timer_if_needed(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   gpr_timespec deadline) {
@@ -58,51 +92,39 @@ static void start_timer_if_needed(grpc_exec_ctx* exec_ctx,
     return;
   }
   grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
-  grpc_deadline_timer_state cur_state;
   grpc_closure* closure = NULL;
-retry:
-  cur_state =
-      (grpc_deadline_timer_state)gpr_atm_acq_load(&deadline_state->timer_state);
-  switch (cur_state) {
+  switch (deadline_state->timer_state) {
     case GRPC_DEADLINE_STATE_PENDING:
       // Note: We do not start the timer if there is already a timer
       return;
     case GRPC_DEADLINE_STATE_FINISHED:
-      if (gpr_atm_rel_cas(&deadline_state->timer_state,
-                          GRPC_DEADLINE_STATE_FINISHED,
-                          GRPC_DEADLINE_STATE_PENDING)) {
-        // If we've already created and destroyed a timer, we always create a
-        // new closure: we have no other guarantee that the inlined closure is
-        // not in use (it may hold a pending call to timer_callback)
-        closure = GRPC_CLOSURE_CREATE(timer_callback, elem,
-                                      grpc_schedule_on_exec_ctx);
-      } else {
-        goto retry;
-      }
+      deadline_state->timer_state = GRPC_DEADLINE_STATE_PENDING;
+      // If we've already created and destroyed a timer, we always create a
+      // new closure: we have no other guarantee that the inlined closure is
+      // not in use (it may hold a pending call to timer_callback)
+      closure =
+          GRPC_CLOSURE_CREATE(timer_callback, elem, grpc_schedule_on_exec_ctx);
       break;
     case GRPC_DEADLINE_STATE_INITIAL:
-      if (gpr_atm_rel_cas(&deadline_state->timer_state,
-                          GRPC_DEADLINE_STATE_INITIAL,
-                          GRPC_DEADLINE_STATE_PENDING)) {
-        closure =
-            GRPC_CLOSURE_INIT(&deadline_state->timer_callback, timer_callback,
-                              elem, grpc_schedule_on_exec_ctx);
-      } else {
-        goto retry;
-      }
+      deadline_state->timer_state = GRPC_DEADLINE_STATE_PENDING;
+      closure =
+          GRPC_CLOSURE_INIT(&deadline_state->timer_callback, timer_callback,
+                            elem, grpc_schedule_on_exec_ctx);
       break;
   }
-  GPR_ASSERT(closure);
+  GPR_ASSERT(closure != NULL);
   GRPC_CALL_STACK_REF(deadline_state->call_stack, "deadline_timer");
   grpc_timer_init(exec_ctx, &deadline_state->timer, deadline, closure,
                   gpr_now(GPR_CLOCK_MONOTONIC));
 }
 
 // Cancels the deadline timer.
+// This is called via the call combiner, so access to deadline_state is
+// synchronized.
 static void cancel_timer_if_needed(grpc_exec_ctx* exec_ctx,
                                    grpc_deadline_state* deadline_state) {
-  if (gpr_atm_rel_cas(&deadline_state->timer_state, GRPC_DEADLINE_STATE_PENDING,
-                      GRPC_DEADLINE_STATE_FINISHED)) {
+  if (deadline_state->timer_state == GRPC_DEADLINE_STATE_PENDING) {
+    deadline_state->timer_state = GRPC_DEADLINE_STATE_FINISHED;
     grpc_timer_cancel(exec_ctx, &deadline_state->timer);
   } else {
     // timer was either in STATE_INITAL (nothing to cancel)
@@ -131,6 +153,7 @@ static void inject_on_complete_cb(grpc_deadline_state* deadline_state,
 // Callback and associated state for starting the timer after call stack
 // initialization has been completed.
 struct start_timer_after_init_state {
+  bool in_call_combiner;
   grpc_call_element* elem;
   gpr_timespec deadline;
   grpc_closure closure;
@@ -138,15 +161,29 @@ struct start_timer_after_init_state {
 static void start_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
                                    grpc_error* error) {
   struct start_timer_after_init_state* state = arg;
+  grpc_deadline_state* deadline_state = state->elem->call_data;
+  if (!state->in_call_combiner) {
+    // We are initially called without holding the call combiner, so we
+    // need to bounce ourselves into it.
+    state->in_call_combiner = true;
+    GRPC_CALL_COMBINER_START(exec_ctx, deadline_state->call_combiner,
+                             &state->closure, GRPC_ERROR_REF(error),
+                             "scheduling deadline timer");
+    return;
+  }
   start_timer_if_needed(exec_ctx, state->elem, state->deadline);
   gpr_free(state);
+  GRPC_CALL_COMBINER_STOP(exec_ctx, deadline_state->call_combiner,
+                          "done scheduling deadline timer");
 }
 
 void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               grpc_call_stack* call_stack,
+                              grpc_call_combiner* call_combiner,
                               gpr_timespec deadline) {
   grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   deadline_state->call_stack = call_stack;
+  deadline_state->call_combiner = call_combiner;
   // Deadline will always be infinite on servers, so the timer will only be
   // set on clients with a finite deadline.
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
@@ -158,7 +195,7 @@ void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     // call stack initialization is finished.  To avoid that problem, we
     // create a closure to start the timer, and we schedule that closure
     // to be run after call stack initialization is done.
-    struct start_timer_after_init_state* state = gpr_malloc(sizeof(*state));
+    struct start_timer_after_init_state* state = gpr_zalloc(sizeof(*state));
     state->elem = elem;
     state->deadline = deadline;
     GRPC_CLOSURE_INIT(&state->closure, start_timer_after_init, state,
@@ -232,7 +269,8 @@ typedef struct server_call_data {
 static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   const grpc_call_element_args* args) {
-  grpc_deadline_state_init(exec_ctx, elem, args->call_stack, args->deadline);
+  grpc_deadline_state_init(exec_ctx, elem, args->call_stack,
+                           args->call_combiner, args->deadline);
   return GRPC_ERROR_NONE;
 }
 
@@ -310,7 +348,6 @@ const grpc_channel_filter grpc_client_deadline_filter = {
     0,  // sizeof(channel_data)
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "deadline",
 };
@@ -325,7 +362,6 @@ const grpc_channel_filter grpc_server_deadline_filter = {
     0,  // sizeof(channel_data)
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "deadline",
 };
diff --git a/src/core/ext/filters/deadline/deadline_filter.h b/src/core/ext/filters/deadline/deadline_filter.h
index 420bf7065a..3eb102ad28 100644
--- a/src/core/ext/filters/deadline/deadline_filter.h
+++ b/src/core/ext/filters/deadline/deadline_filter.h
@@ -31,7 +31,8 @@ typedef enum grpc_deadline_timer_state {
 typedef struct grpc_deadline_state {
   // We take a reference to the call stack for the timer callback.
   grpc_call_stack* call_stack;
-  gpr_atm timer_state;
+  grpc_call_combiner* call_combiner;
+  grpc_deadline_timer_state timer_state;
   grpc_timer timer;
   grpc_closure timer_callback;
   // Closure to invoke when the call is complete.
@@ -50,6 +51,7 @@ typedef struct grpc_deadline_state {
 // assumes elem->call_data is zero'd
 void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               grpc_call_stack* call_stack,
+                              grpc_call_combiner* call_combiner,
                               gpr_timespec deadline);
 void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx,
                                  grpc_call_element* elem);
@@ -61,6 +63,8 @@ void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx,
 // to ensure that the timer callback is not invoked while it is in the
 // process of being reset, which means that attempting to increase the
 // deadline may result in the timer being called twice.
+//
+// Note: Must be called while holding the call combiner.
 void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                                gpr_timespec new_deadline);
 
@@ -70,6 +74,8 @@ void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
 //
 // Note: It is the caller's responsibility to chain to the next filter if
 // necessary after this function returns.
+//
+// Note: Must be called while holding the call combiner.
 void grpc_deadline_state_client_start_transport_stream_op_batch(
     grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     grpc_transport_stream_op_batch* op);
diff --git a/src/core/ext/filters/http/client/http_client_filter.c b/src/core/ext/filters/http/client/http_client_filter.c
index 90f0aed7a0..99ddd08e6a 100644
--- a/src/core/ext/filters/http/client/http_client_filter.c
+++ b/src/core/ext/filters/http/client/http_client_filter.c
@@ -36,41 +36,30 @@
 static const size_t kMaxPayloadSizeForGet = 2048;
 
 typedef struct call_data {
+  grpc_call_combiner *call_combiner;
+  // State for handling send_initial_metadata ops.
   grpc_linked_mdelem method;
   grpc_linked_mdelem scheme;
   grpc_linked_mdelem authority;
   grpc_linked_mdelem te_trailers;
   grpc_linked_mdelem content_type;
   grpc_linked_mdelem user_agent;
-
+  // State for handling recv_initial_metadata ops.
   grpc_metadata_batch *recv_initial_metadata;
+  grpc_closure *original_recv_initial_metadata_ready;
+  grpc_closure recv_initial_metadata_ready;
+  // State for handling recv_trailing_metadata ops.
   grpc_metadata_batch *recv_trailing_metadata;
-  uint8_t *payload_bytes;
-
-  /* Vars to read data off of send_message */
-  grpc_transport_stream_op_batch *send_op;
-  uint32_t send_length;
-  uint32_t send_flags;
-  grpc_slice incoming_slice;
-  grpc_slice_buffer_stream replacement_stream;
-  grpc_slice_buffer slices;
-  /* flag that indicates that all slices of send_messages aren't availble */
-  bool send_message_blocked;
-
-  /** Closure to call when finished with the hc_on_recv hook */
-  grpc_closure *on_done_recv_initial_metadata;
-  grpc_closure *on_done_recv_trailing_metadata;
-  grpc_closure *on_complete;
-  grpc_closure *post_send;
-
-  /** Receive closures are chained: we inject this closure as the on_done_recv
-      up-call on transport_op, and remember to call our on_done_recv member
-      after handling it. */
-  grpc_closure hc_on_recv_initial_metadata;
-  grpc_closure hc_on_recv_trailing_metadata;
-  grpc_closure hc_on_complete;
-  grpc_closure got_slice;
-  grpc_closure send_done;
+  grpc_closure *original_recv_trailing_metadata_on_complete;
+  grpc_closure recv_trailing_metadata_on_complete;
+  // State for handling send_message ops.
+  grpc_transport_stream_op_batch *send_message_batch;
+  size_t send_message_bytes_read;
+  grpc_byte_stream_cache send_message_cache;
+  grpc_caching_byte_stream send_message_caching_stream;
+  grpc_closure on_send_message_next_done;
+  grpc_closure *original_send_message_on_complete;
+  grpc_closure send_message_on_complete;
 } call_data;
 
 typedef struct channel_data {
@@ -148,7 +137,7 @@ static grpc_error *client_filter_incoming_metadata(grpc_exec_ctx *exec_ctx,
   return GRPC_ERROR_NONE;
 }
 
-static void hc_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx,
+static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
                                         void *user_data, grpc_error *error) {
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
@@ -158,11 +147,13 @@ static void hc_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx,
   } else {
     GRPC_ERROR_REF(error);
   }
-  GRPC_CLOSURE_RUN(exec_ctx, calld->on_done_recv_initial_metadata, error);
+  GRPC_CLOSURE_RUN(exec_ctx, calld->original_recv_initial_metadata_ready,
+                   error);
 }
 
-static void hc_on_recv_trailing_metadata(grpc_exec_ctx *exec_ctx,
-                                         void *user_data, grpc_error *error) {
+static void recv_trailing_metadata_on_complete(grpc_exec_ctx *exec_ctx,
+                                               void *user_data,
+                                               grpc_error *error) {
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
   if (error == GRPC_ERROR_NONE) {
@@ -171,25 +162,131 @@ static void hc_on_recv_trailing_metadata(grpc_exec_ctx *exec_ctx,
   } else {
     GRPC_ERROR_REF(error);
   }
-  GRPC_CLOSURE_RUN(exec_ctx, calld->on_done_recv_trailing_metadata, error);
+  GRPC_CLOSURE_RUN(exec_ctx, calld->original_recv_trailing_metadata_on_complete,
+                   error);
 }
 
-static void hc_on_complete(grpc_exec_ctx *exec_ctx, void *user_data,
-                           grpc_error *error) {
-  grpc_call_element *elem = user_data;
-  call_data *calld = elem->call_data;
-  if (calld->payload_bytes) {
-    gpr_free(calld->payload_bytes);
-    calld->payload_bytes = NULL;
+static void send_message_on_complete(grpc_exec_ctx *exec_ctx, void *arg,
+                                     grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  grpc_byte_stream_cache_destroy(exec_ctx, &calld->send_message_cache);
+  GRPC_CLOSURE_RUN(exec_ctx, calld->original_send_message_on_complete,
+                   GRPC_ERROR_REF(error));
+}
+
+// Pulls a slice from the send_message byte stream, updating
+// calld->send_message_bytes_read.
+static grpc_error *pull_slice_from_send_message(grpc_exec_ctx *exec_ctx,
+                                                call_data *calld) {
+  grpc_slice incoming_slice;
+  grpc_error *error = grpc_byte_stream_pull(
+      exec_ctx, &calld->send_message_caching_stream.base, &incoming_slice);
+  if (error == GRPC_ERROR_NONE) {
+    calld->send_message_bytes_read += GRPC_SLICE_LENGTH(incoming_slice);
+    grpc_slice_unref_internal(exec_ctx, incoming_slice);
   }
-  calld->on_complete->cb(exec_ctx, calld->on_complete->cb_arg, error);
+  return error;
 }
 
-static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
-  grpc_call_element *elem = elemp;
-  call_data *calld = elem->call_data;
-  grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &calld->slices);
-  calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error);
+// Reads as many slices as possible from the send_message byte stream.
+// Upon successful return, if calld->send_message_bytes_read ==
+// calld->send_message_caching_stream.base.length, then we have completed
+// reading from the byte stream; otherwise, an async read has been dispatched
+// and on_send_message_next_done() will be invoked when it is complete.
+static grpc_error *read_all_available_send_message_data(grpc_exec_ctx *exec_ctx,
+                                                        call_data *calld) {
+  while (grpc_byte_stream_next(exec_ctx,
+                               &calld->send_message_caching_stream.base,
+                               ~(size_t)0, &calld->on_send_message_next_done)) {
+    grpc_error *error = pull_slice_from_send_message(exec_ctx, calld);
+    if (error != GRPC_ERROR_NONE) return error;
+    if (calld->send_message_bytes_read ==
+        calld->send_message_caching_stream.base.length) {
+      break;
+    }
+  }
+  return GRPC_ERROR_NONE;
+}
+
+// Async callback for grpc_byte_stream_next().
+static void on_send_message_next_done(grpc_exec_ctx *exec_ctx, void *arg,
+                                      grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  if (error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, calld->send_message_batch, error, calld->call_combiner);
+    return;
+  }
+  error = pull_slice_from_send_message(exec_ctx, calld);
+  if (error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, calld->send_message_batch, error, calld->call_combiner);
+    return;
+  }
+  // There may or may not be more to read, but we don't care.  If we got
+  // here, then we know that all of the data was not available
+  // synchronously, so we were not able to do a cached call.  Instead,
+  // we just reset the byte stream and then send down the batch as-is.
+  grpc_caching_byte_stream_reset(&calld->send_message_caching_stream);
+  grpc_call_next_op(exec_ctx, elem, calld->send_message_batch);
+}
+
+static char *slice_buffer_to_string(grpc_slice_buffer *slice_buffer) {
+  char *payload_bytes = gpr_malloc(slice_buffer->length + 1);
+  size_t offset = 0;
+  for (size_t i = 0; i < slice_buffer->count; ++i) {
+    memcpy(payload_bytes + offset,
+           GRPC_SLICE_START_PTR(slice_buffer->slices[i]),
+           GRPC_SLICE_LENGTH(slice_buffer->slices[i]));
+    offset += GRPC_SLICE_LENGTH(slice_buffer->slices[i]);
+  }
+  *(payload_bytes + offset) = '\0';
+  return payload_bytes;
+}
+
+// Modifies the path entry in the batch's send_initial_metadata to
+// append the base64-encoded query for a GET request.
+static grpc_error *update_path_for_get(grpc_exec_ctx *exec_ctx,
+                                       grpc_call_element *elem,
+                                       grpc_transport_stream_op_batch *batch) {
+  call_data *calld = (call_data *)elem->call_data;
+  grpc_slice path_slice =
+      GRPC_MDVALUE(batch->payload->send_initial_metadata.send_initial_metadata
+                       ->idx.named.path->md);
+  /* sum up individual component's lengths and allocate enough memory to
+   * hold combined path+query */
+  size_t estimated_len = GRPC_SLICE_LENGTH(path_slice);
+  estimated_len++; /* for the '?' */
+  estimated_len += grpc_base64_estimate_encoded_size(
+      batch->payload->send_message.send_message->length, true /* url_safe */,
+      false /* multi_line */);
+  grpc_slice path_with_query_slice = GRPC_SLICE_MALLOC(estimated_len);
+  /* memcopy individual pieces into this slice */
+  char *write_ptr = (char *)GRPC_SLICE_START_PTR(path_with_query_slice);
+  char *original_path = (char *)GRPC_SLICE_START_PTR(path_slice);
+  memcpy(write_ptr, original_path, GRPC_SLICE_LENGTH(path_slice));
+  write_ptr += GRPC_SLICE_LENGTH(path_slice);
+  *write_ptr++ = '?';
+  char *payload_bytes =
+      slice_buffer_to_string(&calld->send_message_cache.cache_buffer);
+  grpc_base64_encode_core((char *)write_ptr, payload_bytes,
+                          batch->payload->send_message.send_message->length,
+                          true /* url_safe */, false /* multi_line */);
+  gpr_free(payload_bytes);
+  /* remove trailing unused memory and add trailing 0 to terminate string */
+  char *t = (char *)GRPC_SLICE_START_PTR(path_with_query_slice);
+  /* safe to use strlen since base64_encode will always add '\0' */
+  path_with_query_slice =
+      grpc_slice_sub_no_ref(path_with_query_slice, 0, strlen(t));
+  /* substitute previous path with the new path+query */
+  grpc_mdelem mdelem_path_and_query =
+      grpc_mdelem_from_slices(exec_ctx, GRPC_MDSTR_PATH, path_with_query_slice);
+  grpc_metadata_batch *b =
+      batch->payload->send_initial_metadata.send_initial_metadata;
+  return grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path,
+                                        mdelem_path_and_query);
 }
 
 static void remove_if_present(grpc_exec_ctx *exec_ctx,
@@ -200,273 +297,154 @@ static void remove_if_present(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static void continue_send_message(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem) {
+static void hc_start_transport_stream_op_batch(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
-  uint8_t *wrptr = calld->payload_bytes;
-  while (grpc_byte_stream_next(
-      exec_ctx, calld->send_op->payload->send_message.send_message, ~(size_t)0,
-      &calld->got_slice)) {
-    grpc_byte_stream_pull(exec_ctx,
-                          calld->send_op->payload->send_message.send_message,
-                          &calld->incoming_slice);
-    if (GRPC_SLICE_LENGTH(calld->incoming_slice) > 0) {
-      memcpy(wrptr, GRPC_SLICE_START_PTR(calld->incoming_slice),
-             GRPC_SLICE_LENGTH(calld->incoming_slice));
-    }
-    wrptr += GRPC_SLICE_LENGTH(calld->incoming_slice);
-    grpc_slice_buffer_add(&calld->slices, calld->incoming_slice);
-    if (calld->send_length == calld->slices.length) {
-      calld->send_message_blocked = false;
-      break;
-    }
-  }
-}
+  channel_data *channeld = elem->channel_data;
+  GPR_TIMER_BEGIN("hc_start_transport_stream_op_batch", 0);
+  GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
 
-static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
-  grpc_call_element *elem = elemp;
-  call_data *calld = elem->call_data;
-  calld->send_message_blocked = false;
-  if (GRPC_ERROR_NONE !=
-      grpc_byte_stream_pull(exec_ctx,
-                            calld->send_op->payload->send_message.send_message,
-                            &calld->incoming_slice)) {
-    /* Should never reach here */
-    abort();
-  }
-  grpc_slice_buffer_add(&calld->slices, calld->incoming_slice);
-  if (calld->send_length == calld->slices.length) {
-    /* Pass down the original send_message op that was blocked.*/
-    grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices,
-                                  calld->send_flags);
-    calld->send_op->payload->send_message.send_message =
-        &calld->replacement_stream.base;
-    calld->post_send = calld->send_op->on_complete;
-    calld->send_op->on_complete = &calld->send_done;
-    grpc_call_next_op(exec_ctx, elem, calld->send_op);
-  } else {
-    continue_send_message(exec_ctx, elem);
+  if (batch->recv_initial_metadata) {
+    /* substitute our callback for the higher callback */
+    calld->recv_initial_metadata =
+        batch->payload->recv_initial_metadata.recv_initial_metadata;
+    calld->original_recv_initial_metadata_ready =
+        batch->payload->recv_initial_metadata.recv_initial_metadata_ready;
+    batch->payload->recv_initial_metadata.recv_initial_metadata_ready =
+        &calld->recv_initial_metadata_ready;
   }
-}
 
-static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx,
-                                grpc_call_element *elem,
-                                grpc_transport_stream_op_batch *op) {
-  /* grab pointers to our data from the call element */
-  call_data *calld = elem->call_data;
-  channel_data *channeld = elem->channel_data;
-  grpc_error *error;
+  if (batch->recv_trailing_metadata) {
+    /* substitute our callback for the higher callback */
+    calld->recv_trailing_metadata =
+        batch->payload->recv_trailing_metadata.recv_trailing_metadata;
+    calld->original_recv_trailing_metadata_on_complete = batch->on_complete;
+    batch->on_complete = &calld->recv_trailing_metadata_on_complete;
+  }
 
-  if (op->send_initial_metadata) {
-    /* Decide which HTTP VERB to use. We use GET if the request is marked
-    cacheable, and the operation contains both initial metadata and send
-    message, and the payload is below the size threshold, and all the data
-    for this request is immediately available. */
+  grpc_error *error = GRPC_ERROR_NONE;
+  bool batch_will_be_handled_asynchronously = false;
+  if (batch->send_initial_metadata) {
+    // Decide which HTTP VERB to use. We use GET if the request is marked
+    // cacheable, and the operation contains both initial metadata and send
+    // message, and the payload is below the size threshold, and all the data
+    // for this request is immediately available.
     grpc_mdelem method = GRPC_MDELEM_METHOD_POST;
-    if (op->send_message &&
-        (op->payload->send_initial_metadata.send_initial_metadata_flags &
+    if (batch->send_message &&
+        (batch->payload->send_initial_metadata.send_initial_metadata_flags &
          GRPC_INITIAL_METADATA_CACHEABLE_REQUEST) &&
-        op->payload->send_message.send_message->length <
+        batch->payload->send_message.send_message->length <
             channeld->max_payload_size_for_get) {
-      method = GRPC_MDELEM_METHOD_GET;
-      /* The following write to calld->send_message_blocked isn't racy with
-      reads in hc_start_transport_op (which deals with SEND_MESSAGE ops) because
-      being here means ops->send_message is not NULL, which is primarily
-      guarding the read there. */
-      calld->send_message_blocked = true;
-    } else if (op->payload->send_initial_metadata.send_initial_metadata_flags &
-               GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) {
-      method = GRPC_MDELEM_METHOD_PUT;
-    }
-
-    /* Attempt to read the data from send_message and create a header field. */
-    if (grpc_mdelem_eq(method, GRPC_MDELEM_METHOD_GET)) {
-      /* allocate memory to hold the entire payload */
-      calld->payload_bytes =
-          gpr_malloc(op->payload->send_message.send_message->length);
-
-      /* read slices of send_message and copy into payload_bytes */
-      calld->send_op = op;
-      calld->send_length = op->payload->send_message.send_message->length;
-      calld->send_flags = op->payload->send_message.send_message->flags;
-      continue_send_message(exec_ctx, elem);
-
-      if (calld->send_message_blocked == false) {
-        /* when all the send_message data is available, then modify the path
-         * MDELEM by appending base64 encoded query to the path */
-        const int k_url_safe = 1;
-        const int k_multi_line = 0;
-        const unsigned char k_query_separator = '?';
-
-        grpc_slice path_slice =
-            GRPC_MDVALUE(op->payload->send_initial_metadata
-                             .send_initial_metadata->idx.named.path->md);
-        /* sum up individual component's lengths and allocate enough memory to
-         * hold combined path+query */
-        size_t estimated_len = GRPC_SLICE_LENGTH(path_slice);
-        estimated_len++; /* for the '?' */
-        estimated_len += grpc_base64_estimate_encoded_size(
-            op->payload->send_message.send_message->length, k_url_safe,
-            k_multi_line);
-        grpc_slice path_with_query_slice = GRPC_SLICE_MALLOC(estimated_len);
-
-        /* memcopy individual pieces into this slice */
-        uint8_t *write_ptr =
-            (uint8_t *)GRPC_SLICE_START_PTR(path_with_query_slice);
-        uint8_t *original_path = (uint8_t *)GRPC_SLICE_START_PTR(path_slice);
-        memcpy(write_ptr, original_path, GRPC_SLICE_LENGTH(path_slice));
-        write_ptr += GRPC_SLICE_LENGTH(path_slice);
-
-        *write_ptr = k_query_separator;
-        write_ptr++; /* for the '?' */
-
-        grpc_base64_encode_core((char *)write_ptr, calld->payload_bytes,
-                                op->payload->send_message.send_message->length,
-                                k_url_safe, k_multi_line);
-
-        /* remove trailing unused memory and add trailing 0 to terminate string
-         */
-        char *t = (char *)GRPC_SLICE_START_PTR(path_with_query_slice);
-        /* safe to use strlen since base64_encode will always add '\0' */
-        path_with_query_slice =
-            grpc_slice_sub_no_ref(path_with_query_slice, 0, strlen(t));
-
-        /* substitute previous path with the new path+query */
-        grpc_mdelem mdelem_path_and_query = grpc_mdelem_from_slices(
-            exec_ctx, GRPC_MDSTR_PATH, path_with_query_slice);
-        grpc_metadata_batch *b =
-            op->payload->send_initial_metadata.send_initial_metadata;
-        error = grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path,
-                                               mdelem_path_and_query);
-        if (error != GRPC_ERROR_NONE) return error;
-
-        calld->on_complete = op->on_complete;
-        op->on_complete = &calld->hc_on_complete;
-        op->send_message = false;
+      calld->send_message_bytes_read = 0;
+      grpc_byte_stream_cache_init(&calld->send_message_cache,
+                                  batch->payload->send_message.send_message);
+      grpc_caching_byte_stream_init(&calld->send_message_caching_stream,
+                                    &calld->send_message_cache);
+      batch->payload->send_message.send_message =
+          &calld->send_message_caching_stream.base;
+      calld->original_send_message_on_complete = batch->on_complete;
+      batch->on_complete = &calld->send_message_on_complete;
+      calld->send_message_batch = batch;
+      error = read_all_available_send_message_data(exec_ctx, calld);
+      if (error != GRPC_ERROR_NONE) goto done;
+      // If all the data has been read, then we can use GET.
+      if (calld->send_message_bytes_read ==
+          calld->send_message_caching_stream.base.length) {
+        method = GRPC_MDELEM_METHOD_GET;
+        error = update_path_for_get(exec_ctx, elem, batch);
+        if (error != GRPC_ERROR_NONE) goto done;
+        batch->send_message = false;
+        grpc_byte_stream_destroy(exec_ctx,
+                                 &calld->send_message_caching_stream.base);
       } else {
-        /* Not all data is available. Fall back to POST. */
+        // Not all data is available.  The batch will be sent down
+        // asynchronously in on_send_message_next_done().
+        batch_will_be_handled_asynchronously = true;
+        // Fall back to POST.
         gpr_log(GPR_DEBUG,
-                "Request is marked Cacheable but not all data is available.\
-                            Falling back to POST");
-        method = GRPC_MDELEM_METHOD_POST;
+                "Request is marked Cacheable but not all data is available.  "
+                "Falling back to POST");
       }
+    } else if (batch->payload->send_initial_metadata
+                   .send_initial_metadata_flags &
+               GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) {
+      method = GRPC_MDELEM_METHOD_PUT;
     }
 
-    remove_if_present(exec_ctx,
-                      op->payload->send_initial_metadata.send_initial_metadata,
-                      GRPC_BATCH_METHOD);
-    remove_if_present(exec_ctx,
-                      op->payload->send_initial_metadata.send_initial_metadata,
-                      GRPC_BATCH_SCHEME);
-    remove_if_present(exec_ctx,
-                      op->payload->send_initial_metadata.send_initial_metadata,
-                      GRPC_BATCH_TE);
-    remove_if_present(exec_ctx,
-                      op->payload->send_initial_metadata.send_initial_metadata,
-                      GRPC_BATCH_CONTENT_TYPE);
-    remove_if_present(exec_ctx,
-                      op->payload->send_initial_metadata.send_initial_metadata,
-                      GRPC_BATCH_USER_AGENT);
+    remove_if_present(
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
+        GRPC_BATCH_METHOD);
+    remove_if_present(
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
+        GRPC_BATCH_SCHEME);
+    remove_if_present(
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
+        GRPC_BATCH_TE);
+    remove_if_present(
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
+        GRPC_BATCH_CONTENT_TYPE);
+    remove_if_present(
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
+        GRPC_BATCH_USER_AGENT);
 
     /* Send : prefixed headers, which have to be before any application
        layer headers. */
     error = grpc_metadata_batch_add_head(
-        exec_ctx, op->payload->send_initial_metadata.send_initial_metadata,
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
         &calld->method, method);
-    if (error != GRPC_ERROR_NONE) return error;
+    if (error != GRPC_ERROR_NONE) goto done;
     error = grpc_metadata_batch_add_head(
-        exec_ctx, op->payload->send_initial_metadata.send_initial_metadata,
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
         &calld->scheme, channeld->static_scheme);
-    if (error != GRPC_ERROR_NONE) return error;
+    if (error != GRPC_ERROR_NONE) goto done;
     error = grpc_metadata_batch_add_tail(
-        exec_ctx, op->payload->send_initial_metadata.send_initial_metadata,
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
         &calld->te_trailers, GRPC_MDELEM_TE_TRAILERS);
-    if (error != GRPC_ERROR_NONE) return error;
+    if (error != GRPC_ERROR_NONE) goto done;
     error = grpc_metadata_batch_add_tail(
-        exec_ctx, op->payload->send_initial_metadata.send_initial_metadata,
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
         &calld->content_type, GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC);
-    if (error != GRPC_ERROR_NONE) return error;
+    if (error != GRPC_ERROR_NONE) goto done;
     error = grpc_metadata_batch_add_tail(
-        exec_ctx, op->payload->send_initial_metadata.send_initial_metadata,
+        exec_ctx, batch->payload->send_initial_metadata.send_initial_metadata,
         &calld->user_agent, GRPC_MDELEM_REF(channeld->user_agent));
-    if (error != GRPC_ERROR_NONE) return error;
+    if (error != GRPC_ERROR_NONE) goto done;
   }
 
-  if (op->recv_initial_metadata) {
-    /* substitute our callback for the higher callback */
-    calld->recv_initial_metadata =
-        op->payload->recv_initial_metadata.recv_initial_metadata;
-    calld->on_done_recv_initial_metadata =
-        op->payload->recv_initial_metadata.recv_initial_metadata_ready;
-    op->payload->recv_initial_metadata.recv_initial_metadata_ready =
-        &calld->hc_on_recv_initial_metadata;
-  }
-
-  if (op->recv_trailing_metadata) {
-    /* substitute our callback for the higher callback */
-    calld->recv_trailing_metadata =
-        op->payload->recv_trailing_metadata.recv_trailing_metadata;
-    calld->on_done_recv_trailing_metadata = op->on_complete;
-    op->on_complete = &calld->hc_on_recv_trailing_metadata;
-  }
-
-  return GRPC_ERROR_NONE;
-}
-
-static void hc_start_transport_op(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem,
-                                  grpc_transport_stream_op_batch *op) {
-  GPR_TIMER_BEGIN("hc_start_transport_op", 0);
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-  grpc_error *error = hc_mutate_op(exec_ctx, elem, op);
+done:
   if (error != GRPC_ERROR_NONE) {
-    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
-  } else {
-    call_data *calld = elem->call_data;
-    if (op->send_message && calld->send_message_blocked) {
-      /* Don't forward the op. send_message contains slices that aren't ready
-         yet. The call will be forwarded by the op_complete of slice read call.
-      */
-    } else {
-      grpc_call_next_op(exec_ctx, elem, op);
-    }
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, calld->send_message_batch, error, calld->call_combiner);
+  } else if (!batch_will_be_handled_asynchronously) {
+    grpc_call_next_op(exec_ctx, elem, batch);
   }
-  GPR_TIMER_END("hc_start_transport_op", 0);
+  GPR_TIMER_END("hc_start_transport_stream_op_batch", 0);
 }
 
 /* Constructor for call_data */
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
-  call_data *calld = elem->call_data;
-  calld->on_done_recv_initial_metadata = NULL;
-  calld->on_done_recv_trailing_metadata = NULL;
-  calld->on_complete = NULL;
-  calld->payload_bytes = NULL;
-  calld->send_message_blocked = false;
-  grpc_slice_buffer_init(&calld->slices);
-  GRPC_CLOSURE_INIT(&calld->hc_on_recv_initial_metadata,
-                    hc_on_recv_initial_metadata, elem,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->hc_on_recv_trailing_metadata,
-                    hc_on_recv_trailing_metadata, elem,
+  call_data *calld = (call_data *)elem->call_data;
+  calld->call_combiner = args->call_combiner;
+  GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
+                    recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->hc_on_complete, hc_on_complete, elem,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->got_slice, got_slice, elem,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->send_done, send_done, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_on_complete,
+                    recv_trailing_metadata_on_complete, elem,
                     grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&calld->send_message_on_complete, send_message_on_complete,
+                    elem, grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&calld->on_send_message_next_done,
+                    on_send_message_next_done, elem, grpc_schedule_on_exec_ctx);
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
-                              grpc_closure *ignored) {
-  call_data *calld = elem->call_data;
-  grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices);
-}
+                              grpc_closure *ignored) {}
 
 static grpc_mdelem scheme_from_args(const grpc_channel_args *args) {
   unsigned i;
@@ -580,7 +558,7 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 }
 
 const grpc_channel_filter grpc_http_client_filter = {
-    hc_start_transport_op,
+    hc_start_transport_stream_op_batch,
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
@@ -589,6 +567,5 @@ const grpc_channel_filter grpc_http_client_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "http-client"};
diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c
index 71a8bc5bec..98a503cafc 100644
--- a/src/core/ext/filters/http/message_compress/message_compress_filter.c
+++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c
@@ -35,40 +35,33 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-#define INITIAL_METADATA_UNSEEN 0
-#define HAS_COMPRESSION_ALGORITHM 2
-#define NO_COMPRESSION_ALGORITHM 4
-
-#define CANCELLED_BIT ((gpr_atm)1)
+typedef enum {
+  // Initial metadata not yet seen.
+  INITIAL_METADATA_UNSEEN = 0,
+  // Initial metadata seen; compression algorithm set.
+  HAS_COMPRESSION_ALGORITHM,
+  // Initial metadata seen; no compression algorithm set.
+  NO_COMPRESSION_ALGORITHM,
+} initial_metadata_state;
 
 typedef struct call_data {
-  grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */
+  grpc_call_combiner *call_combiner;
   grpc_linked_mdelem compression_algorithm_storage;
+  grpc_linked_mdelem stream_compression_algorithm_storage;
   grpc_linked_mdelem accept_encoding_storage;
-  uint32_t remaining_slice_bytes;
+  grpc_linked_mdelem accept_stream_encoding_storage;
   /** Compression algorithm we'll try to use. It may be given by incoming
    * metadata, or by the channel's default compression settings. */
   grpc_compression_algorithm compression_algorithm;
-
-  /* Atomic recording the state of initial metadata; allowed values:
-     INITIAL_METADATA_UNSEEN - initial metadata op not seen
-     HAS_COMPRESSION_ALGORITHM - initial metadata seen; compression algorithm
-                                 set
-     NO_COMPRESSION_ALGORITHM - initial metadata seen; no compression algorithm
-                                set
-     pointer - a stalled op containing a send_message that's waiting on initial
-               metadata
-     pointer | CANCELLED_BIT - request was cancelled with error pointed to */
-  gpr_atm send_initial_metadata_state;
-
-  grpc_transport_stream_op_batch *send_op;
-  uint32_t send_length;
-  uint32_t send_flags;
-  grpc_slice incoming_slice;
+  initial_metadata_state send_initial_metadata_state;
+  grpc_error *cancel_error;
+  grpc_closure start_send_message_batch_in_call_combiner;
+  grpc_transport_stream_op_batch *send_message_batch;
+  grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */
   grpc_slice_buffer_stream replacement_stream;
-  grpc_closure *post_send;
-  grpc_closure send_done;
-  grpc_closure got_slice;
+  grpc_closure *original_send_message_on_complete;
+  grpc_closure send_message_on_complete;
+  grpc_closure on_send_message_next_done;
 } call_data;
 
 typedef struct channel_data {
@@ -78,6 +71,13 @@ typedef struct channel_data {
   uint32_t enabled_algorithms_bitset;
   /** Supported compression algorithms */
   uint32_t supported_compression_algorithms;
+
+  /** The default, channel-level, stream compression algorithm */
+  grpc_stream_compression_algorithm default_stream_compression_algorithm;
+  /** Bitset of enabled stream compression algorithms */
+  uint32_t enabled_stream_compression_algorithms_bitset;
+  /** Supported stream compression algorithms */
+  uint32_t supported_stream_compression_algorithms;
 } channel_data;
 
 static bool skip_compression(grpc_call_element *elem, uint32_t flags,
@@ -86,13 +86,13 @@ static bool skip_compression(grpc_call_element *elem, uint32_t flags,
   channel_data *channeld = elem->channel_data;
 
   if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) {
-    return 1;
+    return true;
   }
   if (has_compression_algorithm) {
     if (calld->compression_algorithm == GRPC_COMPRESS_NONE) {
-      return 1;
+      return true;
     }
-    return 0; /* we have an actual call-specific algorithm */
+    return false; /* we have an actual call-specific algorithm */
   }
   /* no per-call compression override */
   return channeld->default_compression_algorithm == GRPC_COMPRESS_NONE;
@@ -109,31 +109,56 @@ static grpc_error *process_send_initial_metadata(
   call_data *calld = elem->call_data;
   channel_data *channeld = elem->channel_data;
   *has_compression_algorithm = false;
-  /* Parse incoming request for compression. If any, it'll be available
-   * at calld->compression_algorithm */
-  if (initial_metadata->idx.named.grpc_internal_encoding_request != NULL) {
+  grpc_stream_compression_algorithm stream_compression_algorithm =
+      GRPC_STREAM_COMPRESS_NONE;
+  if (initial_metadata->idx.named.grpc_internal_stream_encoding_request !=
+      NULL) {
     grpc_mdelem md =
-        initial_metadata->idx.named.grpc_internal_encoding_request->md;
-    if (!grpc_compression_algorithm_parse(GRPC_MDVALUE(md),
-                                          &calld->compression_algorithm)) {
+        initial_metadata->idx.named.grpc_internal_stream_encoding_request->md;
+    if (!grpc_stream_compression_algorithm_parse(
+            GRPC_MDVALUE(md), &stream_compression_algorithm)) {
       char *val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
       gpr_log(GPR_ERROR,
-              "Invalid compression algorithm: '%s' (unknown). Ignoring.", val);
+              "Invalid stream compression algorithm: '%s' (unknown). Ignoring.",
+              val);
       gpr_free(val);
-      calld->compression_algorithm = GRPC_COMPRESS_NONE;
+      stream_compression_algorithm = GRPC_STREAM_COMPRESS_NONE;
+    }
+    if (!GPR_BITGET(channeld->enabled_stream_compression_algorithms_bitset,
+                    stream_compression_algorithm)) {
+      char *val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
+      gpr_log(
+          GPR_ERROR,
+          "Invalid stream compression algorithm: '%s' (previously disabled). "
+          "Ignoring.",
+          val);
+      gpr_free(val);
+      stream_compression_algorithm = GRPC_STREAM_COMPRESS_NONE;
     }
-    if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
-                    calld->compression_algorithm)) {
+    *has_compression_algorithm = true;
+    grpc_metadata_batch_remove(
+        exec_ctx, initial_metadata,
+        initial_metadata->idx.named.grpc_internal_stream_encoding_request);
+    /* Disable message-wise compression */
+    calld->compression_algorithm = GRPC_COMPRESS_NONE;
+    if (initial_metadata->idx.named.grpc_internal_encoding_request != NULL) {
+      grpc_metadata_batch_remove(
+          exec_ctx, initial_metadata,
+          initial_metadata->idx.named.grpc_internal_encoding_request);
+    }
+  } else if (initial_metadata->idx.named.grpc_internal_encoding_request !=
+             NULL) {
+    grpc_mdelem md =
+        initial_metadata->idx.named.grpc_internal_encoding_request->md;
+    if (!grpc_compression_algorithm_parse(GRPC_MDVALUE(md),
+                                          &calld->compression_algorithm)) {
       char *val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
       gpr_log(GPR_ERROR,
-              "Invalid compression algorithm: '%s' (previously disabled). "
-              "Ignoring.",
-              val);
+              "Invalid compression algorithm: '%s' (unknown). Ignoring.", val);
       gpr_free(val);
       calld->compression_algorithm = GRPC_COMPRESS_NONE;
     }
     *has_compression_algorithm = true;
-
     grpc_metadata_batch_remove(
         exec_ctx, initial_metadata,
         initial_metadata->idx.named.grpc_internal_encoding_request);
@@ -141,13 +166,25 @@ static grpc_error *process_send_initial_metadata(
     /* If no algorithm was found in the metadata and we aren't
      * exceptionally skipping compression, fall back to the channel
      * default */
-    calld->compression_algorithm = channeld->default_compression_algorithm;
+    if (channeld->default_stream_compression_algorithm !=
+        GRPC_STREAM_COMPRESS_NONE) {
+      stream_compression_algorithm =
+          channeld->default_stream_compression_algorithm;
+      calld->compression_algorithm = GRPC_COMPRESS_NONE;
+    } else {
+      calld->compression_algorithm = channeld->default_compression_algorithm;
+    }
     *has_compression_algorithm = true;
   }
 
   grpc_error *error = GRPC_ERROR_NONE;
   /* hint compression algorithm */
-  if (calld->compression_algorithm != GRPC_COMPRESS_NONE) {
+  if (stream_compression_algorithm != GRPC_STREAM_COMPRESS_NONE) {
+    error = grpc_metadata_batch_add_tail(
+        exec_ctx, initial_metadata,
+        &calld->stream_compression_algorithm_storage,
+        grpc_stream_compression_encoding_mdelem(stream_compression_algorithm));
+  } else if (calld->compression_algorithm != GRPC_COMPRESS_NONE) {
     error = grpc_metadata_batch_add_tail(
         exec_ctx, initial_metadata, &calld->compression_algorithm_storage,
         grpc_compression_encoding_mdelem(calld->compression_algorithm));
@@ -161,27 +198,50 @@ static grpc_error *process_send_initial_metadata(
       GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS(
           channeld->supported_compression_algorithms));
 
+  if (error != GRPC_ERROR_NONE) return error;
+
+  /* Do not overwrite accept-encoding header if it already presents. */
+  if (!initial_metadata->idx.named.accept_encoding) {
+    error = grpc_metadata_batch_add_tail(
+        exec_ctx, initial_metadata, &calld->accept_stream_encoding_storage,
+        GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(
+            channeld->supported_stream_compression_algorithms));
+  }
+
   return error;
 }
 
-static void continue_send_message(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem);
-
-static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
-  grpc_call_element *elem = elemp;
-  call_data *calld = elem->call_data;
+static void send_message_on_complete(grpc_exec_ctx *exec_ctx, void *arg,
+                                     grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
   grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &calld->slices);
-  calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error);
+  GRPC_CLOSURE_RUN(exec_ctx, calld->original_send_message_on_complete,
+                   GRPC_ERROR_REF(error));
+}
+
+static void send_message_batch_continue(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_element *elem) {
+  call_data *calld = (call_data *)elem->call_data;
+  // Note: The call to grpc_call_next_op() results in yielding the
+  // call combiner, so we need to clear calld->send_message_batch
+  // before we do that.
+  grpc_transport_stream_op_batch *send_message_batch =
+      calld->send_message_batch;
+  calld->send_message_batch = NULL;
+  grpc_call_next_op(exec_ctx, elem, send_message_batch);
 }
 
 static void finish_send_message(grpc_exec_ctx *exec_ctx,
                                 grpc_call_element *elem) {
-  call_data *calld = elem->call_data;
-  int did_compress;
+  call_data *calld = (call_data *)elem->call_data;
+  // Compress the data if appropriate.
   grpc_slice_buffer tmp;
   grpc_slice_buffer_init(&tmp);
-  did_compress = grpc_msg_compress(exec_ctx, calld->compression_algorithm,
-                                   &calld->slices, &tmp);
+  uint32_t send_flags =
+      calld->send_message_batch->payload->send_message.send_message->flags;
+  bool did_compress = grpc_msg_compress(exec_ctx, calld->compression_algorithm,
+                                        &calld->slices, &tmp);
   if (did_compress) {
     if (GRPC_TRACER_ON(grpc_compression_trace)) {
       char *algo_name;
@@ -195,7 +255,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
               algo_name, before_size, after_size, 100 * savings_ratio);
     }
     grpc_slice_buffer_swap(&calld->slices, &tmp);
-    calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS;
+    send_flags |= GRPC_WRITE_INTERNAL_COMPRESS;
   } else {
     if (GRPC_TRACER_ON(grpc_compression_trace)) {
       char *algo_name;
@@ -207,160 +267,188 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
               algo_name, calld->slices.length);
     }
   }
-
   grpc_slice_buffer_destroy_internal(exec_ctx, &tmp);
-
+  // Swap out the original byte stream with our new one and send the
+  // batch down.
+  grpc_byte_stream_destroy(
+      exec_ctx, calld->send_message_batch->payload->send_message.send_message);
   grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices,
-                                calld->send_flags);
-  calld->send_op->payload->send_message.send_message =
+                                send_flags);
+  calld->send_message_batch->payload->send_message.send_message =
       &calld->replacement_stream.base;
-  calld->post_send = calld->send_op->on_complete;
-  calld->send_op->on_complete = &calld->send_done;
-
-  grpc_call_next_op(exec_ctx, elem, calld->send_op);
+  calld->original_send_message_on_complete =
+      calld->send_message_batch->on_complete;
+  calld->send_message_batch->on_complete = &calld->send_message_on_complete;
+  send_message_batch_continue(exec_ctx, elem);
 }
 
-static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
-  grpc_call_element *elem = elemp;
-  call_data *calld = elem->call_data;
-  if (GRPC_ERROR_NONE !=
-      grpc_byte_stream_pull(exec_ctx,
-                            calld->send_op->payload->send_message.send_message,
-                            &calld->incoming_slice)) {
-    /* Should never reach here */
-    abort();
+static void fail_send_message_batch_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                                     void *arg,
+                                                     grpc_error *error) {
+  call_data *calld = arg;
+  if (calld->send_message_batch != NULL) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, calld->send_message_batch, GRPC_ERROR_REF(error),
+        calld->call_combiner);
+    calld->send_message_batch = NULL;
   }
-  grpc_slice_buffer_add(&calld->slices, calld->incoming_slice);
-  if (calld->send_length == calld->slices.length) {
-    finish_send_message(exec_ctx, elem);
-  } else {
-    continue_send_message(exec_ctx, elem);
+}
+
+// Pulls a slice from the send_message byte stream and adds it to calld->slices.
+static grpc_error *pull_slice_from_send_message(grpc_exec_ctx *exec_ctx,
+                                                call_data *calld) {
+  grpc_slice incoming_slice;
+  grpc_error *error = grpc_byte_stream_pull(
+      exec_ctx, calld->send_message_batch->payload->send_message.send_message,
+      &incoming_slice);
+  if (error == GRPC_ERROR_NONE) {
+    grpc_slice_buffer_add(&calld->slices, incoming_slice);
   }
+  return error;
 }
 
-static void continue_send_message(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem) {
-  call_data *calld = elem->call_data;
+// Reads as many slices as possible from the send_message byte stream.
+// If all data has been read, invokes finish_send_message().  Otherwise,
+// an async call to grpc_byte_stream_next() has been started, which will
+// eventually result in calling on_send_message_next_done().
+static void continue_reading_send_message(grpc_exec_ctx *exec_ctx,
+                                          grpc_call_element *elem) {
+  call_data *calld = (call_data *)elem->call_data;
   while (grpc_byte_stream_next(
-      exec_ctx, calld->send_op->payload->send_message.send_message, ~(size_t)0,
-      &calld->got_slice)) {
-    grpc_byte_stream_pull(exec_ctx,
-                          calld->send_op->payload->send_message.send_message,
-                          &calld->incoming_slice);
-    grpc_slice_buffer_add(&calld->slices, calld->incoming_slice);
-    if (calld->send_length == calld->slices.length) {
+      exec_ctx, calld->send_message_batch->payload->send_message.send_message,
+      ~(size_t)0, &calld->on_send_message_next_done)) {
+    grpc_error *error = pull_slice_from_send_message(exec_ctx, calld);
+    if (error != GRPC_ERROR_NONE) {
+      // Closure callback; does not take ownership of error.
+      fail_send_message_batch_in_call_combiner(exec_ctx, calld, error);
+      GRPC_ERROR_UNREF(error);
+      return;
+    }
+    if (calld->slices.length ==
+        calld->send_message_batch->payload->send_message.send_message->length) {
       finish_send_message(exec_ctx, elem);
       break;
     }
   }
 }
 
-static void handle_send_message_batch(grpc_exec_ctx *exec_ctx,
-                                      grpc_call_element *elem,
-                                      grpc_transport_stream_op_batch *op,
-                                      bool has_compression_algorithm) {
-  call_data *calld = elem->call_data;
-  if (!skip_compression(elem, op->payload->send_message.send_message->flags,
-                        has_compression_algorithm)) {
-    calld->send_op = op;
-    calld->send_length = op->payload->send_message.send_message->length;
-    calld->send_flags = op->payload->send_message.send_message->flags;
-    continue_send_message(exec_ctx, elem);
+// Async callback for grpc_byte_stream_next().
+static void on_send_message_next_done(grpc_exec_ctx *exec_ctx, void *arg,
+                                      grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  if (error != GRPC_ERROR_NONE) {
+    // Closure callback; does not take ownership of error.
+    fail_send_message_batch_in_call_combiner(exec_ctx, calld, error);
+    return;
+  }
+  error = pull_slice_from_send_message(exec_ctx, calld);
+  if (error != GRPC_ERROR_NONE) {
+    // Closure callback; does not take ownership of error.
+    fail_send_message_batch_in_call_combiner(exec_ctx, calld, error);
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
+  if (calld->slices.length ==
+      calld->send_message_batch->payload->send_message.send_message->length) {
+    finish_send_message(exec_ctx, elem);
+  } else {
+    continue_reading_send_message(exec_ctx, elem);
+  }
+}
+
+static void start_send_message_batch(grpc_exec_ctx *exec_ctx, void *arg,
+                                     grpc_error *unused) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  if (skip_compression(
+          elem,
+          calld->send_message_batch->payload->send_message.send_message->flags,
+          calld->send_initial_metadata_state == HAS_COMPRESSION_ALGORITHM)) {
+    send_message_batch_continue(exec_ctx, elem);
   } else {
-    /* pass control down the stack */
-    grpc_call_next_op(exec_ctx, elem, op);
+    continue_reading_send_message(exec_ctx, elem);
   }
 }
 
 static void compress_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-    grpc_transport_stream_op_batch *op) {
+    grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
-
   GPR_TIMER_BEGIN("compress_start_transport_stream_op_batch", 0);
-
-  if (op->cancel_stream) {
-    GRPC_ERROR_REF(op->payload->cancel_stream.cancel_error);
-    gpr_atm cur = gpr_atm_full_xchg(
-        &calld->send_initial_metadata_state,
-        CANCELLED_BIT | (gpr_atm)op->payload->cancel_stream.cancel_error);
-    switch (cur) {
-      case HAS_COMPRESSION_ALGORITHM:
-      case NO_COMPRESSION_ALGORITHM:
-      case INITIAL_METADATA_UNSEEN:
-        break;
-      default:
-        if ((cur & CANCELLED_BIT) == 0) {
-          grpc_transport_stream_op_batch_finish_with_failure(
-              exec_ctx, (grpc_transport_stream_op_batch *)cur,
-              GRPC_ERROR_REF(op->payload->cancel_stream.cancel_error));
-        } else {
-          GRPC_ERROR_UNREF((grpc_error *)(cur & ~CANCELLED_BIT));
-        }
-        break;
+  // Handle cancel_stream.
+  if (batch->cancel_stream) {
+    GRPC_ERROR_UNREF(calld->cancel_error);
+    calld->cancel_error =
+        GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
+    if (calld->send_message_batch != NULL) {
+      if (calld->send_initial_metadata_state == INITIAL_METADATA_UNSEEN) {
+        GRPC_CALL_COMBINER_START(
+            exec_ctx, calld->call_combiner,
+            GRPC_CLOSURE_CREATE(fail_send_message_batch_in_call_combiner, calld,
+                                grpc_schedule_on_exec_ctx),
+            GRPC_ERROR_REF(calld->cancel_error), "failing send_message op");
+      } else {
+        grpc_byte_stream_shutdown(
+            exec_ctx,
+            calld->send_message_batch->payload->send_message.send_message,
+            GRPC_ERROR_REF(calld->cancel_error));
+      }
     }
+  } else if (calld->cancel_error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, batch, GRPC_ERROR_REF(calld->cancel_error),
+        calld->call_combiner);
+    goto done;
   }
-
-  if (op->send_initial_metadata) {
+  // Handle send_initial_metadata.
+  if (batch->send_initial_metadata) {
+    GPR_ASSERT(calld->send_initial_metadata_state == INITIAL_METADATA_UNSEEN);
     bool has_compression_algorithm;
     grpc_error *error = process_send_initial_metadata(
         exec_ctx, elem,
-        op->payload->send_initial_metadata.send_initial_metadata,
+        batch->payload->send_initial_metadata.send_initial_metadata,
         &has_compression_algorithm);
     if (error != GRPC_ERROR_NONE) {
-      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
-      return;
+      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error,
+                                                         calld->call_combiner);
+      goto done;
     }
-    gpr_atm cur;
-  retry_send_im:
-    cur = gpr_atm_acq_load(&calld->send_initial_metadata_state);
-    GPR_ASSERT(cur != HAS_COMPRESSION_ALGORITHM &&
-               cur != NO_COMPRESSION_ALGORITHM);
-    if ((cur & CANCELLED_BIT) == 0) {
-      if (!gpr_atm_rel_cas(&calld->send_initial_metadata_state, cur,
-                           has_compression_algorithm
-                               ? HAS_COMPRESSION_ALGORITHM
-                               : NO_COMPRESSION_ALGORITHM)) {
-        goto retry_send_im;
-      }
-      if (cur != INITIAL_METADATA_UNSEEN) {
-        handle_send_message_batch(exec_ctx, elem,
-                                  (grpc_transport_stream_op_batch *)cur,
-                                  has_compression_algorithm);
-      }
+    calld->send_initial_metadata_state = has_compression_algorithm
+                                             ? HAS_COMPRESSION_ALGORITHM
+                                             : NO_COMPRESSION_ALGORITHM;
+    // If we had previously received a batch containing a send_message op,
+    // handle it now.  Note that we need to re-enter the call combiner
+    // for this, since we can't send two batches down while holding the
+    // call combiner, since the connected_channel filter (at the bottom of
+    // the call stack) will release the call combiner for each batch it sees.
+    if (calld->send_message_batch != NULL) {
+      GRPC_CALL_COMBINER_START(
+          exec_ctx, calld->call_combiner,
+          &calld->start_send_message_batch_in_call_combiner, GRPC_ERROR_NONE,
+          "starting send_message after send_initial_metadata");
     }
   }
-  if (op->send_message) {
-    gpr_atm cur;
-  retry_send:
-    cur = gpr_atm_acq_load(&calld->send_initial_metadata_state);
-    switch (cur) {
-      case INITIAL_METADATA_UNSEEN:
-        if (!gpr_atm_rel_cas(&calld->send_initial_metadata_state, cur,
-                             (gpr_atm)op)) {
-          goto retry_send;
-        }
-        break;
-      case HAS_COMPRESSION_ALGORITHM:
-      case NO_COMPRESSION_ALGORITHM:
-        handle_send_message_batch(exec_ctx, elem, op,
-                                  cur == HAS_COMPRESSION_ALGORITHM);
-        break;
-      default:
-        if (cur & CANCELLED_BIT) {
-          grpc_transport_stream_op_batch_finish_with_failure(
-              exec_ctx, op,
-              GRPC_ERROR_REF((grpc_error *)(cur & ~CANCELLED_BIT)));
-        } else {
-          /* >1 send_message concurrently */
-          GPR_UNREACHABLE_CODE(break);
-        }
+  // Handle send_message.
+  if (batch->send_message) {
+    GPR_ASSERT(calld->send_message_batch == NULL);
+    calld->send_message_batch = batch;
+    // If we have not yet seen send_initial_metadata, then we have to
+    // wait.  We save the batch in calld and then drop the call
+    // combiner, which we'll have to pick up again later when we get
+    // send_initial_metadata.
+    if (calld->send_initial_metadata_state == INITIAL_METADATA_UNSEEN) {
+      GRPC_CALL_COMBINER_STOP(
+          exec_ctx, calld->call_combiner,
+          "send_message batch pending send_initial_metadata");
+      goto done;
     }
+    start_send_message_batch(exec_ctx, elem, GRPC_ERROR_NONE);
   } else {
-    /* pass control down the stack */
-    grpc_call_next_op(exec_ctx, elem, op);
+    // Pass control down the stack.
+    grpc_call_next_op(exec_ctx, elem, batch);
   }
-
+done:
   GPR_TIMER_END("compress_start_transport_stream_op_batch", 0);
 }
 
@@ -368,16 +456,16 @@ static void compress_start_transport_stream_op_batch(
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
-  /* grab pointers to our data from the call element */
-  call_data *calld = elem->call_data;
-
-  /* initialize members */
+  call_data *calld = (call_data *)elem->call_data;
+  calld->call_combiner = args->call_combiner;
+  calld->cancel_error = GRPC_ERROR_NONE;
   grpc_slice_buffer_init(&calld->slices);
-  GRPC_CLOSURE_INIT(&calld->got_slice, got_slice, elem,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->send_done, send_done, elem,
-                    grpc_schedule_on_exec_ctx);
-
+  GRPC_CLOSURE_INIT(&calld->start_send_message_batch_in_call_combiner,
+                    start_send_message_batch, elem, grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&calld->on_send_message_next_done,
+                    on_send_message_next_done, elem, grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&calld->send_message_on_complete, send_message_on_complete,
+                    elem, grpc_schedule_on_exec_ctx);
   return GRPC_ERROR_NONE;
 }
 
@@ -385,14 +473,9 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
                               grpc_closure *ignored) {
-  /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices);
-  gpr_atm imstate =
-      gpr_atm_no_barrier_load(&calld->send_initial_metadata_state);
-  if (imstate & CANCELLED_BIT) {
-    GRPC_ERROR_UNREF((grpc_error *)(imstate & ~CANCELLED_BIT));
-  }
+  GRPC_ERROR_UNREF(calld->cancel_error);
 }
 
 /* Constructor for channel_data */
@@ -401,6 +484,7 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_element_args *args) {
   channel_data *channeld = elem->channel_data;
 
+  /* Configuration for message compression */
   channeld->enabled_algorithms_bitset =
       grpc_channel_args_compression_algorithm_get_states(args->channel_args);
 
@@ -415,16 +499,32 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
     channeld->default_compression_algorithm = GRPC_COMPRESS_NONE;
   }
 
-  channeld->supported_compression_algorithms = 1; /* always support identity */
-  for (grpc_compression_algorithm algo_idx = 1;
-       algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
-    /* skip disabled algorithms */
-    if (!GPR_BITGET(channeld->enabled_algorithms_bitset, algo_idx)) {
-      continue;
-    }
-    channeld->supported_compression_algorithms |= 1u << algo_idx;
+  channeld->supported_compression_algorithms =
+      (((1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1) &
+       channeld->enabled_algorithms_bitset) |
+      1u;
+
+  /* Configuration for stream compression */
+  channeld->enabled_stream_compression_algorithms_bitset =
+      grpc_channel_args_stream_compression_algorithm_get_states(
+          args->channel_args);
+
+  channeld->default_stream_compression_algorithm =
+      grpc_channel_args_get_stream_compression_algorithm(args->channel_args);
+
+  if (!GPR_BITGET(channeld->enabled_stream_compression_algorithms_bitset,
+                  channeld->default_stream_compression_algorithm)) {
+    gpr_log(GPR_DEBUG,
+            "stream compression algorithm %d not enabled: switching to none",
+            channeld->default_stream_compression_algorithm);
+    channeld->default_stream_compression_algorithm = GRPC_STREAM_COMPRESS_NONE;
   }
 
+  channeld->supported_stream_compression_algorithms =
+      (((1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1) &
+       channeld->enabled_stream_compression_algorithms_bitset) |
+      1u;
+
   GPR_ASSERT(!args->is_last);
   return GRPC_ERROR_NONE;
 }
@@ -443,6 +543,5 @@ const grpc_channel_filter grpc_message_compress_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
-    "compress"};
+    "message_compress"};
diff --git a/src/core/ext/filters/http/server/http_server_filter.c b/src/core/ext/filters/http/server/http_server_filter.c
index b145f12aff..a10e69ba59 100644
--- a/src/core/ext/filters/http/server/http_server_filter.c
+++ b/src/core/ext/filters/http/server/http_server_filter.c
@@ -32,6 +32,8 @@
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
 typedef struct call_data {
+  grpc_call_combiner *call_combiner;
+
   grpc_linked_mdelem status;
   grpc_linked_mdelem content_type;
 
@@ -281,7 +283,11 @@ static void hs_on_complete(grpc_exec_ctx *exec_ctx, void *user_data,
     *calld->pp_recv_message = calld->payload_bin_delivered
                                   ? NULL
                                   : (grpc_byte_stream *)&calld->read_stream;
-    GRPC_CLOSURE_RUN(exec_ctx, calld->recv_message_ready, GRPC_ERROR_REF(err));
+    // Re-enter call combiner for recv_message_ready, since the surface
+    // code will release the call combiner for each callback it receives.
+    GRPC_CALL_COMBINER_START(exec_ctx, calld->call_combiner,
+                             calld->recv_message_ready, GRPC_ERROR_REF(err),
+                             "resuming recv_message_ready from on_complete");
     calld->recv_message_ready = NULL;
     calld->payload_bin_delivered = true;
   }
@@ -293,15 +299,20 @@ static void hs_recv_message_ready(grpc_exec_ctx *exec_ctx, void *user_data,
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
   if (calld->seen_path_with_query) {
-    /* do nothing. This is probably a GET request, and payload will be returned
-    in hs_on_complete callback. */
+    // Do nothing. This is probably a GET request, and payload will be
+    // returned in hs_on_complete callback.
+    // Note that we release the call combiner here, so that other
+    // callbacks can run.
+    GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                            "pausing recv_message_ready until on_complete");
   } else {
     GRPC_CLOSURE_RUN(exec_ctx, calld->recv_message_ready, GRPC_ERROR_REF(err));
   }
 }
 
-static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                         grpc_transport_stream_op_batch *op) {
+static grpc_error *hs_mutate_op(grpc_exec_ctx *exec_ctx,
+                                grpc_call_element *elem,
+                                grpc_transport_stream_op_batch *op) {
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
 
@@ -323,10 +334,7 @@ static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
               server_filter_outgoing_metadata(
                   exec_ctx, elem,
                   op->payload->send_initial_metadata.send_initial_metadata));
-    if (error != GRPC_ERROR_NONE) {
-      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
-      return;
-    }
+    if (error != GRPC_ERROR_NONE) return error;
   }
 
   if (op->recv_initial_metadata) {
@@ -359,21 +367,25 @@ static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_error *error = server_filter_outgoing_metadata(
         exec_ctx, elem,
         op->payload->send_trailing_metadata.send_trailing_metadata);
-    if (error != GRPC_ERROR_NONE) {
-      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
-      return;
-    }
+    if (error != GRPC_ERROR_NONE) return error;
   }
+
+  return GRPC_ERROR_NONE;
 }
 
-static void hs_start_transport_op(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem,
-                                  grpc_transport_stream_op_batch *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-  GPR_TIMER_BEGIN("hs_start_transport_op", 0);
-  hs_mutate_op(exec_ctx, elem, op);
-  grpc_call_next_op(exec_ctx, elem, op);
-  GPR_TIMER_END("hs_start_transport_op", 0);
+static void hs_start_transport_stream_op_batch(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_transport_stream_op_batch *op) {
+  call_data *calld = elem->call_data;
+  GPR_TIMER_BEGIN("hs_start_transport_stream_op_batch", 0);
+  grpc_error *error = hs_mutate_op(exec_ctx, elem, op);
+  if (error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error,
+                                                       calld->call_combiner);
+  } else {
+    grpc_call_next_op(exec_ctx, elem, op);
+  }
+  GPR_TIMER_END("hs_start_transport_stream_op_batch", 0);
 }
 
 /* Constructor for call_data */
@@ -383,6 +395,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   /* initialize members */
+  calld->call_combiner = args->call_combiner;
   GRPC_CLOSURE_INIT(&calld->hs_on_recv, hs_on_recv, elem,
                     grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&calld->hs_on_complete, hs_on_complete, elem,
@@ -414,7 +427,7 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {}
 
 const grpc_channel_filter grpc_http_server_filter = {
-    hs_start_transport_op,
+    hs_start_transport_stream_op_batch,
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
@@ -423,6 +436,5 @@ const grpc_channel_filter grpc_http_server_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "http-server"};
diff --git a/src/core/ext/filters/load_reporting/load_reporting_filter.c b/src/core/ext/filters/load_reporting/load_reporting_filter.c
index 08474efb2e..17e946937f 100644
--- a/src/core/ext/filters/load_reporting/load_reporting_filter.c
+++ b/src/core/ext/filters/load_reporting/load_reporting_filter.c
@@ -223,6 +223,5 @@ const grpc_channel_filter grpc_load_reporting_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "load_reporting"};
diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c
index 7d748b9c32..16c85a70d0 100644
--- a/src/core/ext/filters/max_age/max_age_filter.c
+++ b/src/core/ext/filters/max_age/max_age_filter.c
@@ -391,7 +391,6 @@ const grpc_channel_filter grpc_max_age_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "max_age"};
 
diff --git a/src/core/ext/filters/message_size/message_size_filter.c b/src/core/ext/filters/message_size/message_size_filter.c
index 846c7df69a..47763b1deb 100644
--- a/src/core/ext/filters/message_size/message_size_filter.c
+++ b/src/core/ext/filters/message_size/message_size_filter.c
@@ -68,6 +68,7 @@ static void* message_size_limits_create_from_json(const grpc_json* json) {
 }
 
 typedef struct call_data {
+  grpc_call_combiner* call_combiner;
   message_size_limits limits;
   // Receive closures are chained: we inject this closure as the
   // recv_message_ready up-call on transport_stream_op, and remember to
@@ -131,7 +132,8 @@ static void start_transport_stream_op_batch(
         exec_ctx, op,
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(message_string),
                            GRPC_ERROR_INT_GRPC_STATUS,
-                           GRPC_STATUS_RESOURCE_EXHAUSTED));
+                           GRPC_STATUS_RESOURCE_EXHAUSTED),
+        calld->call_combiner);
     gpr_free(message_string);
     return;
   }
@@ -152,6 +154,7 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   const grpc_call_element_args* args) {
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
+  calld->call_combiner = args->call_combiner;
   calld->next_recv_message_ready = NULL;
   GRPC_CLOSURE_INIT(&calld->recv_message_ready, recv_message_ready, elem,
                     grpc_schedule_on_exec_ctx);
@@ -259,7 +262,6 @@ const grpc_channel_filter grpc_message_size_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "message_size"};
 
diff --git a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c
index b4d2cb4b8c..c8b2fe5f99 100644
--- a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c
+++ b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c
@@ -177,7 +177,6 @@ const grpc_channel_filter grpc_workaround_cronet_compression_filter = {
     0,
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "workaround_cronet_compression"};
 
diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.c b/src/core/ext/transport/chttp2/transport/bin_decoder.c
index fe7b7e4a42..5a99cbeffc 100644
--- a/src/core/ext/transport/chttp2/transport/bin_decoder.c
+++ b/src/core/ext/transport/chttp2/transport/bin_decoder.c
@@ -118,6 +118,7 @@ bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx) {
       switch (input_tail) {
         case 3:
           ctx->output_cur[1] = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+        /* fallthrough */
         case 2:
           ctx->output_cur[0] = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
       }
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
index 731ebf400f..2f0ac85152 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
@@ -114,11 +114,6 @@ static void connectivity_state_set(grpc_exec_ctx *exec_ctx,
                                    grpc_connectivity_state state,
                                    grpc_error *error, const char *reason);
 
-static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx,
-                                                     grpc_chttp2_transport *t,
-                                                     grpc_chttp2_stream *s,
-                                                     size_t max_size_hint,
-                                                     size_t have_already);
 static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
                                                 void *byte_stream,
                                                 grpc_error *error_ignored);
@@ -270,8 +265,9 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   t->endpoint_reading = 1;
   t->next_stream_id = is_client ? 1 : 2;
   t->is_client = is_client;
-  t->outgoing_window = DEFAULT_WINDOW;
-  t->incoming_window = DEFAULT_WINDOW;
+  t->flow_control.remote_window = DEFAULT_WINDOW;
+  t->flow_control.announced_window = DEFAULT_WINDOW;
+  t->flow_control.t = t;
   t->deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0;
   t->is_first_frame = true;
   grpc_connectivity_state_init(
@@ -308,10 +304,10 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                     keepalive_watchdog_fired_locked, t,
                     grpc_combiner_scheduler(t->combiner));
 
-  grpc_bdp_estimator_init(&t->bdp_estimator, t->peer_string);
-  t->last_pid_update = gpr_now(GPR_CLOCK_MONOTONIC);
+  grpc_bdp_estimator_init(&t->flow_control.bdp_estimator, t->peer_string);
+  t->flow_control.last_pid_update = gpr_now(GPR_CLOCK_MONOTONIC);
   grpc_pid_controller_init(
-      &t->pid_controller,
+      &t->flow_control.pid_controller,
       (grpc_pid_controller_args){.gain_p = 4,
                                  .gain_i = 8,
                                  .gain_d = 0,
@@ -344,7 +340,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
   t->sent_local_settings = 0;
   t->write_buffer_size = DEFAULT_WINDOW;
-  t->enable_bdp_probe = true;
+  t->flow_control.enable_bdp_probe = true;
 
   if (is_client) {
     grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string(
@@ -461,7 +457,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
             (grpc_integer_options){0, 0, MAX_WRITE_BUFFER_SIZE});
       } else if (0 ==
                  strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) {
-        t->enable_bdp_probe = grpc_channel_arg_get_integer(
+        t->flow_control.enable_bdp_probe = grpc_channel_arg_get_integer(
             &channel_args->args[i], (grpc_integer_options){1, 0, 1});
       } else if (0 == strcmp(channel_args->args[i].key,
                              GRPC_ARG_KEEPALIVE_TIME_MS)) {
@@ -710,6 +706,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
     post_destructive_reclaimer(exec_ctx, t);
   }
 
+  s->flow_control.s = s;
   GPR_TIMER_END("init_stream", 0);
 
   return 0;
@@ -730,6 +727,14 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp,
   grpc_slice_buffer_destroy_internal(exec_ctx,
                                      &s->unprocessed_incoming_frames_buffer);
   grpc_slice_buffer_destroy_internal(exec_ctx, &s->frame_storage);
+  if (s->compressed_data_buffer) {
+    grpc_slice_buffer_destroy_internal(exec_ctx, s->compressed_data_buffer);
+    gpr_free(s->compressed_data_buffer);
+  }
+  if (s->decompressed_data_buffer) {
+    grpc_slice_buffer_destroy_internal(exec_ctx, s->decompressed_data_buffer);
+    gpr_free(s->decompressed_data_buffer);
+  }
 
   grpc_chttp2_list_remove_stalled_by_transport(t, s);
   grpc_chttp2_list_remove_stalled_by_stream(t, s);
@@ -758,13 +763,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp,
   GRPC_ERROR_UNREF(s->write_closed_error);
   GRPC_ERROR_UNREF(s->byte_stream_error);
 
-  if (s->incoming_window_delta > 0) {
-    GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA(
-        "destroy", t, s, s->incoming_window_delta);
-  } else if (s->incoming_window_delta < 0) {
-    GRPC_CHTTP2_FLOW_CREDIT_STREAM_INCOMING_WINDOW_DELTA(
-        "destroy", t, s, -s->incoming_window_delta);
-  }
+  grpc_chttp2_flowctl_destroy_stream(&t->flow_control, &s->flow_control);
 
   GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "stream");
 
@@ -780,6 +779,15 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt;
   grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs;
 
+  if (s->stream_compression_ctx != NULL) {
+    grpc_stream_compression_context_destroy(s->stream_compression_ctx);
+    s->stream_compression_ctx = NULL;
+  }
+  if (s->stream_decompression_ctx != NULL) {
+    grpc_stream_compression_context_destroy(s->stream_decompression_ctx);
+    s->stream_decompression_ctx = NULL;
+  }
+
   s->destroy_stream_arg = then_schedule_closure;
   GRPC_CLOSURE_SCHED(
       exec_ctx, GRPC_CLOSURE_INIT(&s->destroy_stream, destroy_stream_locked, s,
@@ -1173,6 +1181,7 @@ static void continue_fetching_send_locked(grpc_exec_ctx *exec_ctx,
       return;  /* early out */
     }
     if (s->fetched_send_message_length == s->fetching_send_message->length) {
+      grpc_byte_stream_destroy(exec_ctx, s->fetching_send_message);
       int64_t notify_offset = s->next_message_end_offset;
       if (notify_offset <= s->flow_controlled_bytes_written) {
         grpc_chttp2_complete_closure_step(
@@ -1195,9 +1204,14 @@ static void continue_fetching_send_locked(grpc_exec_ctx *exec_ctx,
       return; /* early out */
     } else if (grpc_byte_stream_next(exec_ctx, s->fetching_send_message,
                                      UINT32_MAX, &s->complete_fetch_locked)) {
-      grpc_byte_stream_pull(exec_ctx, s->fetching_send_message,
-                            &s->fetching_slice);
-      add_fetched_slice_locked(exec_ctx, t, s);
+      grpc_error *error = grpc_byte_stream_pull(
+          exec_ctx, s->fetching_send_message, &s->fetching_slice);
+      if (error != GRPC_ERROR_NONE) {
+        grpc_byte_stream_destroy(exec_ctx, s->fetching_send_message);
+        grpc_chttp2_cancel_stream(exec_ctx, t, s, error);
+      } else {
+        add_fetched_slice_locked(exec_ctx, t, s);
+      }
     }
   }
 }
@@ -1214,10 +1228,9 @@ static void complete_fetch_locked(grpc_exec_ctx *exec_ctx, void *gs,
       continue_fetching_send_locked(exec_ctx, t, s);
     }
   }
-
   if (error != GRPC_ERROR_NONE) {
-    /* TODO(ctiller): what to do here */
-    abort();
+    grpc_byte_stream_destroy(exec_ctx, s->fetching_send_message);
+    grpc_chttp2_cancel_stream(exec_ctx, t, s, error);
   }
 }
 
@@ -1285,6 +1298,15 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   if (op->send_initial_metadata) {
     GPR_ASSERT(s->send_initial_metadata_finished == NULL);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
+
+    /* Identify stream compression */
+    if ((s->stream_compression_send_enabled =
+             (op_payload->send_initial_metadata.send_initial_metadata->idx.named
+                  .content_encoding != NULL)) == true) {
+      s->compressed_data_buffer = gpr_malloc(sizeof(grpc_slice_buffer));
+      grpc_slice_buffer_init(s->compressed_data_buffer);
+    }
+
     s->send_initial_metadata_finished = add_closure_barrier(on_complete);
     s->send_initial_metadata =
         op_payload->send_initial_metadata.send_initial_metadata;
@@ -1348,22 +1370,33 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
             "send_initial_metadata_finished");
       }
     }
+    if (op_payload->send_initial_metadata.peer_string != NULL) {
+      gpr_atm_rel_store(op_payload->send_initial_metadata.peer_string,
+                        (gpr_atm)gpr_strdup(t->peer_string));
+    }
   }
 
   if (op->send_message) {
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->fetching_send_message_finished = add_closure_barrier(op->on_complete);
     if (s->write_closed) {
+      // Return an error unless the client has already received trailing
+      // metadata from the server, since an application using a
+      // streaming call might send another message before getting a
+      // recv_message failure, breaking out of its loop, and then
+      // starting recv_trailing_metadata.
       grpc_chttp2_complete_closure_step(
           exec_ctx, t, s, &s->fetching_send_message_finished,
-          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-              "Attempt to send message after stream was closed",
-              &s->write_closed_error, 1),
+          t->is_client && s->received_trailing_metadata
+              ? GRPC_ERROR_NONE
+              : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                    "Attempt to send message after stream was closed",
+                    &s->write_closed_error, 1),
           "fetching_send_message_finished");
     } else {
       GPR_ASSERT(s->fetching_send_message == NULL);
-      uint8_t *frame_hdr =
-          grpc_slice_buffer_tiny_add(&s->flow_controlled_buffer, 5);
+      uint8_t *frame_hdr = grpc_slice_buffer_tiny_add(
+          &s->flow_controlled_buffer, GRPC_HEADER_SIZE_IN_BYTES);
       uint32_t flags = op_payload->send_message.send_message->flags;
       frame_hdr[0] = (flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0;
       size_t len = op_payload->send_message.send_message->length;
@@ -1444,6 +1477,10 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
         op_payload->recv_initial_metadata.recv_initial_metadata;
     s->trailing_metadata_available =
         op_payload->recv_initial_metadata.trailing_metadata_available;
+    if (op_payload->recv_initial_metadata.peer_string != NULL) {
+      gpr_atm_rel_store(op_payload->recv_initial_metadata.peer_string,
+                        (gpr_atm)gpr_strdup(t->peer_string));
+    }
     grpc_chttp2_maybe_complete_recv_initial_metadata(exec_ctx, t, s);
   }
 
@@ -1454,14 +1491,16 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
     s->recv_message_ready = op_payload->recv_message.recv_message_ready;
     s->recv_message = op_payload->recv_message.recv_message;
     if (s->id != 0) {
-      if (s->pending_byte_stream) {
+      if (!s->read_closed) {
         already_received = s->frame_storage.length;
-      } else {
-        already_received = s->frame_storage.length +
-                           s->unprocessed_incoming_frames_buffer.length;
+        grpc_chttp2_flowctl_incoming_bs_update(
+            &t->flow_control, &s->flow_control, GRPC_HEADER_SIZE_IN_BYTES,
+            already_received);
+        grpc_chttp2_act_on_flowctl_action(
+            exec_ctx,
+            grpc_chttp2_flowctl_get_action(&t->flow_control, &s->flow_control),
+            t, s);
       }
-      incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5,
-                                               already_received);
     }
     grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s);
   }
@@ -1698,10 +1737,43 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx,
         if (s->unprocessed_incoming_frames_buffer.length == 0) {
           grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer,
                                  &s->frame_storage);
+          s->unprocessed_incoming_frames_decompressed = false;
+        }
+        if (s->stream_compression_recv_enabled &&
+            !s->unprocessed_incoming_frames_decompressed) {
+          GPR_ASSERT(s->decompressed_data_buffer->length == 0);
+          bool end_of_context;
+          if (!s->stream_decompression_ctx) {
+            s->stream_decompression_ctx =
+                grpc_stream_compression_context_create(
+                    GRPC_STREAM_COMPRESSION_DECOMPRESS);
+          }
+          if (!grpc_stream_decompress(s->stream_decompression_ctx,
+                                      &s->unprocessed_incoming_frames_buffer,
+                                      s->decompressed_data_buffer, NULL,
+                                      GRPC_HEADER_SIZE_IN_BYTES,
+                                      &end_of_context)) {
+            grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                       &s->frame_storage);
+            grpc_slice_buffer_reset_and_unref_internal(
+                exec_ctx, &s->unprocessed_incoming_frames_buffer);
+            error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                "Stream decompression error.");
+          } else {
+            error = grpc_deframe_unprocessed_incoming_frames(
+                exec_ctx, &s->data_parser, s, s->decompressed_data_buffer, NULL,
+                s->recv_message);
+            if (end_of_context) {
+              grpc_stream_compression_context_destroy(
+                  s->stream_decompression_ctx);
+              s->stream_decompression_ctx = NULL;
+            }
+          }
+        } else {
+          error = grpc_deframe_unprocessed_incoming_frames(
+              exec_ctx, &s->data_parser, s,
+              &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message);
         }
-        error = grpc_deframe_unprocessed_incoming_frames(
-            exec_ctx, &s->data_parser, s,
-            &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message);
         if (error != GRPC_ERROR_NONE) {
           s->seen_error = true;
           grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
@@ -1739,8 +1811,35 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx,
     }
     bool pending_data = s->pending_byte_stream ||
                         s->unprocessed_incoming_frames_buffer.length > 0;
-    if (s->read_closed && s->frame_storage.length == 0 &&
-        (!pending_data || s->seen_error) &&
+    if (s->stream_compression_recv_enabled && s->read_closed &&
+        s->frame_storage.length > 0 && !pending_data && !s->seen_error &&
+        s->recv_trailing_metadata_finished != NULL) {
+      /* Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and
+       * maybe decompress the next 5 bytes in the stream. */
+      bool end_of_context;
+      if (!s->stream_decompression_ctx) {
+        s->stream_decompression_ctx = grpc_stream_compression_context_create(
+            GRPC_STREAM_COMPRESSION_DECOMPRESS);
+      }
+      if (!grpc_stream_decompress(s->stream_decompression_ctx,
+                                  &s->frame_storage,
+                                  &s->unprocessed_incoming_frames_buffer, NULL,
+                                  GRPC_HEADER_SIZE_IN_BYTES, &end_of_context)) {
+        grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage);
+        grpc_slice_buffer_reset_and_unref_internal(
+            exec_ctx, &s->unprocessed_incoming_frames_buffer);
+        s->seen_error = true;
+      } else {
+        if (s->unprocessed_incoming_frames_buffer.length > 0) {
+          s->unprocessed_incoming_frames_decompressed = true;
+        }
+        if (end_of_context) {
+          grpc_stream_compression_context_destroy(s->stream_decompression_ctx);
+          s->stream_decompression_ctx = NULL;
+        }
+      }
+    }
+    if (s->read_closed && s->frame_storage.length == 0 && !pending_data &&
         s->recv_trailing_metadata_finished != NULL) {
       grpc_chttp2_incoming_metadata_buffer_publish(
           exec_ctx, &s->metadata_buffer[1], s->recv_trailing_metadata);
@@ -2148,43 +2247,56 @@ static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
  * INPUT PROCESSING - PARSING
  */
 
-static void update_bdp(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
-                       double bdp_dbl) {
-  // initial window size bounded [1,2^31-1], but we set the min to 128.
-  int32_t bdp = GPR_CLAMP((int32_t)bdp_dbl, 128, INT32_MAX);
-  int64_t delta =
-      (int64_t)bdp -
-      (int64_t)t->settings[GRPC_LOCAL_SETTINGS]
-                          [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
-  if (delta == 0 || (delta > -bdp / 10 && delta < bdp / 10)) {
-    return;
+void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_flowctl_action action,
+                                       grpc_chttp2_transport *t,
+                                       grpc_chttp2_stream *s) {
+  switch (action.send_stream_update) {
+    case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED:
+      break;
+    case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY:
+      grpc_chttp2_become_writable(exec_ctx, t, s,
+                                  GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED,
+                                  "immediate stream flowctl");
+      break;
+    case GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE:
+      grpc_chttp2_become_writable(exec_ctx, t, s,
+                                  GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK,
+                                  "queue stream flowctl");
+      break;
   }
-  if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) {
-    gpr_log(GPR_DEBUG, "%s: update initial window size to %d", t->peer_string,
-            (int)bdp);
+  switch (action.send_transport_update) {
+    case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED:
+      break;
+    case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY:
+      grpc_chttp2_initiate_write(exec_ctx, t, "immediate transport flowctl");
+      break;
+    // this is the same as no action b/c every time the transport enters the
+    // writing path it will maybe do an update
+    case GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE:
+      break;
   }
-  queue_setting_update(exec_ctx, t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE,
-                       (uint32_t)bdp);
-}
-
-static void update_frame(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
-                         double bw_dbl, double bdp_dbl) {
-  int32_t bdp = (int32_t)GPR_CLAMP(bdp_dbl, 128.0, INT32_MAX);
-  int32_t target = (int32_t)GPR_MAX(bw_dbl / 1000, bdp);
-  // frame size is bounded [2^14,2^24-1]
-  int32_t frame_size = GPR_CLAMP(target, 16384, 16777215);
-  int64_t delta = (int64_t)frame_size -
-                  (int64_t)t->settings[GRPC_LOCAL_SETTINGS]
-                                      [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE];
-  if (delta == 0 || (delta > -frame_size / 10 && delta < frame_size / 10)) {
-    return;
+  if (action.send_setting_update != GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED) {
+    if (action.initial_window_size > 0) {
+      queue_setting_update(exec_ctx, t,
+                           GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE,
+                           (uint32_t)action.initial_window_size);
+    }
+    if (action.max_frame_size > 0) {
+      queue_setting_update(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE,
+                           (uint32_t)action.max_frame_size);
+    }
+    if (action.send_setting_update == GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY) {
+      grpc_chttp2_initiate_write(exec_ctx, t, "immediate setting update");
+    }
   }
-  if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) {
-    gpr_log(GPR_DEBUG, "%s: update max_frame size to %d", t->peer_string,
-            (int)frame_size);
+  if (action.need_ping) {
+    GRPC_CHTTP2_REF_TRANSPORT(t, "bdp_ping");
+    grpc_bdp_estimator_schedule_ping(&t->flow_control.bdp_estimator);
+    send_ping_locked(exec_ctx, t,
+                     GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE,
+                     &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked);
   }
-  queue_setting_update(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE,
-                       (uint32_t)frame_size);
 }
 
 static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx,
@@ -2222,7 +2334,6 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp,
   GPR_TIMER_BEGIN("reading_action_locked", 0);
 
   grpc_chttp2_transport *t = tp;
-  bool need_bdp_ping = false;
 
   GRPC_ERROR_REF(error);
 
@@ -2241,11 +2352,9 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp,
     grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE,
                              GRPC_ERROR_NONE};
     for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) {
-      if (grpc_bdp_estimator_add_incoming_bytes(
-              &t->bdp_estimator,
-              (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i]))) {
-        need_bdp_ping = true;
-      }
+      grpc_bdp_estimator_add_incoming_bytes(
+          &t->flow_control.bdp_estimator,
+          (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i]));
       errors[1] =
           grpc_chttp2_perform_read(exec_ctx, t, t->read_buffer.slices[i]);
     }
@@ -2261,8 +2370,8 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp,
     GPR_TIMER_END("reading_action.parse", 0);
 
     GPR_TIMER_BEGIN("post_parse_locked", 0);
-    if (t->initial_window_update != 0) {
-      if (t->initial_window_update > 0) {
+    if (t->flow_control.initial_window_update != 0) {
+      if (t->flow_control.initial_window_update > 0) {
         grpc_chttp2_stream *s;
         while (grpc_chttp2_list_pop_stalled_by_stream(t, &s)) {
           grpc_chttp2_become_writable(
@@ -2270,7 +2379,7 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp,
               "unstalled");
         }
       }
-      t->initial_window_update = 0;
+      t->flow_control.initial_window_update = 0;
     }
     GPR_TIMER_END("post_parse_locked", 0);
   }
@@ -2292,45 +2401,9 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp,
   if (keep_reading) {
     grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer,
                        &t->read_action_locked);
-
-    if (t->enable_bdp_probe) {
-      if (need_bdp_ping) {
-        GRPC_CHTTP2_REF_TRANSPORT(t, "bdp_ping");
-        grpc_bdp_estimator_schedule_ping(&t->bdp_estimator);
-        send_ping_locked(exec_ctx, t,
-                         GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE,
-                         &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked);
-      }
-
-      int64_t estimate = -1;
-      double bdp_guess = -1;
-      if (grpc_bdp_estimator_get_estimate(&t->bdp_estimator, &estimate)) {
-        double target = 1 + log2((double)estimate);
-        double memory_pressure = grpc_resource_quota_get_memory_pressure(
-            grpc_resource_user_quota(grpc_endpoint_get_resource_user(t->ep)));
-        if (memory_pressure > 0.8) {
-          target *= 1 - GPR_MIN(1, (memory_pressure - 0.8) / 0.1);
-        }
-        double bdp_error =
-            target - grpc_pid_controller_last(&t->pid_controller);
-        gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
-        gpr_timespec dt_timespec = gpr_time_sub(now, t->last_pid_update);
-        double dt = (double)dt_timespec.tv_sec + dt_timespec.tv_nsec * 1e-9;
-        if (dt > 0.1) {
-          dt = 0.1;
-        }
-        double log2_bdp_guess =
-            grpc_pid_controller_update(&t->pid_controller, bdp_error, dt);
-        bdp_guess = pow(2, log2_bdp_guess);
-        update_bdp(exec_ctx, t, bdp_guess);
-        t->last_pid_update = now;
-      }
-
-      double bw = -1;
-      if (grpc_bdp_estimator_get_bw(&t->bdp_estimator, &bw)) {
-        update_frame(exec_ctx, t, bw, bdp_guess);
-      }
-    }
+    grpc_chttp2_act_on_flowctl_action(
+        exec_ctx, grpc_chttp2_flowctl_get_bdp_action(&t->flow_control), t,
+        NULL);
     GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "keep_reading");
   } else {
     GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "reading_action");
@@ -2353,7 +2426,7 @@ static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp,
   if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) {
     grpc_timer_cancel(exec_ctx, &t->keepalive_ping_timer);
   }
-  grpc_bdp_estimator_start_ping(&t->bdp_estimator);
+  grpc_bdp_estimator_start_ping(&t->flow_control.bdp_estimator);
 }
 
 static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp,
@@ -2362,7 +2435,7 @@ static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp,
   if (GRPC_TRACER_ON(grpc_http_trace)) {
     gpr_log(GPR_DEBUG, "%s: Complete BDP ping", t->peer_string);
   }
-  grpc_bdp_estimator_complete_ping(&t->bdp_estimator);
+  grpc_bdp_estimator_complete_ping(&t->flow_control.bdp_estimator);
 
   GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "bdp_ping");
 }
@@ -2544,54 +2617,6 @@ static void incoming_byte_stream_unref(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx,
-                                                     grpc_chttp2_transport *t,
-                                                     grpc_chttp2_stream *s,
-                                                     size_t max_size_hint,
-                                                     size_t have_already) {
-  uint32_t max_recv_bytes;
-  uint32_t initial_window_size =
-      t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
-
-  /* clamp max recv hint to an allowable size */
-  if (max_size_hint >= UINT32_MAX - initial_window_size) {
-    max_recv_bytes = UINT32_MAX - initial_window_size;
-  } else {
-    max_recv_bytes = (uint32_t)max_size_hint;
-  }
-
-  /* account for bytes already received but unknown to higher layers */
-  if (max_recv_bytes >= have_already) {
-    max_recv_bytes -= (uint32_t)have_already;
-  } else {
-    max_recv_bytes = 0;
-  }
-
-  /* add some small lookahead to keep pipelines flowing */
-  GPR_ASSERT(max_recv_bytes <= UINT32_MAX - initial_window_size);
-  if (s->incoming_window_delta < max_recv_bytes && !s->read_closed) {
-    uint32_t add_max_recv_bytes =
-        (uint32_t)(max_recv_bytes - s->incoming_window_delta);
-    grpc_chttp2_stream_write_type write_type =
-        GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED;
-    if (s->incoming_window_delta + initial_window_size <
-        (int64_t)have_already) {
-      write_type = GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED;
-    }
-    GRPC_CHTTP2_FLOW_CREDIT_STREAM_INCOMING_WINDOW_DELTA("op", t, s,
-                                                         add_max_recv_bytes);
-    GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, announce_window,
-                                   add_max_recv_bytes);
-    if ((int64_t)s->incoming_window_delta + (int64_t)initial_window_size -
-            (int64_t)s->announce_window >
-        (int64_t)initial_window_size / 2) {
-      write_type = GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK;
-    }
-    grpc_chttp2_become_writable(exec_ctx, t, s, write_type,
-                                "read_incoming_stream");
-  }
-}
-
 static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx,
                                              void *argp,
                                              grpc_error *error_ignored) {
@@ -2600,13 +2625,20 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_stream *s = bs->stream;
 
   size_t cur_length = s->frame_storage.length;
-  incoming_byte_stream_update_flow_control(
-      exec_ctx, t, s, bs->next_action.max_size_hint, cur_length);
-
+  if (!s->read_closed) {
+    grpc_chttp2_flowctl_incoming_bs_update(&t->flow_control, &s->flow_control,
+                                           bs->next_action.max_size_hint,
+                                           cur_length);
+    grpc_chttp2_act_on_flowctl_action(
+        exec_ctx,
+        grpc_chttp2_flowctl_get_action(&t->flow_control, &s->flow_control), t,
+        s);
+  }
   GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0);
   if (s->frame_storage.length > 0) {
     grpc_slice_buffer_swap(&s->frame_storage,
                            &s->unprocessed_incoming_frames_buffer);
+    s->unprocessed_incoming_frames_decompressed = false;
     GRPC_CLOSURE_SCHED(exec_ctx, bs->next_action.on_complete, GRPC_ERROR_NONE);
   } else if (s->byte_stream_error != GRPC_ERROR_NONE) {
     GRPC_CLOSURE_SCHED(exec_ctx, bs->next_action.on_complete,
@@ -2668,17 +2700,44 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_incoming_byte_stream *bs =
       (grpc_chttp2_incoming_byte_stream *)byte_stream;
   grpc_chttp2_stream *s = bs->stream;
+  grpc_error *error;
 
   if (s->unprocessed_incoming_frames_buffer.length > 0) {
-    grpc_error *error = grpc_deframe_unprocessed_incoming_frames(
+    if (s->stream_compression_recv_enabled &&
+        !s->unprocessed_incoming_frames_decompressed) {
+      bool end_of_context;
+      if (!s->stream_decompression_ctx) {
+        s->stream_decompression_ctx = grpc_stream_compression_context_create(
+            GRPC_STREAM_COMPRESSION_DECOMPRESS);
+      }
+      if (!grpc_stream_decompress(s->stream_decompression_ctx,
+                                  &s->unprocessed_incoming_frames_buffer,
+                                  s->decompressed_data_buffer, NULL, MAX_SIZE_T,
+                                  &end_of_context)) {
+        error =
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Stream decompression error.");
+        return error;
+      }
+      GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0);
+      grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer,
+                             s->decompressed_data_buffer);
+      s->unprocessed_incoming_frames_decompressed = true;
+      if (end_of_context) {
+        grpc_stream_compression_context_destroy(s->stream_decompression_ctx);
+        s->stream_decompression_ctx = NULL;
+      }
+      if (s->unprocessed_incoming_frames_buffer.length == 0) {
+        *slice = grpc_empty_slice();
+      }
+    }
+    error = grpc_deframe_unprocessed_incoming_frames(
         exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer,
         slice, NULL);
     if (error != GRPC_ERROR_NONE) {
       return error;
     }
   } else {
-    grpc_error *error =
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message");
+    error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message");
     GRPC_CLOSURE_SCHED(exec_ctx, &s->reset_byte_stream, GRPC_ERROR_REF(error));
     return error;
   }
@@ -2686,22 +2745,9 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx,
   return GRPC_ERROR_NONE;
 }
 
-static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
-                                         grpc_byte_stream *byte_stream);
-
 static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
                                                 void *byte_stream,
-                                                grpc_error *error_ignored) {
-  grpc_chttp2_incoming_byte_stream *bs = byte_stream;
-  grpc_chttp2_stream *s = bs->stream;
-  grpc_chttp2_transport *t = s->t;
-
-  GPR_ASSERT(bs->base.destroy == incoming_byte_stream_destroy);
-  incoming_byte_stream_unref(exec_ctx, bs);
-  s->pending_byte_stream = false;
-  grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s);
-  grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s);
-}
+                                                grpc_error *error_ignored);
 
 static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
                                          grpc_byte_stream *byte_stream) {
@@ -2768,6 +2814,33 @@ grpc_error *grpc_chttp2_incoming_byte_stream_finished(
   return error;
 }
 
+static void incoming_byte_stream_shutdown(grpc_exec_ctx *exec_ctx,
+                                          grpc_byte_stream *byte_stream,
+                                          grpc_error *error) {
+  grpc_chttp2_incoming_byte_stream *bs =
+      (grpc_chttp2_incoming_byte_stream *)byte_stream;
+  GRPC_ERROR_UNREF(grpc_chttp2_incoming_byte_stream_finished(
+      exec_ctx, bs, error, true /* reset_on_error */));
+}
+
+static const grpc_byte_stream_vtable grpc_chttp2_incoming_byte_stream_vtable = {
+    incoming_byte_stream_next, incoming_byte_stream_pull,
+    incoming_byte_stream_shutdown, incoming_byte_stream_destroy};
+
+static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
+                                                void *byte_stream,
+                                                grpc_error *error_ignored) {
+  grpc_chttp2_incoming_byte_stream *bs = byte_stream;
+  grpc_chttp2_stream *s = bs->stream;
+  grpc_chttp2_transport *t = s->t;
+
+  GPR_ASSERT(bs->base.vtable == &grpc_chttp2_incoming_byte_stream_vtable);
+  incoming_byte_stream_unref(exec_ctx, bs);
+  s->pending_byte_stream = false;
+  grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s);
+  grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s);
+}
+
 grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s,
     uint32_t frame_size, uint32_t flags) {
@@ -2776,9 +2849,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create(
   incoming_byte_stream->base.length = frame_size;
   incoming_byte_stream->remaining_bytes = frame_size;
   incoming_byte_stream->base.flags = flags;
-  incoming_byte_stream->base.next = incoming_byte_stream_next;
-  incoming_byte_stream->base.pull = incoming_byte_stream_pull;
-  incoming_byte_stream->base.destroy = incoming_byte_stream_destroy;
+  incoming_byte_stream->base.vtable = &grpc_chttp2_incoming_byte_stream_vtable;
   gpr_ref_init(&incoming_byte_stream->refs, 2);
   incoming_byte_stream->transport = t;
   incoming_byte_stream->stream = s;
@@ -2875,91 +2946,6 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg,
 }
 
 /*******************************************************************************
- * TRACING
- */
-
-static char *format_flowctl_context_var(const char *context, const char *var,
-                                        int64_t val, uint32_t id) {
-  char *name;
-  if (context == NULL) {
-    name = gpr_strdup(var);
-  } else if (0 == strcmp(context, "t")) {
-    GPR_ASSERT(id == 0);
-    gpr_asprintf(&name, "TRANSPORT:%s", var);
-  } else if (0 == strcmp(context, "s")) {
-    GPR_ASSERT(id != 0);
-    gpr_asprintf(&name, "STREAM[%d]:%s", id, var);
-  } else {
-    gpr_asprintf(&name, "BAD_CONTEXT[%s][%d]:%s", context, id, var);
-  }
-  char *name_fld = gpr_leftpad(name, ' ', 64);
-  char *value;
-  gpr_asprintf(&value, "%" PRId64, val);
-  char *value_fld = gpr_leftpad(value, ' ', 8);
-  char *result;
-  gpr_asprintf(&result, "%s %s", name_fld, value_fld);
-  gpr_free(name);
-  gpr_free(name_fld);
-  gpr_free(value);
-  gpr_free(value_fld);
-  return result;
-}
-
-void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
-                               grpc_chttp2_flowctl_op op, const char *context1,
-                               const char *var1, const char *context2,
-                               const char *var2, int is_client,
-                               uint32_t stream_id, int64_t val1, int64_t val2) {
-  char *tmp_phase;
-  char *label1 = format_flowctl_context_var(context1, var1, val1, stream_id);
-  char *label2 = format_flowctl_context_var(context2, var2, val2, stream_id);
-  char *clisvr = is_client ? "client" : "server";
-  char *prefix;
-
-  tmp_phase = gpr_leftpad(phase, ' ', 8);
-  gpr_asprintf(&prefix, "FLOW %s: %s ", tmp_phase, clisvr);
-  gpr_free(tmp_phase);
-
-  switch (op) {
-    case GRPC_CHTTP2_FLOWCTL_MOVE:
-      if (val2 != 0) {
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sMOVE   %s <- %s giving %" PRId64, prefix, label1, label2,
-                val1 + val2);
-      }
-      break;
-    case GRPC_CHTTP2_FLOWCTL_CREDIT:
-      GPR_ASSERT(val2 >= 0);
-      if (val2 != 0) {
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sCREDIT %s by %s giving %" PRId64, prefix, label1, label2,
-                val1 + val2);
-      }
-      break;
-    case GRPC_CHTTP2_FLOWCTL_DEBIT:
-      GPR_ASSERT(val2 >= 0);
-      if (val2 != 0) {
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sDEBIT  %s by %s giving %" PRId64, prefix, label1, label2,
-                val1 - val2);
-      }
-      break;
-  }
-
-  gpr_free(label1);
-  gpr_free(label2);
-  gpr_free(prefix);
-}
-
-/*******************************************************************************
- * INTEGRATION GLUE
- */
-
-static char *chttp2_get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
-  return gpr_strdup(((grpc_chttp2_transport *)t)->peer_string);
-}
-
-/*******************************************************************************
  * MONITORING
  */
 static grpc_endpoint *chttp2_get_endpoint(grpc_exec_ctx *exec_ctx,
@@ -2976,7 +2962,6 @@ static const grpc_transport_vtable vtable = {sizeof(grpc_chttp2_stream),
                                              perform_transport_op,
                                              destroy_stream,
                                              destroy_transport,
-                                             chttp2_get_peer,
                                              chttp2_get_endpoint};
 
 grpc_transport *grpc_create_chttp2_transport(
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c
new file mode 100644
index 0000000000..cec99f6fb6
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/flow_control.c
@@ -0,0 +1,500 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/ext/transport/chttp2/transport/internal.h"
+
+#include <math.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/support/string.h"
+
+static uint32_t grpc_chttp2_target_announced_window(
+    const grpc_chttp2_transport_flowctl* tfc);
+
+#ifndef NDEBUG
+
+typedef struct {
+  int64_t remote_window;
+  int64_t target_window;
+  int64_t announced_window;
+  int64_t remote_window_delta;
+  int64_t local_window_delta;
+  int64_t announced_window_delta;
+  uint32_t local_init_window;
+  uint32_t local_max_frame;
+} shadow_flow_control;
+
+static void pretrace(shadow_flow_control* shadow_fc,
+                     grpc_chttp2_transport_flowctl* tfc,
+                     grpc_chttp2_stream_flowctl* sfc) {
+  shadow_fc->remote_window = tfc->remote_window;
+  shadow_fc->target_window = grpc_chttp2_target_announced_window(tfc);
+  shadow_fc->announced_window = tfc->announced_window;
+  if (sfc != NULL) {
+    shadow_fc->remote_window_delta = sfc->remote_window_delta;
+    shadow_fc->local_window_delta = sfc->local_window_delta;
+    shadow_fc->announced_window_delta = sfc->announced_window_delta;
+  }
+}
+
+#define TRACE_PADDING 30
+
+static char* fmt_int64_diff_str(int64_t old, int64_t new) {
+  char* str;
+  if (old != new) {
+    gpr_asprintf(&str, "%" PRId64 " -> %" PRId64 "", old, new);
+  } else {
+    gpr_asprintf(&str, "%" PRId64 "", old);
+  }
+  char* str_lp = gpr_leftpad(str, ' ', TRACE_PADDING);
+  gpr_free(str);
+  return str_lp;
+}
+
+static char* fmt_uint32_diff_str(uint32_t old, uint32_t new) {
+  char* str;
+  if (new > 0 && old != new) {
+    gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old, new);
+  } else {
+    gpr_asprintf(&str, "%" PRIu32 "", old);
+  }
+  char* str_lp = gpr_leftpad(str, ' ', TRACE_PADDING);
+  gpr_free(str);
+  return str_lp;
+}
+
+static void posttrace(shadow_flow_control* shadow_fc,
+                      grpc_chttp2_transport_flowctl* tfc,
+                      grpc_chttp2_stream_flowctl* sfc, char* reason) {
+  uint32_t acked_local_window =
+      tfc->t->settings[GRPC_SENT_SETTINGS]
+                      [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+  uint32_t remote_window =
+      tfc->t->settings[GRPC_PEER_SETTINGS]
+                      [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+  char* trw_str =
+      fmt_int64_diff_str(shadow_fc->remote_window, tfc->remote_window);
+  char* tlw_str = fmt_int64_diff_str(shadow_fc->target_window,
+                                     grpc_chttp2_target_announced_window(tfc));
+  char* taw_str =
+      fmt_int64_diff_str(shadow_fc->announced_window, tfc->announced_window);
+  char* srw_str;
+  char* slw_str;
+  char* saw_str;
+  if (sfc != NULL) {
+    srw_str = fmt_int64_diff_str(shadow_fc->remote_window_delta + remote_window,
+                                 sfc->remote_window_delta + remote_window);
+    slw_str =
+        fmt_int64_diff_str(shadow_fc->local_window_delta + acked_local_window,
+                           sfc->local_window_delta + acked_local_window);
+    saw_str = fmt_int64_diff_str(
+        shadow_fc->announced_window_delta + acked_local_window,
+        sfc->announced_window_delta + acked_local_window);
+  } else {
+    srw_str = gpr_leftpad("", ' ', TRACE_PADDING);
+    slw_str = gpr_leftpad("", ' ', TRACE_PADDING);
+    saw_str = gpr_leftpad("", ' ', TRACE_PADDING);
+  }
+  gpr_log(GPR_DEBUG,
+          "%p[%u][%s] | %s | trw:%s, ttw:%s, taw:%s, srw:%s, slw:%s, saw:%s",
+          tfc, sfc != NULL ? sfc->s->id : 0, tfc->t->is_client ? "cli" : "svr",
+          reason, trw_str, tlw_str, taw_str, srw_str, slw_str, saw_str);
+  gpr_free(trw_str);
+  gpr_free(tlw_str);
+  gpr_free(taw_str);
+  gpr_free(srw_str);
+  gpr_free(slw_str);
+  gpr_free(saw_str);
+}
+
+static char* urgency_to_string(grpc_chttp2_flowctl_urgency urgency) {
+  switch (urgency) {
+    case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED:
+      return "no action";
+    case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY:
+      return "update immediately";
+    case GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE:
+      return "queue update";
+    default:
+      GPR_UNREACHABLE_CODE(return "unknown");
+  }
+  GPR_UNREACHABLE_CODE(return "unknown");
+}
+
+static void trace_action(grpc_chttp2_transport_flowctl* tfc,
+                         grpc_chttp2_flowctl_action action) {
+  char* iw_str = fmt_uint32_diff_str(
+      tfc->t->settings[GRPC_SENT_SETTINGS]
+                      [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE],
+      action.initial_window_size);
+  char* mf_str = fmt_uint32_diff_str(
+      tfc->t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE],
+      action.max_frame_size);
+  gpr_log(GPR_DEBUG, "t[%s],  s[%s], settings[%s] iw:%s mf:%s",
+          urgency_to_string(action.send_transport_update),
+          urgency_to_string(action.send_stream_update),
+          urgency_to_string(action.send_setting_update), iw_str, mf_str);
+  gpr_free(iw_str);
+  gpr_free(mf_str);
+}
+
+#define PRETRACE(tfc, sfc)       \
+  shadow_flow_control shadow_fc; \
+  GRPC_FLOW_CONTROL_IF_TRACING(pretrace(&shadow_fc, tfc, sfc))
+#define POSTTRACE(tfc, sfc, reason) \
+  GRPC_FLOW_CONTROL_IF_TRACING(posttrace(&shadow_fc, tfc, sfc, reason))
+#define TRACEACTION(tfc, action) \
+  GRPC_FLOW_CONTROL_IF_TRACING(trace_action(tfc, action))
+#else
+#define PRETRACE(tfc, sfc)
+#define POSTTRACE(tfc, sfc, reason)
+#define TRACEACTION(tfc, action)
+#endif
+
+/* How many bytes of incoming flow control would we like to advertise */
+static uint32_t grpc_chttp2_target_announced_window(
+    const grpc_chttp2_transport_flowctl* tfc) {
+  return (uint32_t)GPR_MIN(
+      (int64_t)((1u << 31) - 1),
+      tfc->announced_stream_total_over_incoming_window +
+          tfc->t->settings[GRPC_SENT_SETTINGS]
+                          [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
+}
+
+// we have sent data on the wire, we must track this in our bookkeeping for the
+// remote peer's flow control.
+void grpc_chttp2_flowctl_sent_data(grpc_chttp2_transport_flowctl* tfc,
+                                   grpc_chttp2_stream_flowctl* sfc,
+                                   int64_t size) {
+  PRETRACE(tfc, sfc);
+  tfc->remote_window -= size;
+  sfc->remote_window_delta -= size;
+  POSTTRACE(tfc, sfc, "  data sent");
+}
+
+static void announced_window_delta_preupdate(grpc_chttp2_transport_flowctl* tfc,
+                                             grpc_chttp2_stream_flowctl* sfc) {
+  if (sfc->announced_window_delta > 0) {
+    tfc->announced_stream_total_over_incoming_window -=
+        sfc->announced_window_delta;
+  } else {
+    tfc->announced_stream_total_under_incoming_window +=
+        -sfc->announced_window_delta;
+  }
+}
+
+static void announced_window_delta_postupdate(
+    grpc_chttp2_transport_flowctl* tfc, grpc_chttp2_stream_flowctl* sfc) {
+  if (sfc->announced_window_delta > 0) {
+    tfc->announced_stream_total_over_incoming_window +=
+        sfc->announced_window_delta;
+  } else {
+    tfc->announced_stream_total_under_incoming_window -=
+        -sfc->announced_window_delta;
+  }
+}
+
+// We have received data from the wire. We must track this in our own flow
+// control bookkeeping.
+// Returns an error if the incoming frame violates our flow control.
+grpc_error* grpc_chttp2_flowctl_recv_data(grpc_chttp2_transport_flowctl* tfc,
+                                          grpc_chttp2_stream_flowctl* sfc,
+                                          int64_t incoming_frame_size) {
+  uint32_t sent_init_window =
+      tfc->t->settings[GRPC_SENT_SETTINGS]
+                      [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+  uint32_t acked_init_window =
+      tfc->t->settings[GRPC_ACKED_SETTINGS]
+                      [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+  PRETRACE(tfc, sfc);
+  if (incoming_frame_size > tfc->announced_window) {
+    char* msg;
+    gpr_asprintf(&msg,
+                 "frame of size %" PRId64 " overflows local window of %" PRId64,
+                 incoming_frame_size, tfc->announced_window);
+    grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg);
+    gpr_free(msg);
+    return err;
+  }
+
+  if (sfc != NULL) {
+    int64_t acked_stream_window =
+        sfc->announced_window_delta + acked_init_window;
+    int64_t sent_stream_window = sfc->announced_window_delta + sent_init_window;
+    if (incoming_frame_size > acked_stream_window) {
+      if (incoming_frame_size <= sent_stream_window) {
+        gpr_log(
+            GPR_ERROR,
+            "Incoming frame of size %" PRId64
+            " exceeds local window size of %" PRId64
+            ".\n"
+            "The (un-acked, future) window size would be %" PRId64
+            " which is not exceeded.\n"
+            "This would usually cause a disconnection, but allowing it due to"
+            "broken HTTP2 implementations in the wild.\n"
+            "See (for example) https://github.com/netty/netty/issues/6520.",
+            incoming_frame_size, acked_stream_window, sent_stream_window);
+      } else {
+        char* msg;
+        gpr_asprintf(&msg, "frame of size %" PRId64
+                           " overflows local window of %" PRId64,
+                     incoming_frame_size, acked_stream_window);
+        grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg);
+        gpr_free(msg);
+        return err;
+      }
+    }
+
+    announced_window_delta_preupdate(tfc, sfc);
+    sfc->announced_window_delta -= incoming_frame_size;
+    announced_window_delta_postupdate(tfc, sfc);
+    sfc->local_window_delta -= incoming_frame_size;
+  }
+
+  tfc->announced_window -= incoming_frame_size;
+
+  POSTTRACE(tfc, sfc, "  data recv");
+  return GRPC_ERROR_NONE;
+}
+
+// Returns a non zero announce integer if we should send a transport window
+// update
+uint32_t grpc_chttp2_flowctl_maybe_send_transport_update(
+    grpc_chttp2_transport_flowctl* tfc) {
+  PRETRACE(tfc, NULL);
+  uint32_t target_announced_window = grpc_chttp2_target_announced_window(tfc);
+  uint32_t threshold_to_send_transport_window_update =
+      tfc->t->outbuf.count > 0 ? 3 * target_announced_window / 4
+                               : target_announced_window / 2;
+  if (tfc->announced_window <= threshold_to_send_transport_window_update &&
+      tfc->announced_window != target_announced_window) {
+    uint32_t announce = (uint32_t)GPR_CLAMP(
+        target_announced_window - tfc->announced_window, 0, UINT32_MAX);
+    tfc->announced_window += announce;
+    POSTTRACE(tfc, NULL, "t updt sent");
+    return announce;
+  }
+  GRPC_FLOW_CONTROL_IF_TRACING(
+      gpr_log(GPR_DEBUG, "%p[0][%s] will not send transport update", tfc,
+              tfc->t->is_client ? "cli" : "svr"));
+  return 0;
+}
+
+// Returns a non zero announce integer if we should send a stream window update
+uint32_t grpc_chttp2_flowctl_maybe_send_stream_update(
+    grpc_chttp2_transport_flowctl* tfc, grpc_chttp2_stream_flowctl* sfc) {
+  PRETRACE(tfc, sfc);
+  if (sfc->local_window_delta > sfc->announced_window_delta) {
+    uint32_t announce = (uint32_t)GPR_CLAMP(
+        sfc->local_window_delta - sfc->announced_window_delta, 0, UINT32_MAX);
+    announced_window_delta_preupdate(tfc, sfc);
+    sfc->announced_window_delta += announce;
+    announced_window_delta_postupdate(tfc, sfc);
+    POSTTRACE(tfc, sfc, "s updt sent");
+    return announce;
+  }
+  GRPC_FLOW_CONTROL_IF_TRACING(
+      gpr_log(GPR_DEBUG, "%p[%u][%s] will not send stream update", tfc,
+              sfc->s->id, tfc->t->is_client ? "cli" : "svr"));
+  return 0;
+}
+
+// we have received a WINDOW_UPDATE frame for a transport
+void grpc_chttp2_flowctl_recv_transport_update(
+    grpc_chttp2_transport_flowctl* tfc, uint32_t size) {
+  PRETRACE(tfc, NULL);
+  tfc->remote_window += size;
+  POSTTRACE(tfc, NULL, "t updt recv");
+}
+
+// we have received a WINDOW_UPDATE frame for a stream
+void grpc_chttp2_flowctl_recv_stream_update(grpc_chttp2_transport_flowctl* tfc,
+                                            grpc_chttp2_stream_flowctl* sfc,
+                                            uint32_t size) {
+  PRETRACE(tfc, sfc);
+  sfc->remote_window_delta += size;
+  POSTTRACE(tfc, sfc, "s updt recv");
+}
+
+void grpc_chttp2_flowctl_incoming_bs_update(grpc_chttp2_transport_flowctl* tfc,
+                                            grpc_chttp2_stream_flowctl* sfc,
+                                            size_t max_size_hint,
+                                            size_t have_already) {
+  PRETRACE(tfc, sfc);
+  uint32_t max_recv_bytes;
+  uint32_t sent_init_window =
+      tfc->t->settings[GRPC_SENT_SETTINGS]
+                      [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+
+  /* clamp max recv hint to an allowable size */
+  if (max_size_hint >= UINT32_MAX - sent_init_window) {
+    max_recv_bytes = UINT32_MAX - sent_init_window;
+  } else {
+    max_recv_bytes = (uint32_t)max_size_hint;
+  }
+
+  /* account for bytes already received but unknown to higher layers */
+  if (max_recv_bytes >= have_already) {
+    max_recv_bytes -= (uint32_t)have_already;
+  } else {
+    max_recv_bytes = 0;
+  }
+
+  /* add some small lookahead to keep pipelines flowing */
+  GPR_ASSERT(max_recv_bytes <= UINT32_MAX - sent_init_window);
+  if (sfc->local_window_delta < max_recv_bytes) {
+    uint32_t add_max_recv_bytes =
+        (uint32_t)(max_recv_bytes - sfc->local_window_delta);
+    sfc->local_window_delta += add_max_recv_bytes;
+  }
+  POSTTRACE(tfc, sfc, "app st recv");
+}
+
+void grpc_chttp2_flowctl_destroy_stream(grpc_chttp2_transport_flowctl* tfc,
+                                        grpc_chttp2_stream_flowctl* sfc) {
+  announced_window_delta_preupdate(tfc, sfc);
+}
+
+// Returns an urgency with which to make an update
+static grpc_chttp2_flowctl_urgency delta_is_significant(
+    const grpc_chttp2_transport_flowctl* tfc, int32_t value,
+    grpc_chttp2_setting_id setting_id) {
+  int64_t delta = (int64_t)value -
+                  (int64_t)tfc->t->settings[GRPC_LOCAL_SETTINGS][setting_id];
+  // TODO(ncteisen): tune this
+  if (delta != 0 && (delta <= -value / 5 || delta >= value / 5)) {
+    return GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE;
+  } else {
+    return GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED;
+  }
+}
+
+// Takes in a target and uses the pid controller to return a stabilized
+// guess at the new bdp.
+static double get_pid_controller_guess(grpc_chttp2_transport_flowctl* tfc,
+                                       double target) {
+  double bdp_error = target - grpc_pid_controller_last(&tfc->pid_controller);
+  gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+  gpr_timespec dt_timespec = gpr_time_sub(now, tfc->last_pid_update);
+  double dt = (double)dt_timespec.tv_sec + dt_timespec.tv_nsec * 1e-9;
+  if (dt > 0.1) {
+    dt = 0.1;
+  }
+  double log2_bdp_guess =
+      grpc_pid_controller_update(&tfc->pid_controller, bdp_error, dt);
+  tfc->last_pid_update = now;
+  return pow(2, log2_bdp_guess);
+}
+
+// Take in a target and modifies it based on the memory pressure of the system
+static double get_target_under_memory_pressure(
+    grpc_chttp2_transport_flowctl* tfc, double target) {
+  // do not increase window under heavy memory pressure.
+  double memory_pressure = grpc_resource_quota_get_memory_pressure(
+      grpc_resource_user_quota(grpc_endpoint_get_resource_user(tfc->t->ep)));
+  if (memory_pressure > 0.8) {
+    target *= 1 - GPR_MIN(1, (memory_pressure - 0.8) / 0.1);
+  }
+  return target;
+}
+
+grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_action(
+    grpc_chttp2_transport_flowctl* tfc, grpc_chttp2_stream_flowctl* sfc) {
+  grpc_chttp2_flowctl_action action;
+  memset(&action, 0, sizeof(action));
+  uint32_t target_announced_window = grpc_chttp2_target_announced_window(tfc);
+  if (tfc->announced_window < target_announced_window / 2) {
+    action.send_transport_update = GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY;
+  }
+  // TODO(ncteisen): tune this
+  if (sfc != NULL && !sfc->s->read_closed) {
+    uint32_t sent_init_window =
+        tfc->t->settings[GRPC_SENT_SETTINGS]
+                        [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+    if ((int64_t)sfc->local_window_delta >
+            (int64_t)sfc->announced_window_delta &&
+        (int64_t)sfc->announced_window_delta + sent_init_window <=
+            sent_init_window / 2) {
+      action.send_stream_update = GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY;
+    } else if (sfc->local_window_delta > sfc->announced_window_delta) {
+      action.send_stream_update = GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE;
+    }
+  }
+  TRACEACTION(tfc, action);
+  return action;
+}
+
+grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_bdp_action(
+    grpc_chttp2_transport_flowctl* tfc) {
+  grpc_chttp2_flowctl_action action;
+  memset(&action, 0, sizeof(action));
+  if (tfc->enable_bdp_probe) {
+    action.need_ping = grpc_bdp_estimator_need_ping(&tfc->bdp_estimator);
+
+    // get bdp estimate and update initial_window accordingly.
+    int64_t estimate = -1;
+    int32_t bdp = -1;
+    if (grpc_bdp_estimator_get_estimate(&tfc->bdp_estimator, &estimate)) {
+      double target = 1 + log2((double)estimate);
+
+      // target might change based on how much memory pressure we are under
+      // TODO(ncteisen): experiment with setting target to be huge under low
+      // memory pressure.
+      target = get_target_under_memory_pressure(tfc, target);
+
+      // run our target through the pid controller to stabilize change.
+      // TODO(ncteisen): experiment with other controllers here.
+      double bdp_guess = get_pid_controller_guess(tfc, target);
+
+      // Though initial window 'could' drop to 0, we keep the floor at 128
+      bdp = GPR_MAX((int32_t)bdp_guess, 128);
+
+      grpc_chttp2_flowctl_urgency init_window_update_urgency =
+          delta_is_significant(tfc, bdp,
+                               GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE);
+      if (init_window_update_urgency != GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED) {
+        action.send_setting_update = init_window_update_urgency;
+        action.initial_window_size = (uint32_t)bdp;
+      }
+    }
+
+    // get bandwidth estimate and update max_frame accordingly.
+    double bw_dbl = -1;
+    if (grpc_bdp_estimator_get_bw(&tfc->bdp_estimator, &bw_dbl)) {
+      // we target the max of BDP or bandwidth in microseconds.
+      int32_t frame_size = (int32_t)GPR_CLAMP(
+          GPR_MAX((int32_t)bw_dbl / 1000, bdp), 16384, 16777215);
+      grpc_chttp2_flowctl_urgency frame_size_urgency = delta_is_significant(
+          tfc, frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE);
+      if (frame_size_urgency != GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED) {
+        if (frame_size_urgency > action.send_setting_update) {
+          action.send_setting_update = frame_size_urgency;
+        }
+        action.max_frame_size = (uint32_t)frame_size;
+      }
+    }
+  }
+
+  TRACEACTION(tfc, action);
+  return action;
+}
diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c
index dead6be77f..222d2177b2 100644
--- a/src/core/ext/transport/chttp2/transport/frame_data.c
+++ b/src/core/ext/transport/chttp2/transport/frame_data.c
@@ -293,7 +293,6 @@ grpc_error *grpc_chttp2_data_parser_parse(grpc_exec_ctx *exec_ctx, void *parser,
                                           grpc_chttp2_transport *t,
                                           grpc_chttp2_stream *s,
                                           grpc_slice slice, int is_last) {
-  /* grpc_error *error = parse_inner_buffer(exec_ctx, p, t, s, slice); */
   if (!s->pending_byte_stream) {
     grpc_slice_ref_internal(slice);
     grpc_slice_buffer_add(&s->frame_storage, slice);
@@ -304,6 +303,7 @@ grpc_error *grpc_chttp2_data_parser_parse(grpc_exec_ctx *exec_ctx, void *parser,
     grpc_slice_buffer_add(&s->unprocessed_incoming_frames_buffer, slice);
     GRPC_CLOSURE_SCHED(exec_ctx, s->on_next, GRPC_ERROR_NONE);
     s->on_next = NULL;
+    s->unprocessed_incoming_frames_decompressed = false;
   } else {
     grpc_slice_ref_internal(slice);
     grpc_slice_buffer_add(&s->frame_storage, slice);
diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c
index e3e432a94a..057d3d9ed3 100644
--- a/src/core/ext/transport/chttp2/transport/frame_settings.c
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.c
@@ -201,11 +201,13 @@ grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, void *p,
           }
           if (id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE &&
               parser->incoming_settings[id] != parser->value) {
-            t->initial_window_update +=
+            t->flow_control.initial_window_update +=
                 (int64_t)parser->value - parser->incoming_settings[id];
-            if (GRPC_TRACER_ON(grpc_http_trace)) {
-              gpr_log(GPR_DEBUG, "adding %d for initial_window change",
-                      (int)t->initial_window_update);
+            if (GRPC_TRACER_ON(grpc_http_trace) ||
+                GRPC_TRACER_ON(grpc_flowctl_trace)) {
+              gpr_log(GPR_DEBUG, "%p[%s] adding %d for initial_window change",
+                      t, t->is_client ? "cli" : "svr",
+                      (int)t->flow_control.initial_window_update);
             }
           }
           parser->incoming_settings[id] = parser->value;
diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c
index 682be2c89b..65f3b01d77 100644
--- a/src/core/ext/transport/chttp2/transport/frame_window_update.c
+++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c
@@ -95,8 +95,8 @@ grpc_error *grpc_chttp2_window_update_parser_parse(
 
     if (t->incoming_stream_id != 0) {
       if (s != NULL) {
-        GRPC_CHTTP2_FLOW_CREDIT_STREAM("parse", t, s, outgoing_window_delta,
-                                       received_update);
+        grpc_chttp2_flowctl_recv_stream_update(
+            &t->flow_control, &s->flow_control, received_update);
         if (grpc_chttp2_list_remove_stalled_by_stream(t, s)) {
           grpc_chttp2_become_writable(
               exec_ctx, t, s, GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED,
@@ -104,10 +104,10 @@ grpc_error *grpc_chttp2_window_update_parser_parse(
         }
       }
     } else {
-      bool was_zero = t->outgoing_window <= 0;
-      GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parse", t, outgoing_window,
-                                        received_update);
-      bool is_zero = t->outgoing_window <= 0;
+      bool was_zero = t->flow_control.remote_window <= 0;
+      grpc_chttp2_flowctl_recv_transport_update(&t->flow_control,
+                                                received_update);
+      bool is_zero = t->flow_control.remote_window <= 0;
       if (was_zero && !is_zero) {
         grpc_chttp2_initiate_write(exec_ctx, t, "new_global_flow_control");
       }
diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c
index 7f37365558..c21d76ba71 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_parser.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c
@@ -1655,6 +1655,23 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp,
   GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst");
 }
 
+static void parse_stream_compression_md(grpc_exec_ctx *exec_ctx,
+                                        grpc_chttp2_transport *t,
+                                        grpc_chttp2_stream *s,
+                                        grpc_metadata_batch *initial_metadata) {
+  if (initial_metadata->idx.named.content_encoding != NULL) {
+    grpc_slice content_encoding =
+        GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md);
+    if (!grpc_slice_eq(content_encoding, GRPC_MDSTR_IDENTITY)) {
+      if (grpc_slice_eq(content_encoding, GRPC_MDSTR_GZIP)) {
+        s->stream_compression_recv_enabled = true;
+        s->decompressed_data_buffer = gpr_malloc(sizeof(grpc_slice_buffer));
+        grpc_slice_buffer_init(s->decompressed_data_buffer);
+      }
+    }
+  }
+}
+
 grpc_error *grpc_chttp2_header_parser_parse(grpc_exec_ctx *exec_ctx,
                                             void *hpack_parser,
                                             grpc_chttp2_transport *t,
@@ -1681,9 +1698,16 @@ grpc_error *grpc_chttp2_header_parser_parse(grpc_exec_ctx *exec_ctx,
     if (s != NULL) {
       if (parser->is_boundary) {
         if (s->header_frames_received == GPR_ARRAY_SIZE(s->metadata_buffer)) {
+          GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0);
           return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
               "Too many trailer frames");
         }
+        /* Process stream compression md element if it exists */
+        if (s->header_frames_received ==
+            0) { /* Only acts on initial metadata */
+          parse_stream_compression_md(exec_ctx, t, s,
+                                      &s->metadata_buffer[0].batch);
+        }
         s->published_metadata[s->header_frames_received] =
             GRPC_METADATA_PUBLISHED_FROM_WIRE;
         maybe_complete_funcs[s->header_frames_received](exec_ctx, t, s);
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 4563b78e75..9fff30d54f 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -213,6 +213,45 @@ typedef enum {
   GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED,
 } grpc_chttp2_keepalive_state;
 
+typedef struct {
+  /** initial window change. This is tracked as we parse settings frames from
+   * the remote peer. If there is a positive delta, then we will make all
+   * streams readable since they may have become unstalled */
+  int64_t initial_window_update;
+
+  /** Our bookkeeping for the remote peer's available window */
+  int64_t remote_window;
+
+  /** calculating what we should give for local window:
+      we track the total amount of flow control over initial window size
+      across all streams: this is data that we want to receive right now (it
+      has an outstanding read)
+      and the total amount of flow control under initial window size across all
+      streams: this is data we've read early
+      we want to adjust incoming_window such that:
+      incoming_window = total_over - max(bdp - total_under, 0) */
+  int64_t announced_stream_total_over_incoming_window;
+  int64_t announced_stream_total_under_incoming_window;
+
+  /** This is out window according to what we have sent to our remote peer. The
+   * difference between this and target window is what we use to decide when
+   * to send WINDOW_UPDATE frames. */
+  int64_t announced_window;
+
+  /** should we probe bdp? */
+  bool enable_bdp_probe;
+
+  /* bdp estimation */
+  grpc_bdp_estimator bdp_estimator;
+
+  /* pid controller */
+  grpc_pid_controller pid_controller;
+  gpr_timespec last_pid_update;
+
+  // pointer back to transport for tracing
+  const grpc_chttp2_transport *t;
+} grpc_chttp2_transport_flowctl;
+
 struct grpc_chttp2_transport {
   grpc_transport base; /* must be first */
   gpr_refcount refs;
@@ -232,9 +271,6 @@ struct grpc_chttp2_transport {
   /** is there a read request to the endpoint outstanding? */
   uint8_t endpoint_reading;
 
-  /** should we probe bdp? */
-  bool enable_bdp_probe;
-
   grpc_chttp2_optimization_target opt_target;
 
   /** various lists of streams */
@@ -271,7 +307,6 @@ struct grpc_chttp2_transport {
   grpc_slice_buffer outbuf;
   /** hpack encoding */
   grpc_chttp2_hpack_compressor hpack_compressor;
-  int64_t outgoing_window;
   /** is this a client? */
   uint8_t is_client;
 
@@ -328,21 +363,7 @@ struct grpc_chttp2_transport {
   /** parser for goaway frames */
   grpc_chttp2_goaway_parser goaway_parser;
 
-  /** initial window change */
-  int64_t initial_window_update;
-
-  /** window available for peer to send to us */
-  int64_t incoming_window;
-  /** calculating what we should give for incoming window:
-      we track the total amount of flow control over initial window size
-      across all streams: this is data that we want to receive right now (it
-      has an outstanding read)
-      and the total amount of flow control under initial window size across all
-      streams: this is data we've read early
-      we want to adjust incoming_window such that:
-      incoming_window = total_over - max(bdp - total_under, 0) */
-  int64_t stream_total_over_incoming_window;
-  int64_t stream_total_under_incoming_window;
+  grpc_chttp2_transport_flowctl flow_control;
 
   /* deframing */
   grpc_chttp2_deframe_transport_state deframe_state;
@@ -369,11 +390,8 @@ struct grpc_chttp2_transport {
   grpc_chttp2_write_cb *write_cb_pool;
 
   /* bdp estimator */
-  grpc_bdp_estimator bdp_estimator;
-  grpc_pid_controller pid_controller;
   grpc_closure start_bdp_ping_locked;
   grpc_closure finish_bdp_ping_locked;
-  gpr_timespec last_pid_update;
 
   /* if non-NULL, close the transport with this error when writes are finished
    */
@@ -422,6 +440,25 @@ typedef enum {
   GPRC_METADATA_PUBLISHED_AT_CLOSE
 } grpc_published_metadata_method;
 
+typedef struct {
+  /** window available for us to send to peer, over or under the initial window
+   * size of the transport... ie:
+   * remote_window = remote_window_delta + transport.initial_window_size */
+  int64_t remote_window_delta;
+
+  /** window available for peer to send to us (as a delta on
+   * transport.initial_window_size)
+   * local_window = local_window_delta + transport.initial_window_size */
+  int64_t local_window_delta;
+
+  /** window available for peer to send to us over this stream that we have
+   * announced to the peer */
+  int64_t announced_window_delta;
+
+  // read only pointer back to stream for data
+  const grpc_chttp2_stream *s;
+} grpc_chttp2_stream_flowctl;
+
 struct grpc_chttp2_stream {
   grpc_chttp2_transport *t;
   grpc_stream_refcount *refcount;
@@ -435,10 +472,6 @@ struct grpc_chttp2_stream {
   /** HTTP2 stream id for this stream, or zero if one has not been assigned */
   uint32_t id;
 
-  /** window available for us to send to peer, over or under the initial window
-   * size of the transport... ie:
-   * outgoing_window = outgoing_window_delta + transport.initial_window_size */
-  int64_t outgoing_window_delta;
   /** things the upper layers would like to send */
   grpc_metadata_batch *send_initial_metadata;
   grpc_closure *send_initial_metadata_finished;
@@ -476,6 +509,8 @@ struct grpc_chttp2_stream {
   /** Are we buffering writes on this stream? If yes, we won't become writable
       until there's enough queued up in the flow_controlled_buffer */
   bool write_buffering;
+  /** Has trailing metadata been received. */
+  bool received_trailing_metadata;
 
   /** the error that resulted in this stream being read-closed */
   grpc_error *read_closed_error;
@@ -505,10 +540,6 @@ struct grpc_chttp2_stream {
   grpc_error *forced_close_error;
   /** how many header frames have we received? */
   uint8_t header_frames_received;
-  /** window available for peer to send to us (as a delta on
-   * transport.initial_window_size)
-   * incoming_window = incoming_window_delta + transport.initial_window_size */
-  int64_t incoming_window_delta;
   /** parsing state for data frames */
   /* Accessed only by transport thread when stream->pending_byte_stream == false
    * Accessed only by application thread when stream->pending_byte_stream ==
@@ -519,13 +550,34 @@ struct grpc_chttp2_stream {
 
   bool sent_initial_metadata;
   bool sent_trailing_metadata;
-  /** how much window should we announce? */
-  uint32_t announce_window;
+
+  grpc_chttp2_stream_flowctl flow_control;
+
   grpc_slice_buffer flow_controlled_buffer;
 
   grpc_chttp2_write_cb *on_write_finished_cbs;
   grpc_chttp2_write_cb *finish_after_write;
   size_t sending_bytes;
+
+  /** Whether stream compression send is enabled */
+  bool stream_compression_recv_enabled;
+  /** Whether stream compression recv is enabled */
+  bool stream_compression_send_enabled;
+  /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed
+   */
+  bool unprocessed_incoming_frames_decompressed;
+  /** Stream compression decompress context */
+  grpc_stream_compression_context *stream_decompression_ctx;
+  /** Stream compression compress context */
+  grpc_stream_compression_context *stream_compression_ctx;
+
+  /** Buffer storing data that is compressed but not sent */
+  grpc_slice_buffer *compressed_data_buffer;
+  /** Amount of uncompressed bytes sent out when compressed_data_buffer is
+   * emptied */
+  size_t uncompressed_data_size;
+  /** Temporary buffer storing decompressed data */
+  grpc_slice_buffer *decompressed_data_buffer;
 };
 
 /** Transport writing call flow:
@@ -601,6 +653,81 @@ bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t,
 bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s);
 
+/********* Flow Control ***************/
+
+// we have sent data on the wire
+void grpc_chttp2_flowctl_sent_data(grpc_chttp2_transport_flowctl *tfc,
+                                   grpc_chttp2_stream_flowctl *sfc,
+                                   int64_t size);
+
+// we have received data from the wire
+grpc_error *grpc_chttp2_flowctl_recv_data(grpc_chttp2_transport_flowctl *tfc,
+                                          grpc_chttp2_stream_flowctl *sfc,
+                                          int64_t incoming_frame_size);
+
+// returns an announce if we should send a transport update to our peer,
+// else returns zero
+uint32_t grpc_chttp2_flowctl_maybe_send_transport_update(
+    grpc_chttp2_transport_flowctl *tfc);
+
+// returns an announce if we should send a stream update to our peer, else
+// returns zero
+uint32_t grpc_chttp2_flowctl_maybe_send_stream_update(
+    grpc_chttp2_transport_flowctl *tfc, grpc_chttp2_stream_flowctl *sfc);
+
+// we have received a WINDOW_UPDATE frame for a transport
+void grpc_chttp2_flowctl_recv_transport_update(
+    grpc_chttp2_transport_flowctl *tfc, uint32_t size);
+
+// we have received a WINDOW_UPDATE frame for a stream
+void grpc_chttp2_flowctl_recv_stream_update(grpc_chttp2_transport_flowctl *tfc,
+                                            grpc_chttp2_stream_flowctl *sfc,
+                                            uint32_t size);
+
+// the application is asking for a certain amount of bytes
+void grpc_chttp2_flowctl_incoming_bs_update(grpc_chttp2_transport_flowctl *tfc,
+                                            grpc_chttp2_stream_flowctl *sfc,
+                                            size_t max_size_hint,
+                                            size_t have_already);
+
+void grpc_chttp2_flowctl_destroy_stream(grpc_chttp2_transport_flowctl *tfc,
+                                        grpc_chttp2_stream_flowctl *sfc);
+
+typedef enum {
+  // Nothing to be done.
+  GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED = 0,
+  // Initiate a write to update the initial window immediately.
+  GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY,
+  // Push the flow control update into a send buffer, to be sent
+  // out the next time a write is initiated.
+  GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE,
+} grpc_chttp2_flowctl_urgency;
+
+typedef struct {
+  grpc_chttp2_flowctl_urgency send_stream_update;
+  grpc_chttp2_flowctl_urgency send_transport_update;
+  grpc_chttp2_flowctl_urgency send_setting_update;
+  uint32_t initial_window_size;
+  uint32_t max_frame_size;
+  bool need_ping;
+} grpc_chttp2_flowctl_action;
+
+// Reads the flow control data and returns and actionable struct that will tell
+// chttp2 exactly what it needs to do
+grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_action(
+    grpc_chttp2_transport_flowctl *tfc, grpc_chttp2_stream_flowctl *sfc);
+
+grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_bdp_action(
+    grpc_chttp2_transport_flowctl *tfc);
+
+// Takes in a flow control action and performs all the needed operations.
+void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_flowctl_action action,
+                                       grpc_chttp2_transport *t,
+                                       grpc_chttp2_stream *s);
+
+/********* End of Flow Control ***************/
+
 grpc_chttp2_stream *grpc_chttp2_parsing_lookup_stream(grpc_chttp2_transport *t,
                                                       uint32_t id);
 grpc_chttp2_stream *grpc_chttp2_parsing_accept_stream(grpc_exec_ctx *exec_ctx,
@@ -621,6 +748,9 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
                                        grpc_closure **pclosure,
                                        grpc_error *error, const char *desc);
 
+#define GRPC_HEADER_SIZE_IN_BYTES 5
+#define MAX_SIZE_T (~(size_t)0)
+
 #define GRPC_CHTTP2_CLIENT_CONNECT_STRING "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
 #define GRPC_CHTTP2_CLIENT_CONNECT_STRLEN \
   (sizeof(GRPC_CHTTP2_CLIENT_CONNECT_STRING) - 1)
@@ -628,126 +758,22 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
 extern grpc_tracer_flag grpc_http_trace;
 extern grpc_tracer_flag grpc_flowctl_trace;
 
+#ifndef NDEBUG
+#define GRPC_FLOW_CONTROL_IF_TRACING(stmt)   \
+  if (!(GRPC_TRACER_ON(grpc_flowctl_trace))) \
+    ;                                        \
+  else                                       \
+  stmt
+#else
+#define GRPC_FLOW_CONTROL_IF_TRACING(stmt)
+#endif
+
 #define GRPC_CHTTP2_IF_TRACING(stmt)      \
   if (!(GRPC_TRACER_ON(grpc_http_trace))) \
     ;                                     \
   else                                    \
   stmt
 
-typedef enum {
-  GRPC_CHTTP2_FLOWCTL_MOVE,
-  GRPC_CHTTP2_FLOWCTL_CREDIT,
-  GRPC_CHTTP2_FLOWCTL_DEBIT
-} grpc_chttp2_flowctl_op;
-
-#define GRPC_CHTTP2_FLOW_MOVE_COMMON(phase, transport, id1, id2, dst_context, \
-                                     dst_var, src_context, src_var)           \
-  do {                                                                        \
-    assert(id1 == id2);                                                       \
-    if (GRPC_TRACER_ON(grpc_flowctl_trace)) {                                 \
-      grpc_chttp2_flowctl_trace(                                              \
-          __FILE__, __LINE__, phase, GRPC_CHTTP2_FLOWCTL_MOVE, #dst_context,  \
-          #dst_var, #src_context, #src_var, transport->is_client, id1,        \
-          dst_context->dst_var, src_context->src_var);                        \
-    }                                                                         \
-    dst_context->dst_var += src_context->src_var;                             \
-    src_context->src_var = 0;                                                 \
-  } while (0)
-
-#define GRPC_CHTTP2_FLOW_MOVE_STREAM(phase, transport, dst_context, dst_var, \
-                                     src_context, src_var)                   \
-  GRPC_CHTTP2_FLOW_MOVE_COMMON(phase, transport, dst_context->id,            \
-                               src_context->id, dst_context, dst_var,        \
-                               src_context, src_var)
-#define GRPC_CHTTP2_FLOW_MOVE_TRANSPORT(phase, dst_context, dst_var,           \
-                                        src_context, src_var)                  \
-  GRPC_CHTTP2_FLOW_MOVE_COMMON(phase, dst_context, 0, 0, dst_context, dst_var, \
-                               src_context, src_var)
-
-#define GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, id, dst_context,      \
-                                       dst_var, amount)                        \
-  do {                                                                         \
-    if (GRPC_TRACER_ON(grpc_flowctl_trace)) {                                  \
-      grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase,                     \
-                                GRPC_CHTTP2_FLOWCTL_CREDIT, #dst_context,      \
-                                #dst_var, NULL, #amount, transport->is_client, \
-                                id, dst_context->dst_var, amount);             \
-    }                                                                          \
-    dst_context->dst_var += amount;                                            \
-  } while (0)
-
-#define GRPC_CHTTP2_FLOW_CREDIT_STREAM(phase, transport, dst_context, dst_var, \
-                                       amount)                                 \
-  GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, dst_context->id,            \
-                                 dst_context, dst_var, amount)
-#define GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT(phase, dst_context, dst_var, amount) \
-  GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, dst_context, 0, dst_context, dst_var,  \
-                                 amount)
-
-#define GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_PREUPDATE( \
-    phase, transport, dst_context)                               \
-  if (dst_context->incoming_window_delta < 0) {                  \
-    transport->stream_total_under_incoming_window +=             \
-        dst_context->incoming_window_delta;                      \
-  } else if (dst_context->incoming_window_delta > 0) {           \
-    transport->stream_total_over_incoming_window -=              \
-        dst_context->incoming_window_delta;                      \
-  }
-
-#define GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_POSTUPDATE( \
-    phase, transport, dst_context)                                \
-  if (dst_context->incoming_window_delta < 0) {                   \
-    transport->stream_total_under_incoming_window -=              \
-        dst_context->incoming_window_delta;                       \
-  } else if (dst_context->incoming_window_delta > 0) {            \
-    transport->stream_total_over_incoming_window +=               \
-        dst_context->incoming_window_delta;                       \
-  }
-
-#define GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA(                 \
-    phase, transport, dst_context, amount)                                   \
-  GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_PREUPDATE(phase, transport,  \
-                                                          dst_context);      \
-  GRPC_CHTTP2_FLOW_DEBIT_STREAM(phase, transport, dst_context,               \
-                                incoming_window_delta, amount);              \
-  GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_POSTUPDATE(phase, transport, \
-                                                           dst_context);
-
-#define GRPC_CHTTP2_FLOW_CREDIT_STREAM_INCOMING_WINDOW_DELTA(                \
-    phase, transport, dst_context, amount)                                   \
-  GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_PREUPDATE(phase, transport,  \
-                                                          dst_context);      \
-  GRPC_CHTTP2_FLOW_CREDIT_STREAM(phase, transport, dst_context,              \
-                                 incoming_window_delta, amount);             \
-  GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_POSTUPDATE(phase, transport, \
-                                                           dst_context);
-
-#define GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, id, dst_context,       \
-                                      dst_var, amount)                         \
-  do {                                                                         \
-    if (GRPC_TRACER_ON(grpc_flowctl_trace)) {                                  \
-      grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase,                     \
-                                GRPC_CHTTP2_FLOWCTL_DEBIT, #dst_context,       \
-                                #dst_var, NULL, #amount, transport->is_client, \
-                                id, dst_context->dst_var, amount);             \
-    }                                                                          \
-    dst_context->dst_var -= amount;                                            \
-  } while (0)
-
-#define GRPC_CHTTP2_FLOW_DEBIT_STREAM(phase, transport, dst_context, dst_var, \
-                                      amount)                                 \
-  GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, dst_context->id,            \
-                                dst_context, dst_var, amount)
-#define GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT(phase, dst_context, dst_var, amount) \
-  GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, dst_context, 0, dst_context, dst_var,  \
-                                amount)
-
-void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
-                               grpc_chttp2_flowctl_op op, const char *context1,
-                               const char *var1, const char *context2,
-                               const char *var2, int is_client,
-                               uint32_t stream_id, int64_t val1, int64_t val2);
-
 void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                              grpc_chttp2_stream *stream, grpc_error *error);
 void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx,
@@ -849,8 +875,6 @@ void grpc_chttp2_fail_pending_writes(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_transport *t,
                                      grpc_chttp2_stream *s, grpc_error *error);
 
-uint32_t grpc_chttp2_target_incoming_window(grpc_chttp2_transport *t);
-
 /** Set the default keepalive configurations, must only be called at
     initialization */
 void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args,
diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c
index 3c8b470b4f..19bd86fd0c 100644
--- a/src/core/ext/transport/chttp2/transport/parsing.c
+++ b/src/core/ext/transport/chttp2/transport/parsing.c
@@ -349,93 +349,25 @@ void grpc_chttp2_parsing_become_skip_parser(grpc_exec_ctx *exec_ctx,
                          t->parser == grpc_chttp2_header_parser_parse);
 }
 
-static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx,
-                                          grpc_chttp2_transport *t,
-                                          grpc_chttp2_stream *s) {
-  uint32_t incoming_frame_size = t->incoming_frame_size;
-  if (incoming_frame_size > t->incoming_window) {
-    char *msg;
-    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64,
-                 t->incoming_frame_size, t->incoming_window);
-    grpc_error *err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg);
-    gpr_free(msg);
-    return err;
-  }
-
-  if (s != NULL) {
-    if (incoming_frame_size >
-        s->incoming_window_delta +
-            t->settings[GRPC_ACKED_SETTINGS]
-                       [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) {
-      if (incoming_frame_size <=
-          s->incoming_window_delta +
-              t->settings[GRPC_SENT_SETTINGS]
-                         [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) {
-        gpr_log(
-            GPR_ERROR,
-            "Incoming frame of size %d exceeds incoming window size of %" PRId64
-            ".\n"
-            "The (un-acked, future) window size would be %" PRId64
-            " which is not exceeded.\n"
-            "This would usually cause a disconnection, but allowing it due to "
-            "broken HTTP2 implementations in the wild.\n"
-            "See (for example) https://github.com/netty/netty/issues/6520.",
-            t->incoming_frame_size,
-            s->incoming_window_delta +
-                t->settings[GRPC_ACKED_SETTINGS]
-                           [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE],
-            s->incoming_window_delta +
-                t->settings[GRPC_SENT_SETTINGS]
-                           [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
-      } else {
-        char *msg;
-        gpr_asprintf(&msg,
-                     "frame of size %d overflows incoming window of %" PRId64,
-                     t->incoming_frame_size,
-                     s->incoming_window_delta +
-                         t->settings[GRPC_ACKED_SETTINGS]
-                                    [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
-        grpc_error *err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg);
-        gpr_free(msg);
-        return err;
-      }
-    }
-
-    GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA("parse", t, s,
-                                                        incoming_frame_size);
-    if ((int64_t)s->incoming_window_delta - (int64_t)s->announce_window <=
-        -(int64_t)t->settings[GRPC_SENT_SETTINGS]
-                             [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] /
-            2) {
-      grpc_chttp2_become_writable(exec_ctx, t, s,
-                                  GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED,
-                                  "window-update-required");
-    }
-    s->received_bytes += incoming_frame_size;
-  }
-
-  uint32_t target_incoming_window = grpc_chttp2_target_incoming_window(t);
-  GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", t, incoming_window,
-                                   incoming_frame_size);
-  if (t->incoming_window <= target_incoming_window / 2) {
-    grpc_chttp2_initiate_write(exec_ctx, t, "flow_control");
-  }
-
-  return GRPC_ERROR_NONE;
-}
-
 static grpc_error *init_data_frame_parser(grpc_exec_ctx *exec_ctx,
                                           grpc_chttp2_transport *t) {
   grpc_chttp2_stream *s =
       grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id);
   grpc_error *err = GRPC_ERROR_NONE;
-  err = update_incoming_window(exec_ctx, t, s);
+  err = grpc_chttp2_flowctl_recv_data(&t->flow_control,
+                                      s == NULL ? NULL : &s->flow_control,
+                                      t->incoming_frame_size);
+  grpc_chttp2_act_on_flowctl_action(
+      exec_ctx, grpc_chttp2_flowctl_get_action(
+                    &t->flow_control, s == NULL ? NULL : &s->flow_control),
+      t, s);
   if (err != GRPC_ERROR_NONE) {
     goto error_handler;
   }
   if (s == NULL) {
     return init_skip_frame_parser(exec_ctx, t, 0);
   }
+  s->received_bytes += t->incoming_frame_size;
   s->stats.incoming.framing_bytes += 9;
   if (err == GRPC_ERROR_NONE && s->read_closed) {
     return init_skip_frame_parser(exec_ctx, t, 0);
@@ -657,6 +589,10 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx,
           "ignoring grpc_chttp2_stream with non-client generated index %d",
           t->incoming_stream_id));
       return init_skip_frame_parser(exec_ctx, t, 1);
+    } else if (grpc_chttp2_stream_map_size(&t->stream_map) >=
+               t->settings[GRPC_ACKED_SETTINGS]
+                          [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS]) {
+      return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Max stream count exceeded");
     }
     t->last_new_stream_id = t->incoming_stream_id;
     s = t->incoming_stream =
@@ -687,6 +623,7 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx,
           *s->trailing_metadata_available = true;
         }
         t->hpack_parser.on_header = on_trailing_header;
+        s->received_trailing_metadata = true;
       } else {
         GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "parsing initial_metadata"));
         t->hpack_parser.on_header = on_initial_header;
@@ -695,6 +632,7 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx,
     case 1:
       GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "parsing trailing_metadata"));
       t->hpack_parser.on_header = on_trailing_header;
+      s->received_trailing_metadata = true;
       break;
     case 2:
       gpr_log(GPR_ERROR, "too many header frames received");
diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c
index 1bf5b34510..7cc85dea9c 100644
--- a/src/core/ext/transport/chttp2/transport/stream_lists.c
+++ b/src/core/ext/transport/chttp2/transport/stream_lists.c
@@ -150,12 +150,17 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t,
 
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s) {
+  GRPC_FLOW_CONTROL_IF_TRACING(
+      gpr_log(GPR_DEBUG, "stream %u stalled by transport", s->id));
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
 }
 
 bool grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream **s) {
-  return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
+  bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
+  GRPC_FLOW_CONTROL_IF_TRACING(if (ret) gpr_log(
+      GPR_DEBUG, "stream %u un-stalled by transport", (*s)->id));
+  return ret;
 }
 
 void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t,
@@ -165,15 +170,23 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t,
 
 void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport *t,
                                             grpc_chttp2_stream *s) {
+  GRPC_FLOW_CONTROL_IF_TRACING(
+      gpr_log(GPR_DEBUG, "stream %u stalled by stream", s->id));
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
 }
 
 bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t,
                                             grpc_chttp2_stream **s) {
-  return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
+  bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
+  GRPC_FLOW_CONTROL_IF_TRACING(
+      if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", (*s)->id));
+  return ret;
 }
 
 bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s) {
-  return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
+  bool ret = stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
+  GRPC_FLOW_CONTROL_IF_TRACING(
+      if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", s->id));
+  return ret;
 }
diff --git a/src/core/ext/transport/chttp2/transport/varint.c b/src/core/ext/transport/chttp2/transport/varint.c
index 5f93a23a94..0d94ddcbc3 100644
--- a/src/core/ext/transport/chttp2/transport/varint.c
+++ b/src/core/ext/transport/chttp2/transport/varint.c
@@ -37,12 +37,16 @@ void grpc_chttp2_hpack_write_varint_tail(uint32_t tail_value, uint8_t* target,
   switch (tail_length) {
     case 5:
       target[4] = (uint8_t)((tail_value >> 28) | 0x80);
+    /* fallthrough */
     case 4:
       target[3] = (uint8_t)((tail_value >> 21) | 0x80);
+    /* fallthrough */
     case 3:
       target[2] = (uint8_t)((tail_value >> 14) | 0x80);
+    /* fallthrough */
     case 2:
       target[1] = (uint8_t)((tail_value >> 7) | 0x80);
+    /* fallthrough */
     case 1:
       target[0] = (uint8_t)((tail_value) | 0x80);
   }
diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c
index 315f2a67a2..80eb51ff0d 100644
--- a/src/core/ext/transport/chttp2/transport/writing.c
+++ b/src/core/ext/transport/chttp2/transport/writing.c
@@ -148,15 +148,6 @@ static bool stream_ref_if_not_destroyed(gpr_refcount *r) {
   return true;
 }
 
-/* How many bytes of incoming flow control would we like to advertise */
-uint32_t grpc_chttp2_target_incoming_window(grpc_chttp2_transport *t) {
-  return (uint32_t)GPR_MIN(
-      (int64_t)((1u << 31) - 1),
-      t->stream_total_over_incoming_window +
-          t->settings[GRPC_SENT_SETTINGS]
-                     [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
-}
-
 /* How many bytes would we like to put on the wire during a single syscall */
 static uint32_t target_write_size(grpc_chttp2_transport *t) {
   return 1024 * 1024;
@@ -201,7 +192,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
       &t->hpack_compressor,
       t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]);
 
-  if (t->outgoing_window > 0) {
+  if (t->flow_control.remote_window > 0) {
     while (grpc_chttp2_list_pop_stalled_by_transport(t, &s)) {
       if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s) &&
           stream_ref_if_not_destroyed(&s->refcount->refs)) {
@@ -227,10 +218,12 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
     bool sent_initial_metadata = s->sent_initial_metadata;
     bool now_writing = false;
 
-    GRPC_CHTTP2_IF_TRACING(gpr_log(
-        GPR_DEBUG, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t,
-        t->is_client ? "CLIENT" : "SERVER", s->id, sent_initial_metadata,
-        s->send_initial_metadata != NULL, s->announce_window));
+    GRPC_CHTTP2_IF_TRACING(
+        gpr_log(GPR_DEBUG, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t,
+                t->is_client ? "CLIENT" : "SERVER", s->id,
+                sent_initial_metadata, s->send_initial_metadata != NULL,
+                (int)(s->flow_control.local_window_delta -
+                      s->flow_control.announced_window_delta)));
 
     grpc_mdelem *extra_headers_for_trailing_metadata[2];
     size_t num_extra_headers_for_trailing_metadata = 0;
@@ -287,11 +280,12 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
       sent_initial_metadata = true;
     }
     /* send any window updates */
-    if (s->announce_window > 0) {
-      uint32_t announce = s->announce_window;
-      grpc_slice_buffer_add(&t->outbuf,
-                            grpc_chttp2_window_update_create(
-                                s->id, s->announce_window, &s->stats.outgoing));
+    uint32_t stream_announce = grpc_chttp2_flowctl_maybe_send_stream_update(
+        &t->flow_control, &s->flow_control);
+    if (stream_announce > 0) {
+      grpc_slice_buffer_add(
+          &t->outbuf, grpc_chttp2_window_update_create(s->id, stream_announce,
+                                                       &s->stats.outgoing));
       t->ping_state.pings_before_data_required =
           t->ping_policy.max_pings_without_data;
       if (!t->is_client) {
@@ -299,36 +293,75 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
             gpr_inf_past(GPR_CLOCK_MONOTONIC);
         t->ping_recv_state.ping_strikes = 0;
       }
-      GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, announce_window, announce);
     }
     if (sent_initial_metadata) {
       /* send any body bytes, if allowed by flow control */
-      if (s->flow_controlled_buffer.length > 0) {
-        uint32_t stream_outgoing_window = (uint32_t)GPR_MAX(
+      if (s->flow_controlled_buffer.length > 0 ||
+          (s->stream_compression_send_enabled &&
+           s->compressed_data_buffer->length > 0)) {
+        uint32_t stream_remote_window = (uint32_t)GPR_MAX(
             0,
-            s->outgoing_window_delta +
+            s->flow_control.remote_window_delta +
                 (int64_t)t->settings[GRPC_PEER_SETTINGS]
                                     [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
         uint32_t max_outgoing = (uint32_t)GPR_MIN(
             t->settings[GRPC_PEER_SETTINGS]
                        [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE],
-            GPR_MIN(stream_outgoing_window, t->outgoing_window));
+            GPR_MIN(stream_remote_window, t->flow_control.remote_window));
         if (max_outgoing > 0) {
-          uint32_t send_bytes =
-              (uint32_t)GPR_MIN(max_outgoing, s->flow_controlled_buffer.length);
-          bool is_last_data_frame =
-              s->fetching_send_message == NULL &&
-              send_bytes == s->flow_controlled_buffer.length;
-          bool is_last_frame =
-              is_last_data_frame && s->send_trailing_metadata != NULL &&
-              grpc_metadata_batch_is_empty(s->send_trailing_metadata);
-          grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, send_bytes,
-                                  is_last_frame, &s->stats.outgoing,
-                                  &t->outbuf);
-          GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, outgoing_window_delta,
-                                        send_bytes);
-          GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", t, outgoing_window,
-                                           send_bytes);
+          bool is_last_data_frame = false;
+          bool is_last_frame = false;
+          if (s->stream_compression_send_enabled) {
+            while ((s->flow_controlled_buffer.length > 0 ||
+                    s->compressed_data_buffer->length > 0) &&
+                   max_outgoing > 0) {
+              if (s->compressed_data_buffer->length > 0) {
+                uint32_t send_bytes = (uint32_t)GPR_MIN(
+                    max_outgoing, s->compressed_data_buffer->length);
+                is_last_data_frame =
+                    (send_bytes == s->compressed_data_buffer->length &&
+                     s->flow_controlled_buffer.length == 0 &&
+                     s->fetching_send_message == NULL);
+                is_last_frame =
+                    is_last_data_frame && s->send_trailing_metadata != NULL &&
+                    grpc_metadata_batch_is_empty(s->send_trailing_metadata);
+                grpc_chttp2_encode_data(s->id, s->compressed_data_buffer,
+                                        send_bytes, is_last_frame,
+                                        &s->stats.outgoing, &t->outbuf);
+                grpc_chttp2_flowctl_sent_data(&t->flow_control,
+                                              &s->flow_control, send_bytes);
+                max_outgoing -= send_bytes;
+                if (s->compressed_data_buffer->length == 0) {
+                  s->sending_bytes += s->uncompressed_data_size;
+                }
+              } else {
+                if (s->stream_compression_ctx == NULL) {
+                  s->stream_compression_ctx =
+                      grpc_stream_compression_context_create(
+                          GRPC_STREAM_COMPRESSION_COMPRESS);
+                }
+                s->uncompressed_data_size = s->flow_controlled_buffer.length;
+                GPR_ASSERT(grpc_stream_compress(
+                    s->stream_compression_ctx, &s->flow_controlled_buffer,
+                    s->compressed_data_buffer, NULL, MAX_SIZE_T,
+                    GRPC_STREAM_COMPRESSION_FLUSH_SYNC));
+              }
+            }
+          } else {
+            uint32_t send_bytes = (uint32_t)GPR_MIN(
+                max_outgoing, s->flow_controlled_buffer.length);
+            is_last_data_frame = s->fetching_send_message == NULL &&
+                                 send_bytes == s->flow_controlled_buffer.length;
+            is_last_frame =
+                is_last_data_frame && s->send_trailing_metadata != NULL &&
+                grpc_metadata_batch_is_empty(s->send_trailing_metadata);
+            grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer,
+                                    send_bytes, is_last_frame,
+                                    &s->stats.outgoing, &t->outbuf);
+            grpc_chttp2_flowctl_sent_data(&t->flow_control, &s->flow_control,
+                                          send_bytes);
+            s->sending_bytes += send_bytes;
+          }
           t->ping_state.pings_before_data_required =
               t->ping_policy.max_pings_without_data;
           if (!t->is_client) {
@@ -345,23 +378,26 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
                                                     &s->stats.outgoing));
             }
           }
-          s->sending_bytes += send_bytes;
           now_writing = true;
-          if (s->flow_controlled_buffer.length > 0) {
+          if (s->flow_controlled_buffer.length > 0 ||
+              (s->stream_compression_send_enabled &&
+               s->compressed_data_buffer->length > 0)) {
             GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:fork");
             grpc_chttp2_list_add_writable_stream(t, s);
           }
-        } else if (t->outgoing_window == 0) {
+        } else if (t->flow_control.remote_window == 0) {
           grpc_chttp2_list_add_stalled_by_transport(t, s);
           now_writing = true;
-        } else if (stream_outgoing_window == 0) {
+        } else if (stream_remote_window == 0) {
           grpc_chttp2_list_add_stalled_by_stream(t, s);
           now_writing = true;
         }
       }
       if (s->send_trailing_metadata != NULL &&
           s->fetching_send_message == NULL &&
-          s->flow_controlled_buffer.length == 0) {
+          s->flow_controlled_buffer.length == 0 &&
+          (!s->stream_compression_send_enabled ||
+           s->compressed_data_buffer->length == 0)) {
         GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata"));
         if (grpc_metadata_batch_is_empty(s->send_trailing_metadata)) {
           grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, 0, true,
@@ -406,22 +442,15 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
     }
   }
 
-  /* if the grpc_chttp2_transport is ready to send a window update, do so here
-     also; 3/4 is a magic number that will likely get tuned soon */
-  uint32_t target_incoming_window = grpc_chttp2_target_incoming_window(t);
-  uint32_t threshold_to_send_transport_window_update =
-      t->outbuf.count > 0 ? 3 * target_incoming_window / 4
-                          : target_incoming_window / 2;
-  if (t->incoming_window <= threshold_to_send_transport_window_update &&
-      t->incoming_window != target_incoming_window) {
+  uint32_t transport_announce =
+      grpc_chttp2_flowctl_maybe_send_transport_update(&t->flow_control);
+  if (transport_announce) {
     maybe_initiate_ping(exec_ctx, t,
                         GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE);
-    uint32_t announced = (uint32_t)GPR_CLAMP(
-        target_incoming_window - t->incoming_window, 0, UINT32_MAX);
-    GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("write", t, incoming_window, announced);
     grpc_transport_one_way_stats throwaway_stats;
-    grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_window_update_create(
-                                          0, announced, &throwaway_stats));
+    grpc_slice_buffer_add(
+        &t->outbuf, grpc_chttp2_window_update_create(0, transport_announce,
+                                                     &throwaway_stats));
     t->ping_state.pings_before_data_required =
         t->ping_policy.max_pings_without_data;
     if (!t->is_client) {
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c
index 29dfa885de..09420d92e7 100644
--- a/src/core/ext/transport/cronet/transport/cronet_transport.c
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.c
@@ -637,7 +637,8 @@ static void on_response_trailers_received(
  Utility function that takes the data from s->write_slice_buffer and assembles
  into a contiguous byte stream with 5 byte gRPC header prepended.
 */
-static void create_grpc_frame(grpc_slice_buffer *write_slice_buffer,
+static void create_grpc_frame(grpc_exec_ctx *exec_ctx,
+                              grpc_slice_buffer *write_slice_buffer,
                               char **pp_write_buffer,
                               size_t *p_write_buffer_size, uint32_t flags) {
   grpc_slice slice = grpc_slice_buffer_take_first(write_slice_buffer);
@@ -657,6 +658,7 @@ static void create_grpc_frame(grpc_slice_buffer *write_slice_buffer,
   *p++ = (uint8_t)(length);
   /* append actual data */
   memcpy(p, GRPC_SLICE_START_PTR(slice), length);
+  grpc_slice_unref_internal(exec_ctx, slice);
 }
 
 /*
@@ -968,6 +970,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
     s->header_array.capacity = s->header_array.count;
     CRONET_LOG(GPR_DEBUG, "bidirectional_stream_start(%p, %s)", s->cbs, url);
     bidirectional_stream_start(s->cbs, url, 0, method, &s->header_array, false);
+    if (url) {
+      gpr_free(url);
+    }
     unsigned int header_index;
     for (header_index = 0; header_index < s->header_array.count;
          header_index++) {
@@ -1014,14 +1019,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
       }
       if (write_slice_buffer.count > 0) {
         size_t write_buffer_size;
-        create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer,
-                          &write_buffer_size,
+        create_grpc_frame(exec_ctx, &write_slice_buffer,
+                          &stream_state->ws.write_buffer, &write_buffer_size,
                           stream_op->payload->send_message.send_message->flags);
         CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, %p)", s->cbs,
                    stream_state->ws.write_buffer);
         stream_state->state_callback_received[OP_SEND_MESSAGE] = false;
         bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer,
                                    (int)write_buffer_size, false);
+        grpc_slice_buffer_destroy_internal(exec_ctx, &write_slice_buffer);
         if (t->use_packet_coalescing) {
           if (!stream_op->send_trailing_metadata) {
             CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs);
@@ -1150,6 +1156,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
         } else {
           stream_state->rs.remaining_bytes = 0;
           CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response.");
+          /* Clean up read_slice_buffer in case there is unread data. */
+          grpc_slice_buffer_destroy_internal(
+              exec_ctx, &stream_state->rs.read_slice_buffer);
           grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer);
           grpc_slice_buffer_stream_init(&stream_state->rs.sbs,
                                         &stream_state->rs.read_slice_buffer, 0);
@@ -1203,6 +1212,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
       memcpy(dst_p, stream_state->rs.read_buffer,
              (size_t)stream_state->rs.length_field);
       null_and_maybe_free_read_buffer(s);
+      /* Clean up read_slice_buffer in case there is unread data. */
+      grpc_slice_buffer_destroy_internal(exec_ctx,
+                                         &stream_state->rs.read_slice_buffer);
       grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer);
       grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer,
                             read_data_slice);
@@ -1366,16 +1378,14 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                            grpc_closure *then_schedule_closure) {
   stream_obj *s = (stream_obj *)gs;
   null_and_maybe_free_read_buffer(s);
+  /* Clean up read_slice_buffer in case there is unread data. */
+  grpc_slice_buffer_destroy_internal(exec_ctx, &s->state.rs.read_slice_buffer);
   GRPC_ERROR_UNREF(s->state.cancel_error);
   GRPC_CLOSURE_SCHED(exec_ctx, then_schedule_closure, GRPC_ERROR_NONE);
 }
 
 static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {}
 
-static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
-  return NULL;
-}
-
 static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx,
                                    grpc_transport *gt) {
   return NULL;
@@ -1394,7 +1404,6 @@ static const grpc_transport_vtable grpc_cronet_vtable = {
     perform_op,
     destroy_stream,
     destroy_transport,
-    get_peer,
     get_endpoint};
 
 grpc_transport *grpc_create_cronet_transport(void *engine, const char *target,
diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c
index 4df64d81e2..b2d6f2d0c9 100644
--- a/src/core/ext/transport/inproc/inproc_transport.c
+++ b/src/core/ext/transport/inproc/inproc_transport.c
@@ -72,6 +72,7 @@ typedef struct sb_list_entry {
 typedef struct {
   grpc_byte_stream base;
   sb_list_entry *le;
+  grpc_error *shutdown_error;
 } inproc_slice_byte_stream;
 
 typedef struct {
@@ -190,32 +191,50 @@ typedef struct inproc_stream {
 static bool inproc_slice_byte_stream_next(grpc_exec_ctx *exec_ctx,
                                           grpc_byte_stream *bs, size_t max,
                                           grpc_closure *on_complete) {
-  inproc_slice_byte_stream *stream = (inproc_slice_byte_stream *)bs;
-  return (stream->le->sb.count != 0);
+  // Because inproc transport always provides the entire message atomically,
+  // the byte stream always has data available when this function is called.
+  // Thus, this function always returns true (unlike other transports) and
+  // there is never any need to schedule a closure
+  return true;
 }
 
 static grpc_error *inproc_slice_byte_stream_pull(grpc_exec_ctx *exec_ctx,
                                                  grpc_byte_stream *bs,
                                                  grpc_slice *slice) {
   inproc_slice_byte_stream *stream = (inproc_slice_byte_stream *)bs;
+  if (stream->shutdown_error != GRPC_ERROR_NONE) {
+    return GRPC_ERROR_REF(stream->shutdown_error);
+  }
   *slice = grpc_slice_buffer_take_first(&stream->le->sb);
   return GRPC_ERROR_NONE;
 }
 
+static void inproc_slice_byte_stream_shutdown(grpc_exec_ctx *exec_ctx,
+                                              grpc_byte_stream *bs,
+                                              grpc_error *error) {
+  inproc_slice_byte_stream *stream = (inproc_slice_byte_stream *)bs;
+  GRPC_ERROR_UNREF(stream->shutdown_error);
+  stream->shutdown_error = error;
+}
+
 static void inproc_slice_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
                                              grpc_byte_stream *bs) {
   inproc_slice_byte_stream *stream = (inproc_slice_byte_stream *)bs;
   sb_list_entry_destroy(exec_ctx, stream->le);
+  GRPC_ERROR_UNREF(stream->shutdown_error);
 }
 
+static const grpc_byte_stream_vtable inproc_slice_byte_stream_vtable = {
+    inproc_slice_byte_stream_next, inproc_slice_byte_stream_pull,
+    inproc_slice_byte_stream_shutdown, inproc_slice_byte_stream_destroy};
+
 void inproc_slice_byte_stream_init(inproc_slice_byte_stream *s,
                                    sb_list_entry *le) {
   s->base.length = (uint32_t)le->sb.length;
   s->base.flags = 0;
-  s->base.next = inproc_slice_byte_stream_next;
-  s->base.pull = inproc_slice_byte_stream_pull;
-  s->base.destroy = inproc_slice_byte_stream_destroy;
+  s->base.vtable = &inproc_slice_byte_stream_vtable;
   s->le = le;
+  s->shutdown_error = GRPC_ERROR_NONE;
 }
 
 static void ref_transport(inproc_transport *t) {
@@ -953,11 +972,18 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
         GPR_ASSERT(grpc_byte_stream_next(exec_ctx,
                                          op->payload->send_message.send_message,
                                          SIZE_MAX, &unused));
-        grpc_byte_stream_pull(exec_ctx, op->payload->send_message.send_message,
-                              &message_slice);
+        error = grpc_byte_stream_pull(
+            exec_ctx, op->payload->send_message.send_message, &message_slice);
+        if (error != GRPC_ERROR_NONE) {
+          cancel_stream_locked(exec_ctx, s, GRPC_ERROR_REF(error));
+          break;
+        }
+        GPR_ASSERT(error == GRPC_ERROR_NONE);
         remaining -= GRPC_SLICE_LENGTH(message_slice);
         grpc_slice_buffer_add(dest, message_slice);
       } while (remaining != 0);
+      grpc_byte_stream_destroy(exec_ctx,
+                               op->payload->send_message.send_message);
     }
     if (error == GRPC_ERROR_NONE && op->send_trailing_metadata) {
       grpc_metadata_batch *dest = (other == NULL) ? &s->write_buffer_trailing_md
@@ -1225,20 +1251,14 @@ static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   // Nothing to do here
 }
 
-static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
-  return gpr_strdup("inproc");
-}
-
 static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
   return NULL;
 }
 
 static const grpc_transport_vtable inproc_vtable = {
-    sizeof(inproc_stream), "inproc",
-    init_stream,           set_pollset,
-    set_pollset_set,       perform_stream_op,
-    perform_transport_op,  destroy_stream,
-    destroy_transport,     get_peer,
+    sizeof(inproc_stream), "inproc",        init_stream,
+    set_pollset,           set_pollset_set, perform_stream_op,
+    perform_transport_op,  destroy_stream,  destroy_transport,
     get_endpoint};
 
 /*******************************************************************************
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index 8fdef0bc64..02db798b5c 100644
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -221,6 +221,21 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
   return GRPC_COMPRESS_NONE;
 }
 
+grpc_stream_compression_algorithm
+grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a) {
+  size_t i;
+  if (a == NULL) return 0;
+  for (i = 0; i < a->num_args; ++i) {
+    if (a->args[i].type == GRPC_ARG_INTEGER &&
+        !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM,
+                a->args[i].key)) {
+      return (grpc_stream_compression_algorithm)a->args[i].value.integer;
+      break;
+    }
+  }
+  return GRPC_STREAM_COMPRESS_NONE;
+}
+
 grpc_channel_args *grpc_channel_args_set_compression_algorithm(
     grpc_channel_args *a, grpc_compression_algorithm algorithm) {
   GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT);
@@ -231,6 +246,16 @@ grpc_channel_args *grpc_channel_args_set_compression_algorithm(
   return grpc_channel_args_copy_and_add(a, &tmp, 1);
 }
 
+grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm(
+    grpc_channel_args *a, grpc_stream_compression_algorithm algorithm) {
+  GPR_ASSERT(algorithm < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT);
+  grpc_arg tmp;
+  tmp.type = GRPC_ARG_INTEGER;
+  tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM;
+  tmp.value.integer = algorithm;
+  return grpc_channel_args_copy_and_add(a, &tmp, 1);
+}
+
 /** Returns 1 if the argument for compression algorithm's enabled states bitset
  * was found in \a a, returning the arg's value in \a states. Otherwise, returns
  * 0. */
@@ -251,6 +276,26 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a,
   return 0; /* GPR_FALSE */
 }
 
+/** Returns 1 if the argument for compression algorithm's enabled states bitset
+ * was found in \a a, returning the arg's value in \a states. Otherwise, returns
+ * 0. */
+static int find_stream_compression_algorithm_states_bitset(
+    const grpc_channel_args *a, int **states_arg) {
+  if (a != NULL) {
+    size_t i;
+    for (i = 0; i < a->num_args; ++i) {
+      if (a->args[i].type == GRPC_ARG_INTEGER &&
+          !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET,
+                  a->args[i].key)) {
+        *states_arg = &a->args[i].value.integer;
+        **states_arg |= 0x1; /* forcefully enable support for no compression */
+        return 1;
+      }
+    }
+  }
+  return 0; /* GPR_FALSE */
+}
+
 grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_args **a,
     grpc_compression_algorithm algorithm, int state) {
@@ -292,6 +337,48 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
   return result;
 }
 
+grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state(
+    grpc_exec_ctx *exec_ctx, grpc_channel_args **a,
+    grpc_stream_compression_algorithm algorithm, int state) {
+  int *states_arg = NULL;
+  grpc_channel_args *result = *a;
+  const int states_arg_found =
+      find_stream_compression_algorithm_states_bitset(*a, &states_arg);
+
+  if (grpc_channel_args_get_stream_compression_algorithm(*a) == algorithm &&
+      state == 0) {
+    char *algo_name = NULL;
+    GPR_ASSERT(grpc_stream_compression_algorithm_name(algorithm, &algo_name) !=
+               0);
+    gpr_log(GPR_ERROR,
+            "Tried to disable default stream compression algorithm '%s'. The "
+            "operation has been ignored.",
+            algo_name);
+  } else if (states_arg_found) {
+    if (state != 0) {
+      GPR_BITSET((unsigned *)states_arg, algorithm);
+    } else if (algorithm != GRPC_STREAM_COMPRESS_NONE) {
+      GPR_BITCLEAR((unsigned *)states_arg, algorithm);
+    }
+  } else {
+    /* create a new arg */
+    grpc_arg tmp;
+    tmp.type = GRPC_ARG_INTEGER;
+    tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET;
+    /* all enabled by default */
+    tmp.value.integer = (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1;
+    if (state != 0) {
+      GPR_BITSET((unsigned *)&tmp.value.integer, algorithm);
+    } else if (algorithm != GRPC_STREAM_COMPRESS_NONE) {
+      GPR_BITCLEAR((unsigned *)&tmp.value.integer, algorithm);
+    }
+    result = grpc_channel_args_copy_and_add(*a, &tmp, 1);
+    grpc_channel_args_destroy(exec_ctx, *a);
+    *a = result;
+  }
+  return result;
+}
+
 uint32_t grpc_channel_args_compression_algorithm_get_states(
     const grpc_channel_args *a) {
   int *states_arg;
@@ -302,6 +389,17 @@ uint32_t grpc_channel_args_compression_algorithm_get_states(
   }
 }
 
+uint32_t grpc_channel_args_stream_compression_algorithm_get_states(
+    const grpc_channel_args *a) {
+  int *states_arg;
+  if (find_stream_compression_algorithm_states_bitset(a, &states_arg)) {
+    return (uint32_t)*states_arg;
+  } else {
+    return (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) -
+           1; /* All algs. enabled */
+  }
+}
+
 grpc_channel_args *grpc_channel_args_set_socket_mutator(
     grpc_channel_args *a, grpc_socket_mutator *mutator) {
   grpc_arg tmp = grpc_socket_mutator_to_arg(mutator);
diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h
index f649a8d9ec..0599e189c3 100644
--- a/src/core/lib/channel/channel_args.h
+++ b/src/core/lib/channel/channel_args.h
@@ -59,12 +59,24 @@ void grpc_channel_args_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_args *a);
 grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
     const grpc_channel_args *a);
 
+/** Returns the stream compression algorithm set in \a a. */
+grpc_stream_compression_algorithm
+grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a);
+
 /** Returns a channel arg instance with compression enabled. If \a a is
  * non-NULL, its args are copied. N.B. GRPC_COMPRESS_NONE disables compression
  * for the channel. */
 grpc_channel_args *grpc_channel_args_set_compression_algorithm(
     grpc_channel_args *a, grpc_compression_algorithm algorithm);
 
+/** Returns a channel arg instance with stream compression enabled. If \a a is
+ * non-NULL, its args are copied. N.B. GRPC_STREAM_COMPRESS_NONE disables
+ * stream compression for the channel. If a value other than
+ * GRPC_STREAM_COMPRESS_NONE is set, it takes precedence over message-wise
+ * compression algorithms. */
+grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm(
+    grpc_channel_args *a, grpc_stream_compression_algorithm algorithm);
+
 /** Sets the support for the given compression algorithm. By default, all
  * compression algorithms are enabled. It's an error to disable an algorithm set
  * by grpc_channel_args_set_compression_algorithm.
@@ -76,6 +88,17 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_args **a,
     grpc_compression_algorithm algorithm, int enabled);
 
+/** Sets the support for the given stream compression algorithm. By default, all
+ * stream compression algorithms are enabled. It's an error to disable an
+ * algorithm set by grpc_channel_args_set_stream_compression_algorithm.
+ *
+ * Returns an instance with the updated algorithm states. The \a a pointer is
+ * modified to point to the returned instance (which may be different from the
+ * input value of \a a). */
+grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state(
+    grpc_exec_ctx *exec_ctx, grpc_channel_args **a,
+    grpc_stream_compression_algorithm algorithm, int enabled);
+
 /** Returns the bitset representing the support state (true for enabled, false
  * for disabled) for compression algorithms.
  *
@@ -84,6 +107,14 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
 uint32_t grpc_channel_args_compression_algorithm_get_states(
     const grpc_channel_args *a);
 
+/** Returns the bitset representing the support state (true for enabled, false
+ * for disabled) for stream compression algorithms.
+ *
+ * The i-th bit of the returned bitset corresponds to the i-th entry in the
+ * grpc_stream_compression_algorithm enum. */
+uint32_t grpc_channel_args_stream_compression_algorithm_get_states(
+    const grpc_channel_args *a);
+
 int grpc_channel_args_compare(const grpc_channel_args *a,
                               const grpc_channel_args *b);
 
diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c
index 0f8e33c4be..775c8bc667 100644
--- a/src/core/lib/channel/channel_stack.c
+++ b/src/core/lib/channel/channel_stack.c
@@ -233,15 +233,10 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
 void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                        grpc_transport_stream_op_batch *op) {
   grpc_call_element *next_elem = elem + 1;
+  GRPC_CALL_LOG_OP(GPR_INFO, next_elem, op);
   next_elem->filter->start_transport_stream_op_batch(exec_ctx, next_elem, op);
 }
 
-char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx,
-                              grpc_call_element *elem) {
-  grpc_call_element *next_elem = elem + 1;
-  return next_elem->filter->get_peer(exec_ctx, next_elem);
-}
-
 void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx,
                                 grpc_channel_element *elem,
                                 const grpc_channel_info *channel_info) {
@@ -265,12 +260,3 @@ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem) {
   return (grpc_call_stack *)((char *)(elem)-ROUND_UP_TO_ALIGNMENT_SIZE(
       sizeof(grpc_call_stack)));
 }
-
-void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *elem,
-                                    grpc_error *error) {
-  grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op(NULL);
-  op->cancel_stream = true;
-  op->payload->cancel_stream.cancel_error = error;
-  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op);
-}
diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index a80f8aa826..ae1cac31f7 100644
--- a/src/core/lib/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -40,6 +40,7 @@
 #include <grpc/support/time.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/support/arena.h"
 #include "src/core/lib/transport/transport.h"
@@ -71,6 +72,7 @@ typedef struct {
   gpr_timespec start_time;
   gpr_timespec deadline;
   gpr_arena *arena;
+  grpc_call_combiner *call_combiner;
 } grpc_call_element_args;
 
 typedef struct {
@@ -150,9 +152,6 @@ typedef struct {
   void (*destroy_channel_elem)(grpc_exec_ctx *exec_ctx,
                                grpc_channel_element *elem);
 
-  /* Implement grpc_call_get_peer() */
-  char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem);
-
   /* Implement grpc_channel_get_info() */
   void (*get_channel_info)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
                            const grpc_channel_info *channel_info);
@@ -271,8 +270,6 @@ void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
    stack */
 void grpc_channel_next_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
                           grpc_transport_op *op);
-/* Pass through a request to get_peer to the next child element */
-char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem);
 /* Pass through a request to get_channel_info() to the next child element */
 void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx,
                                 grpc_channel_element *elem,
@@ -288,10 +285,6 @@ void grpc_call_log_op(char *file, int line, gpr_log_severity severity,
                       grpc_call_element *elem,
                       grpc_transport_stream_op_batch *op);
 
-void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *cur_elem,
-                                    grpc_error *error);
-
 extern grpc_tracer_flag grpc_trace_channel;
 
 #define GRPC_CALL_LOG_OP(sev, elem, op) \
diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c
index af06ca802e..8285226fc4 100644
--- a/src/core/lib/channel/connected_channel.c
+++ b/src/core/lib/channel/connected_channel.c
@@ -36,7 +36,57 @@ typedef struct connected_channel_channel_data {
   grpc_transport *transport;
 } channel_data;
 
-typedef struct connected_channel_call_data { void *unused; } call_data;
+typedef struct {
+  grpc_closure closure;
+  grpc_closure *original_closure;
+  grpc_call_combiner *call_combiner;
+  const char *reason;
+} callback_state;
+
+typedef struct connected_channel_call_data {
+  grpc_call_combiner *call_combiner;
+  // Closures used for returning results on the call combiner.
+  callback_state on_complete[6];  // Max number of pending batches.
+  callback_state recv_initial_metadata_ready;
+  callback_state recv_message_ready;
+} call_data;
+
+static void run_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg,
+                                 grpc_error *error) {
+  callback_state *state = (callback_state *)arg;
+  GRPC_CALL_COMBINER_START(exec_ctx, state->call_combiner,
+                           state->original_closure, GRPC_ERROR_REF(error),
+                           state->reason);
+}
+
+static void run_cancel_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  run_in_call_combiner(exec_ctx, arg, error);
+  gpr_free(arg);
+}
+
+static void intercept_callback(call_data *calld, callback_state *state,
+                               bool free_when_done, const char *reason,
+                               grpc_closure **original_closure) {
+  state->original_closure = *original_closure;
+  state->call_combiner = calld->call_combiner;
+  state->reason = reason;
+  *original_closure = GRPC_CLOSURE_INIT(
+      &state->closure,
+      free_when_done ? run_cancel_in_call_combiner : run_in_call_combiner,
+      state, grpc_schedule_on_exec_ctx);
+}
+
+static callback_state *get_state_for_batch(
+    call_data *calld, grpc_transport_stream_op_batch *batch) {
+  if (batch->send_initial_metadata) return &calld->on_complete[0];
+  if (batch->send_message) return &calld->on_complete[1];
+  if (batch->send_trailing_metadata) return &calld->on_complete[2];
+  if (batch->recv_initial_metadata) return &calld->on_complete[3];
+  if (batch->recv_message) return &calld->on_complete[4];
+  if (batch->recv_trailing_metadata) return &calld->on_complete[5];
+  GPR_UNREACHABLE_CODE(return NULL);
+}
 
 /* We perform a small hack to locate transport data alongside the connected
    channel data in call allocations, to allow everything to be pulled in minimal
@@ -49,13 +99,38 @@ typedef struct connected_channel_call_data { void *unused; } call_data;
    into transport stream operations */
 static void con_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-    grpc_transport_stream_op_batch *op) {
+    grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-
+  if (batch->recv_initial_metadata) {
+    callback_state *state = &calld->recv_initial_metadata_ready;
+    intercept_callback(
+        calld, state, false, "recv_initial_metadata_ready",
+        &batch->payload->recv_initial_metadata.recv_initial_metadata_ready);
+  }
+  if (batch->recv_message) {
+    callback_state *state = &calld->recv_message_ready;
+    intercept_callback(calld, state, false, "recv_message_ready",
+                       &batch->payload->recv_message.recv_message_ready);
+  }
+  if (batch->cancel_stream) {
+    // There can be more than one cancellation batch in flight at any
+    // given time, so we can't just pick out a fixed index into
+    // calld->on_complete like we can for the other ops.  However,
+    // cancellation isn't in the fast path, so we just allocate a new
+    // closure for each one.
+    callback_state *state = (callback_state *)gpr_malloc(sizeof(*state));
+    intercept_callback(calld, state, true, "on_complete (cancel_stream)",
+                       &batch->on_complete);
+  } else {
+    callback_state *state = get_state_for_batch(calld, batch);
+    intercept_callback(calld, state, false, "on_complete", &batch->on_complete);
+  }
   grpc_transport_perform_stream_op(exec_ctx, chand->transport,
-                                   TRANSPORT_STREAM_FROM_CALL_DATA(calld), op);
+                                   TRANSPORT_STREAM_FROM_CALL_DATA(calld),
+                                   batch);
+  GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                          "passed batch to transport");
 }
 
 static void con_start_transport_op(grpc_exec_ctx *exec_ctx,
@@ -71,6 +146,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
+  calld->call_combiner = args->call_combiner;
   int r = grpc_transport_init_stream(
       exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld),
       &args->call_stack->refcount, args->server_transport_data, args->arena);
@@ -118,11 +194,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  channel_data *chand = elem->channel_data;
-  return grpc_transport_get_peer(exec_ctx, chand->transport);
-}
-
 /* No-op. */
 static void con_get_channel_info(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem,
@@ -138,7 +209,6 @@ const grpc_channel_filter grpc_connected_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    con_get_peer,
     con_get_channel_info,
     "connected",
 };
diff --git a/src/core/lib/compression/algorithm_metadata.h b/src/core/lib/compression/algorithm_metadata.h
index 4717af6e2b..08feafc1bb 100644
--- a/src/core/lib/compression/algorithm_metadata.h
+++ b/src/core/lib/compression/algorithm_metadata.h
@@ -26,13 +26,27 @@
 grpc_slice grpc_compression_algorithm_slice(
     grpc_compression_algorithm algorithm);
 
+/** Return stream compression algorithm based metadata value */
+grpc_slice grpc_stream_compression_algorithm_slice(
+    grpc_stream_compression_algorithm algorithm);
+
 /** Return compression algorithm based metadata element (grpc-encoding: xxx) */
 grpc_mdelem grpc_compression_encoding_mdelem(
     grpc_compression_algorithm algorithm);
 
+/** Return stream compression algorithm based metadata element
+ * (content-encoding: xxx) */
+grpc_mdelem grpc_stream_compression_encoding_mdelem(
+    grpc_stream_compression_algorithm algorithm);
+
 /** Find compression algorithm based on passed in mdstr - returns
  * GRPC_COMPRESS_ALGORITHM_COUNT on failure */
 grpc_compression_algorithm grpc_compression_algorithm_from_slice(
     grpc_slice str);
 
+/** Find stream compression algorithm based on passed in mdstr - returns
+ * GRPC_STREAM_COMPRESS_ALGORITHM_COUNT on failure */
+grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice(
+    grpc_slice str);
+
 #endif /* GRPC_CORE_LIB_COMPRESSION_ALGORITHM_METADATA_H */
diff --git a/src/core/lib/compression/compression.c b/src/core/lib/compression/compression.c
index 8deae2798f..ec84c01811 100644
--- a/src/core/lib/compression/compression.c
+++ b/src/core/lib/compression/compression.c
@@ -46,6 +46,19 @@ int grpc_compression_algorithm_parse(grpc_slice name,
   }
 }
 
+int grpc_stream_compression_algorithm_parse(
+    grpc_slice name, grpc_stream_compression_algorithm *algorithm) {
+  if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) {
+    *algorithm = GRPC_STREAM_COMPRESS_NONE;
+    return 1;
+  } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) {
+    *algorithm = GRPC_STREAM_COMPRESS_GZIP;
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
 int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm,
                                     char **name) {
   GRPC_API_TRACE("grpc_compression_algorithm_parse(algorithm=%d, name=%p)", 2,
@@ -66,6 +79,24 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm,
   return 0;
 }
 
+int grpc_stream_compression_algorithm_name(
+    grpc_stream_compression_algorithm algorithm, char **name) {
+  GRPC_API_TRACE(
+      "grpc_stream_compression_algorithm_parse(algorithm=%d, name=%p)", 2,
+      ((int)algorithm, name));
+  switch (algorithm) {
+    case GRPC_STREAM_COMPRESS_NONE:
+      *name = "identity";
+      return 1;
+    case GRPC_STREAM_COMPRESS_GZIP:
+      *name = "gzip";
+      return 1;
+    case GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT:
+      return 0;
+  }
+  return 0;
+}
+
 grpc_compression_algorithm grpc_compression_algorithm_from_slice(
     grpc_slice str) {
   if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_COMPRESS_NONE;
@@ -74,6 +105,13 @@ grpc_compression_algorithm grpc_compression_algorithm_from_slice(
   return GRPC_COMPRESS_ALGORITHMS_COUNT;
 }
 
+grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice(
+    grpc_slice str) {
+  if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_STREAM_COMPRESS_NONE;
+  if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_STREAM_COMPRESS_GZIP;
+  return GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT;
+}
+
 grpc_slice grpc_compression_algorithm_slice(
     grpc_compression_algorithm algorithm) {
   switch (algorithm) {
@@ -89,6 +127,19 @@ grpc_slice grpc_compression_algorithm_slice(
   return grpc_empty_slice();
 }
 
+grpc_slice grpc_stream_compression_algorithm_slice(
+    grpc_stream_compression_algorithm algorithm) {
+  switch (algorithm) {
+    case GRPC_STREAM_COMPRESS_NONE:
+      return GRPC_MDSTR_IDENTITY;
+    case GRPC_STREAM_COMPRESS_GZIP:
+      return GRPC_MDSTR_GZIP;
+    case GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT:
+      return grpc_empty_slice();
+  }
+  return grpc_empty_slice();
+}
+
 grpc_mdelem grpc_compression_encoding_mdelem(
     grpc_compression_algorithm algorithm) {
   switch (algorithm) {
@@ -104,10 +155,25 @@ grpc_mdelem grpc_compression_encoding_mdelem(
   return GRPC_MDNULL;
 }
 
+grpc_mdelem grpc_stream_compression_encoding_mdelem(
+    grpc_stream_compression_algorithm algorithm) {
+  switch (algorithm) {
+    case GRPC_STREAM_COMPRESS_NONE:
+      return GRPC_MDELEM_CONTENT_ENCODING_IDENTITY;
+    case GRPC_STREAM_COMPRESS_GZIP:
+      return GRPC_MDELEM_CONTENT_ENCODING_GZIP;
+    default:
+      break;
+  }
+  return GRPC_MDNULL;
+}
+
 void grpc_compression_options_init(grpc_compression_options *opts) {
   memset(opts, 0, sizeof(*opts));
   /* all enabled by default */
   opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
+  opts->enabled_stream_compression_algorithms_bitset =
+      (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1;
 }
 
 void grpc_compression_options_enable_algorithm(
@@ -126,6 +192,13 @@ int grpc_compression_options_is_algorithm_enabled(
   return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
 }
 
+int grpc_compression_options_is_stream_compression_algorithm_enabled(
+    const grpc_compression_options *opts,
+    grpc_stream_compression_algorithm algorithm) {
+  return GPR_BITGET(opts->enabled_stream_compression_algorithms_bitset,
+                    algorithm);
+}
+
 /* TODO(dgq): Add the ability to specify parameters to the individual
  * compression algorithms */
 grpc_compression_algorithm grpc_compression_algorithm_for_level(
@@ -181,3 +254,30 @@ grpc_compression_algorithm grpc_compression_algorithm_for_level(
       abort();
   };
 }
+
+GRPCAPI grpc_stream_compression_algorithm
+grpc_stream_compression_algorithm_for_level(
+    grpc_stream_compression_level level, uint32_t accepted_stream_encodings) {
+  GRPC_API_TRACE("grpc_stream_compression_algorithm_for_level(level=%d)", 1,
+                 ((int)level));
+  if (level > GRPC_STREAM_COMPRESS_LEVEL_HIGH) {
+    gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level);
+    abort();
+  }
+
+  switch (level) {
+    case GRPC_STREAM_COMPRESS_LEVEL_NONE:
+      return GRPC_STREAM_COMPRESS_NONE;
+    case GRPC_STREAM_COMPRESS_LEVEL_LOW:
+    case GRPC_STREAM_COMPRESS_LEVEL_MED:
+    case GRPC_STREAM_COMPRESS_LEVEL_HIGH:
+      if (GPR_BITGET(accepted_stream_encodings, GRPC_STREAM_COMPRESS_GZIP) ==
+          1) {
+        return GRPC_STREAM_COMPRESS_GZIP;
+      } else {
+        return GRPC_STREAM_COMPRESS_NONE;
+      }
+    default:
+      abort();
+  }
+}
diff --git a/src/core/lib/debug/stats.c b/src/core/lib/debug/stats.c
index 6daf03484a..4dbd94c724 100644
--- a/src/core/lib/debug/stats.c
+++ b/src/core/lib/debug/stats.c
@@ -27,7 +27,7 @@
 
 #include "src/core/lib/support/string.h"
 
-grpc_stats_data *grpc_stats_per_cpu_storage;
+grpc_stats_data *grpc_stats_per_cpu_storage = NULL;
 static size_t g_num_cores;
 
 void grpc_stats_init(void) {
diff --git a/src/core/lib/iomgr/call_combiner.c b/src/core/lib/iomgr/call_combiner.c
new file mode 100644
index 0000000000..899f98552d
--- /dev/null
+++ b/src/core/lib/iomgr/call_combiner.c
@@ -0,0 +1,180 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/call_combiner.h"
+
+#include <grpc/support/log.h>
+
+grpc_tracer_flag grpc_call_combiner_trace =
+    GRPC_TRACER_INITIALIZER(false, "call_combiner");
+
+static grpc_error* decode_cancel_state_error(gpr_atm cancel_state) {
+  if (cancel_state & 1) {
+    return (grpc_error*)(cancel_state & ~(gpr_atm)1);
+  }
+  return GRPC_ERROR_NONE;
+}
+
+static gpr_atm encode_cancel_state_error(grpc_error* error) {
+  return (gpr_atm)1 | (gpr_atm)error;
+}
+
+void grpc_call_combiner_init(grpc_call_combiner* call_combiner) {
+  gpr_mpscq_init(&call_combiner->queue);
+}
+
+void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) {
+  gpr_mpscq_destroy(&call_combiner->queue);
+  GRPC_ERROR_UNREF(decode_cancel_state_error(call_combiner->cancel_state));
+}
+
+#ifndef NDEBUG
+#define DEBUG_ARGS , const char *file, int line
+#define DEBUG_FMT_STR "%s:%d: "
+#define DEBUG_FMT_ARGS , file, line
+#else
+#define DEBUG_ARGS
+#define DEBUG_FMT_STR
+#define DEBUG_FMT_ARGS
+#endif
+
+void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
+                              grpc_call_combiner* call_combiner,
+                              grpc_closure* closure,
+                              grpc_error* error DEBUG_ARGS,
+                              const char* reason) {
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG,
+            "==> grpc_call_combiner_start() [%p] closure=%p [" DEBUG_FMT_STR
+            "%s] error=%s",
+            call_combiner, closure DEBUG_FMT_ARGS, reason,
+            grpc_error_string(error));
+  }
+  size_t prev_size =
+      (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)1);
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
+            prev_size + 1);
+  }
+  if (prev_size == 0) {
+    if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      gpr_log(GPR_DEBUG, "  EXECUTING IMMEDIATELY");
+    }
+    // Queue was empty, so execute this closure immediately.
+    GRPC_CLOSURE_SCHED(exec_ctx, closure, error);
+  } else {
+    if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      gpr_log(GPR_INFO, "  QUEUING");
+    }
+    // Queue was not empty, so add closure to queue.
+    closure->error_data.error = error;
+    gpr_mpscq_push(&call_combiner->queue, (gpr_mpscq_node*)closure);
+  }
+}
+
+void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
+                             grpc_call_combiner* call_combiner DEBUG_ARGS,
+                             const char* reason) {
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG,
+            "==> grpc_call_combiner_stop() [%p] [" DEBUG_FMT_STR "%s]",
+            call_combiner DEBUG_FMT_ARGS, reason);
+  }
+  size_t prev_size =
+      (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)-1);
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
+            prev_size - 1);
+  }
+  GPR_ASSERT(prev_size >= 1);
+  if (prev_size > 1) {
+    while (true) {
+      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        gpr_log(GPR_DEBUG, "  checking queue");
+      }
+      bool empty;
+      grpc_closure* closure = (grpc_closure*)gpr_mpscq_pop_and_check_end(
+          &call_combiner->queue, &empty);
+      if (closure == NULL) {
+        // This can happen either due to a race condition within the mpscq
+        // code or because of a race with grpc_call_combiner_start().
+        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          gpr_log(GPR_DEBUG, "  queue returned no result; checking again");
+        }
+        continue;
+      }
+      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        gpr_log(GPR_DEBUG, "  EXECUTING FROM QUEUE: closure=%p error=%s",
+                closure, grpc_error_string(closure->error_data.error));
+      }
+      GRPC_CLOSURE_SCHED(exec_ctx, closure, closure->error_data.error);
+      break;
+    }
+  } else if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG, "  queue empty");
+  }
+}
+
+void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
+                                             grpc_call_combiner* call_combiner,
+                                             grpc_closure* closure) {
+  while (true) {
+    // Decode original state.
+    gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
+    grpc_error* original_error = decode_cancel_state_error(original_state);
+    // If error is set, invoke the cancellation closure immediately.
+    // Otherwise, store the new closure.
+    if (original_error != GRPC_ERROR_NONE) {
+      GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_REF(original_error));
+      break;
+    } else {
+      if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
+                           (gpr_atm)closure)) {
+        break;
+      }
+    }
+    // cas failed, try again.
+  }
+}
+
+void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
+                               grpc_call_combiner* call_combiner,
+                               grpc_error* error) {
+  while (true) {
+    gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
+    grpc_error* original_error = decode_cancel_state_error(original_state);
+    if (original_error != GRPC_ERROR_NONE) {
+      GRPC_ERROR_UNREF(error);
+      break;
+    }
+    if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
+                         encode_cancel_state_error(error))) {
+      if (original_state != 0) {
+        grpc_closure* notify_on_cancel = (grpc_closure*)original_state;
+        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          gpr_log(GPR_DEBUG,
+                  "call_combiner=%p: scheduling notify_on_cancel callback=%p",
+                  call_combiner, notify_on_cancel);
+        }
+        GRPC_CLOSURE_SCHED(exec_ctx, notify_on_cancel, GRPC_ERROR_REF(error));
+      }
+      break;
+    }
+    // cas failed, try again.
+  }
+}
diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h
new file mode 100644
index 0000000000..621e2c3669
--- /dev/null
+++ b/src/core/lib/iomgr/call_combiner.h
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H
+#define GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H
+
+#include <stddef.h>
+
+#include <grpc/support/atm.h>
+
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/support/mpscq.h"
+
+// A simple, lock-free mechanism for serializing activity related to a
+// single call.  This is similar to a combiner but is more lightweight.
+//
+// It requires the callback (or, in the common case where the callback
+// actually kicks off a chain of callbacks, the last callback in that
+// chain) to explicitly indicate (by calling GRPC_CALL_COMBINER_STOP())
+// when it is done with the action that was kicked off by the original
+// callback.
+
+extern grpc_tracer_flag grpc_call_combiner_trace;
+
+typedef struct {
+  gpr_atm size;  // size_t, num closures in queue or currently executing
+  gpr_mpscq queue;
+  // Either 0 (if not cancelled and no cancellation closure set),
+  // a grpc_closure* (if the lowest bit is 0),
+  // or a grpc_error* (if the lowest bit is 1).
+  gpr_atm cancel_state;
+} grpc_call_combiner;
+
+// Assumes memory was initialized to zero.
+void grpc_call_combiner_init(grpc_call_combiner* call_combiner);
+
+void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner);
+
+#ifndef NDEBUG
+#define GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, closure, error,   \
+                                 reason)                                    \
+  grpc_call_combiner_start((exec_ctx), (call_combiner), (closure), (error), \
+                           __FILE__, __LINE__, (reason))
+#define GRPC_CALL_COMBINER_STOP(exec_ctx, call_combiner, reason)           \
+  grpc_call_combiner_stop((exec_ctx), (call_combiner), __FILE__, __LINE__, \
+                          (reason))
+/// Starts processing \a closure on \a call_combiner.
+void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
+                              grpc_call_combiner* call_combiner,
+                              grpc_closure* closure, grpc_error* error,
+                              const char* file, int line, const char* reason);
+/// Yields the call combiner to the next closure in the queue, if any.
+void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
+                             grpc_call_combiner* call_combiner,
+                             const char* file, int line, const char* reason);
+#else
+#define GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, closure, error,   \
+                                 reason)                                    \
+  grpc_call_combiner_start((exec_ctx), (call_combiner), (closure), (error), \
+                           (reason))
+#define GRPC_CALL_COMBINER_STOP(exec_ctx, call_combiner, reason) \
+  grpc_call_combiner_stop((exec_ctx), (call_combiner), (reason))
+/// Starts processing \a closure on \a call_combiner.
+void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
+                              grpc_call_combiner* call_combiner,
+                              grpc_closure* closure, grpc_error* error,
+                              const char* reason);
+/// Yields the call combiner to the next closure in the queue, if any.
+void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
+                             grpc_call_combiner* call_combiner,
+                             const char* reason);
+#endif
+
+/// Tells \a call_combiner to invoke \a closure when
+/// grpc_call_combiner_cancel() is called.  If grpc_call_combiner_cancel()
+/// was previously called, \a closure will be invoked immediately.
+/// If \a closure is NULL, then no closure will be invoked on
+/// cancellation; this effectively unregisters the previously set closure.
+void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
+                                             grpc_call_combiner* call_combiner,
+                                             grpc_closure* closure);
+
+/// Indicates that the call has been cancelled.
+void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
+                               grpc_call_combiner* call_combiner,
+                               grpc_error* error);
+
+#endif /* GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H */
diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c
index c72c37e2b5..9b66987b68 100644
--- a/src/core/lib/iomgr/combiner.c
+++ b/src/core/lib/iomgr/combiner.c
@@ -73,10 +73,8 @@ static const grpc_closure_scheduler_vtable finally_scheduler = {
 static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 
 grpc_combiner *grpc_combiner_create(void) {
-  grpc_combiner *lock = gpr_malloc(sizeof(*lock));
+  grpc_combiner *lock = gpr_zalloc(sizeof(*lock));
   gpr_ref_init(&lock->refs, 1);
-  lock->next_combiner_on_this_exec_ctx = NULL;
-  lock->time_to_execute_final_list = false;
   lock->scheduler.vtable = &scheduler;
   lock->finally_scheduler.vtable = &finally_scheduler;
   gpr_atm_no_barrier_store(&lock->state, STATE_UNORPHANED);
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c
index 532d2887ed..47ad2f45c3 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.c
+++ b/src/core/lib/iomgr/ev_epoll1_linux.c
@@ -46,9 +46,63 @@
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/string.h"
 
 static grpc_wakeup_fd global_wakeup_fd;
-static int g_epfd;
+
+/*******************************************************************************
+ * Singleton epoll set related fields
+ */
+
+#define MAX_EPOLL_EVENTS 100
+#define MAX_EPOLL_EVENTS_HANDLED_PER_ITERATION 1
+
+/* NOTE ON SYNCHRONIZATION:
+ * - Fields in this struct are only modified by the designated poller. Hence
+ *   there is no need for any locks to protect the struct.
+ * - num_events and cursor fields have to be of atomic type to provide memory
+ *   visibility guarantees only. i.e In case of multiple pollers, the designated
+ *   polling thread keeps changing; the thread that wrote these values may be
+ *   different from the thread reading the values
+ */
+typedef struct epoll_set {
+  int epfd;
+
+  /* The epoll_events after the last call to epoll_wait() */
+  struct epoll_event events[MAX_EPOLL_EVENTS];
+
+  /* The number of epoll_events after the last call to epoll_wait() */
+  gpr_atm num_events;
+
+  /* Index of the first event in epoll_events that has to be processed. This
+   * field is only valid if num_events > 0 */
+  gpr_atm cursor;
+} epoll_set;
+
+/* The global singleton epoll set */
+static epoll_set g_epoll_set;
+
+/* Must be called *only* once */
+static bool epoll_set_init() {
+  g_epoll_set.epfd = epoll_create1(EPOLL_CLOEXEC);
+  if (g_epoll_set.epfd < 0) {
+    gpr_log(GPR_ERROR, "epoll unavailable");
+    return false;
+  }
+
+  gpr_log(GPR_INFO, "grpc epoll fd: %d", g_epoll_set.epfd);
+  gpr_atm_no_barrier_store(&g_epoll_set.num_events, 0);
+  gpr_atm_no_barrier_store(&g_epoll_set.cursor, 0);
+  return true;
+}
+
+/* epoll_set_init() MUST be called before calling this. */
+static void epoll_set_shutdown() {
+  if (g_epoll_set.epfd >= 0) {
+    close(g_epoll_set.epfd);
+    g_epoll_set.epfd = -1;
+  }
+}
 
 /*******************************************************************************
  * Fd Declarations
@@ -78,8 +132,21 @@ static void fd_global_shutdown(void);
 
 typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state;
 
+static const char *kick_state_string(kick_state st) {
+  switch (st) {
+    case UNKICKED:
+      return "UNKICKED";
+    case KICKED:
+      return "KICKED";
+    case DESIGNATED_POLLER:
+      return "DESIGNATED_POLLER";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
 struct grpc_pollset_worker {
   kick_state kick_state;
+  int kick_state_mutator;  // which line of code last changed kick state
   bool initialized_cv;
   grpc_pollset_worker *next;
   grpc_pollset_worker *prev;
@@ -87,6 +154,12 @@ struct grpc_pollset_worker {
   grpc_closure_list schedule_on_end_work;
 };
 
+#define SET_KICK_STATE(worker, state)        \
+  do {                                       \
+    (worker)->kick_state = (state);          \
+    (worker)->kick_state_mutator = __LINE__; \
+  } while (false)
+
 #define MAX_NEIGHBOURHOODS 1024
 
 typedef struct pollset_neighbourhood {
@@ -101,10 +174,15 @@ struct grpc_pollset {
   bool reassigning_neighbourhood;
   grpc_pollset_worker *root_worker;
   bool kicked_without_poller;
+
+  /* Set to true if the pollset is observed to have no workers available to
+     poll */
   bool seen_inactive;
-  bool shutting_down;          /* Is the pollset shutting down ? */
-  bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */
+  bool shutting_down;             /* Is the pollset shutting down ? */
   grpc_closure *shutdown_closure; /* Called after after shutdown is complete */
+
+  /* Number of workers who are *about-to* attach themselves to the pollset
+   * worker list */
   int begin_refs;
 
   grpc_pollset *next;
@@ -204,7 +282,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
 
   struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET),
                            .data.ptr = new_fd};
-  if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) {
+  if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, fd, &ev) != 0) {
     gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno));
   }
 
@@ -213,30 +291,43 @@ static grpc_fd *fd_create(int fd, const char *name) {
 
 static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; }
 
-/* Might be called multiple times */
-static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
+/* if 'releasing_fd' is true, it means that we are going to detach the internal
+ * fd from grpc_fd structure (i.e which means we should not be calling
+ * shutdown() syscall on that fd) */
+static void fd_shutdown_internal(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                                 grpc_error *why, bool releasing_fd) {
   if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure,
                              GRPC_ERROR_REF(why))) {
-    shutdown(fd->fd, SHUT_RDWR);
+    if (!releasing_fd) {
+      shutdown(fd->fd, SHUT_RDWR);
+    }
     grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why));
   }
   GRPC_ERROR_UNREF(why);
 }
 
+/* Might be called multiple times */
+static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
+  fd_shutdown_internal(exec_ctx, fd, why, false);
+}
+
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
+  bool is_release_fd = (release_fd != NULL);
 
   if (!grpc_lfev_is_shutdown(&fd->read_closure)) {
-    fd_shutdown(exec_ctx, fd, GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason));
+    fd_shutdown_internal(exec_ctx, fd,
+                         GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason),
+                         is_release_fd);
   }
 
   /* If release_fd is not NULL, we should be relinquishing control of the file
      descriptor fd->fd (but we still own the grpc_fd structure). */
-  if (release_fd != NULL) {
+  if (is_release_fd) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!already_closed) {
     close(fd->fd);
   }
 
@@ -264,29 +355,23 @@ static bool fd_is_shutdown(grpc_fd *fd) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
-
-  /* Note, it is possible that fd_become_readable might be called twice with
-     different 'notifier's when an fd becomes readable and it is in two epoll
-     sets (This can happen briefly during polling island merges). In such cases
-     it does not really matter which notifer is set as the read_notifier_pollset
-     (They would both point to the same polling island anyway) */
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
   /* Use release store to match with acquire load in fd_get_read_notifier */
   gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier);
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 /*******************************************************************************
@@ -295,7 +380,10 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
 
 GPR_TLS_DECL(g_current_thread_pollset);
 GPR_TLS_DECL(g_current_thread_worker);
+
+/* The designated poller */
 static gpr_atm g_active_poller;
+
 static pollset_neighbourhood *g_neighbourhoods;
 static size_t g_num_neighbourhoods;
 
@@ -349,7 +437,8 @@ static grpc_error *pollset_global_init(void) {
   if (err != GRPC_ERROR_NONE) return err;
   struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET),
                            .data.ptr = &global_wakeup_fd};
-  if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) {
+  if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd,
+                &ev) != 0) {
     return GRPC_OS_ERROR(errno, "epoll_ctl");
   }
   g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS);
@@ -375,7 +464,14 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->mu);
   *mu = &pollset->mu;
   pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()];
+  pollset->reassigning_neighbourhood = false;
+  pollset->root_worker = NULL;
+  pollset->kicked_without_poller = false;
   pollset->seen_inactive = true;
+  pollset->shutting_down = false;
+  pollset->shutdown_closure = NULL;
+  pollset->begin_refs = 0;
+  pollset->next = pollset->prev = NULL;
 }
 
 static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
@@ -407,22 +503,33 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
 }
 
 static grpc_error *pollset_kick_all(grpc_pollset *pollset) {
+  GPR_TIMER_BEGIN("pollset_kick_all", 0);
   grpc_error *error = GRPC_ERROR_NONE;
   if (pollset->root_worker != NULL) {
     grpc_pollset_worker *worker = pollset->root_worker;
     do {
-      if (worker->initialized_cv) {
-        worker->kick_state = KICKED;
-        gpr_cv_signal(&worker->cv);
-      } else {
-        worker->kick_state = KICKED;
-        append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd),
-                     "pollset_shutdown");
+      switch (worker->kick_state) {
+        case KICKED:
+          break;
+        case UNKICKED:
+          SET_KICK_STATE(worker, KICKED);
+          if (worker->initialized_cv) {
+            gpr_cv_signal(&worker->cv);
+          }
+          break;
+        case DESIGNATED_POLLER:
+          SET_KICK_STATE(worker, KICKED);
+          append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd),
+                       "pollset_kick_all");
+          break;
       }
 
       worker = worker->next;
     } while (worker != pollset->root_worker);
   }
+  // TODO: sreek.  Check if we need to set 'kicked_without_poller' to true here
+  // in the else case
+  GPR_TIMER_END("pollset_kick_all", 0);
   return error;
 }
 
@@ -430,6 +537,7 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
                                           grpc_pollset *pollset) {
   if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL &&
       pollset->begin_refs == 0) {
+    GPR_TIMER_MARK("pollset_finish_shutdown", 0);
     GRPC_CLOSURE_SCHED(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE);
     pollset->shutdown_closure = NULL;
   }
@@ -437,14 +545,16 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
 
 static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                              grpc_closure *closure) {
+  GPR_TIMER_BEGIN("pollset_shutdown", 0);
   GPR_ASSERT(pollset->shutdown_closure == NULL);
+  GPR_ASSERT(!pollset->shutting_down);
   pollset->shutdown_closure = closure;
+  pollset->shutting_down = true;
   GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset));
   pollset_maybe_finish_shutdown(exec_ctx, pollset);
+  GPR_TIMER_END("pollset_shutdown", 0);
 }
 
-#define MAX_EPOLL_EVENTS 100
-
 static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
                                            gpr_timespec now) {
   gpr_timespec timeout;
@@ -463,20 +573,72 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
   return millis >= 1 ? millis : 1;
 }
 
-static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+/* Process the epoll events found by do_epoll_wait() function.
+   - g_epoll_set.cursor points to the index of the first event to be processed
+   - This function then processes up-to MAX_EPOLL_EVENTS_PER_ITERATION and
+     updates the g_epoll_set.cursor
+
+   NOTE ON SYNCRHONIZATION: Similar to do_epoll_wait(), this function is only
+   called by g_active_poller thread. So there is no need for synchronization
+   when accessing fields in g_epoll_set */
+static grpc_error *process_epoll_events(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset *pollset) {
+  static const char *err_desc = "process_events";
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  GPR_TIMER_BEGIN("process_epoll_events", 0);
+  long num_events = gpr_atm_acq_load(&g_epoll_set.num_events);
+  long cursor = gpr_atm_acq_load(&g_epoll_set.cursor);
+  for (int idx = 0;
+       (idx < MAX_EPOLL_EVENTS_HANDLED_PER_ITERATION) && cursor != num_events;
+       idx++) {
+    long c = cursor++;
+    struct epoll_event *ev = &g_epoll_set.events[c];
+    void *data_ptr = ev->data.ptr;
+
+    if (data_ptr == &global_wakeup_fd) {
+      append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd),
+                   err_desc);
+    } else {
+      grpc_fd *fd = (grpc_fd *)(data_ptr);
+      bool cancel = (ev->events & (EPOLLERR | EPOLLHUP)) != 0;
+      bool read_ev = (ev->events & (EPOLLIN | EPOLLPRI)) != 0;
+      bool write_ev = (ev->events & EPOLLOUT) != 0;
+
+      if (read_ev || cancel) {
+        fd_become_readable(exec_ctx, fd, pollset);
+      }
+
+      if (write_ev || cancel) {
+        fd_become_writable(exec_ctx, fd);
+      }
+    }
+  }
+  gpr_atm_rel_store(&g_epoll_set.cursor, cursor);
+  GPR_TIMER_END("process_epoll_events", 0);
+  return error;
+}
+
+/* Do epoll_wait and store the events in g_epoll_set.events field. This does not
+   "process" any of the events yet; that is done in process_epoll_events().
+   *See process_epoll_events() function for more details.
+
+   NOTE ON SYNCHRONIZATION: At any point of time, only the g_active_poller
+   (i.e the designated poller thread) will be calling this function. So there is
+   no need for any synchronization when accesing fields in g_epoll_set */
+static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
                                  gpr_timespec now, gpr_timespec deadline) {
-  struct epoll_event events[MAX_EPOLL_EVENTS];
-  static const char *err_desc = "pollset_poll";
+  GPR_TIMER_BEGIN("do_epoll_wait", 0);
 
+  int r;
   int timeout = poll_deadline_to_millis_timeout(deadline, now);
-
   if (timeout != 0) {
     GRPC_SCHEDULING_START_BLOCKING_REGION;
   }
-  int r;
   do {
     GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
-    r = epoll_wait(g_epfd, events, MAX_EPOLL_EVENTS, timeout);
+    r = epoll_wait(g_epoll_set.epfd, g_epoll_set.events, MAX_EPOLL_EVENTS,
+                   timeout);
   } while (r < 0 && errno == EINTR);
   if (timeout != 0) {
     GRPC_SCHEDULING_END_BLOCKING_REGION;
@@ -484,38 +646,31 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
   if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait");
 
-  grpc_error *error = GRPC_ERROR_NONE;
-  for (int i = 0; i < r; i++) {
-    void *data_ptr = events[i].data.ptr;
-    if (data_ptr == &global_wakeup_fd) {
-      append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd),
-                   err_desc);
-    } else {
-      grpc_fd *fd = (grpc_fd *)(data_ptr);
-      bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0;
-      bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0;
-      bool write_ev = (events[i].events & EPOLLOUT) != 0;
-      if (read_ev || cancel) {
-        fd_become_readable(exec_ctx, fd, pollset);
-      }
-      if (write_ev || cancel) {
-        fd_become_writable(exec_ctx, fd);
-      }
-    }
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_DEBUG, "ps: %p poll got %d events", ps, r);
   }
 
-  return error;
+  gpr_atm_rel_store(&g_epoll_set.num_events, r);
+  gpr_atm_rel_store(&g_epoll_set.cursor, 0);
+
+  GPR_TIMER_END("do_epoll_wait", 0);
+  return GRPC_ERROR_NONE;
 }
 
 static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
                          grpc_pollset_worker **worker_hdl, gpr_timespec *now,
                          gpr_timespec deadline) {
+  GPR_TIMER_BEGIN("begin_worker", 0);
   if (worker_hdl != NULL) *worker_hdl = worker;
   worker->initialized_cv = false;
-  worker->kick_state = UNKICKED;
+  SET_KICK_STATE(worker, UNKICKED);
   worker->schedule_on_end_work = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
   pollset->begin_refs++;
 
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_ERROR, "PS:%p BEGIN_STARTS:%p", pollset, worker);
+  }
+
   if (pollset->seen_inactive) {
     // pollset has been observed to be inactive, we need to move back to the
     // active list
@@ -531,6 +686,11 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
   retry_lock_neighbourhood:
     gpr_mu_lock(&neighbourhood->mu);
     gpr_mu_lock(&pollset->mu);
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d",
+              pollset, worker, kick_state_string(worker->kick_state),
+              is_reassigning);
+    }
     if (pollset->seen_inactive) {
       if (neighbourhood != pollset->neighbourhood) {
         gpr_mu_unlock(&neighbourhood->mu);
@@ -541,8 +701,14 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
       pollset->seen_inactive = false;
       if (neighbourhood->active_root == NULL) {
         neighbourhood->active_root = pollset->next = pollset->prev = pollset;
-        if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) {
-          worker->kick_state = DESIGNATED_POLLER;
+        /* TODO: sreek. Why would this worker state be other than UNKICKED
+         * here ? (since the worker isn't added to the pollset yet, there is no
+         * way it can be "found" by other threads to get kicked). */
+
+        /* If there is no designated poller, make this the designated poller */
+        if (worker->kick_state == UNKICKED &&
+            gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) {
+          SET_KICK_STATE(worker, DESIGNATED_POLLER);
         }
       } else {
         pollset->next = neighbourhood->active_root;
@@ -556,28 +722,60 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
     }
     gpr_mu_unlock(&neighbourhood->mu);
   }
+
   worker_insert(pollset, worker);
   pollset->begin_refs--;
-  if (worker->kick_state == UNKICKED) {
+  if (worker->kick_state == UNKICKED && !pollset->kicked_without_poller) {
     GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker);
     worker->initialized_cv = true;
     gpr_cv_init(&worker->cv);
-    while (worker->kick_state == UNKICKED &&
-           pollset->shutdown_closure == NULL) {
+    while (worker->kick_state == UNKICKED && !pollset->shutting_down) {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_ERROR, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d",
+                pollset, worker, kick_state_string(worker->kick_state),
+                pollset->shutting_down);
+      }
+
       if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) &&
           worker->kick_state == UNKICKED) {
-        worker->kick_state = KICKED;
+        /* If gpr_cv_wait returns true (i.e a timeout), pretend that the worker
+           received a kick */
+        SET_KICK_STATE(worker, KICKED);
       }
     }
     *now = gpr_now(now->clock_type);
   }
 
-  return worker->kick_state == DESIGNATED_POLLER &&
-         pollset->shutdown_closure == NULL;
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_ERROR,
+            "PS:%p BEGIN_DONE:%p kick_state=%s shutdown=%d "
+            "kicked_without_poller: %d",
+            pollset, worker, kick_state_string(worker->kick_state),
+            pollset->shutting_down, pollset->kicked_without_poller);
+  }
+
+  /* We release pollset lock in this function at a couple of places:
+   *   1. Briefly when assigning pollset to a neighbourhood
+   *   2. When doing gpr_cv_wait()
+   * It is possible that 'kicked_without_poller' was set to true during (1) and
+   * 'shutting_down' is set to true during (1) or (2). If either of them is
+   * true, this worker cannot do polling */
+  /* TODO(sreek): Perhaps there is a better way to handle kicked_without_poller
+   * case; especially when the worker is the DESIGNATED_POLLER */
+
+  if (pollset->kicked_without_poller) {
+    pollset->kicked_without_poller = false;
+    GPR_TIMER_END("begin_worker", 0);
+    return false;
+  }
+
+  GPR_TIMER_END("begin_worker", 0);
+  return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down;
 }
 
 static bool check_neighbourhood_for_available_poller(
     pollset_neighbourhood *neighbourhood) {
+  GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0);
   bool found_worker = false;
   do {
     grpc_pollset *inspect = neighbourhood->active_root;
@@ -593,10 +791,19 @@ static bool check_neighbourhood_for_available_poller(
           case UNKICKED:
             if (gpr_atm_no_barrier_cas(&g_active_poller, 0,
                                        (gpr_atm)inspect_worker)) {
-              inspect_worker->kick_state = DESIGNATED_POLLER;
+              if (GRPC_TRACER_ON(grpc_polling_trace)) {
+                gpr_log(GPR_DEBUG, " .. choose next poller to be %p",
+                        inspect_worker);
+              }
+              SET_KICK_STATE(inspect_worker, DESIGNATED_POLLER);
               if (inspect_worker->initialized_cv) {
+                GPR_TIMER_MARK("signal worker", 0);
                 gpr_cv_signal(&inspect_worker->cv);
               }
+            } else {
+              if (GRPC_TRACER_ON(grpc_polling_trace)) {
+                gpr_log(GPR_DEBUG, " .. beaten to choose next poller");
+              }
             }
             // even if we didn't win the cas, there's a worker, we can stop
             found_worker = true;
@@ -609,9 +816,12 @@ static bool check_neighbourhood_for_available_poller(
             break;
         }
         inspect_worker = inspect_worker->next;
-      } while (inspect_worker != inspect->root_worker);
+      } while (!found_worker && inspect_worker != inspect->root_worker);
     }
     if (!found_worker) {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_DEBUG, " .. mark pollset %p inactive", inspect);
+      }
       inspect->seen_inactive = true;
       if (inspect == neighbourhood->active_root) {
         neighbourhood->active_root =
@@ -623,21 +833,30 @@ static bool check_neighbourhood_for_available_poller(
     }
     gpr_mu_unlock(&inspect->mu);
   } while (!found_worker);
+  GPR_TIMER_END("check_neighbourhood_for_available_poller", 0);
   return found_worker;
 }
 
 static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                        grpc_pollset_worker *worker,
                        grpc_pollset_worker **worker_hdl) {
+  GPR_TIMER_BEGIN("end_worker", 0);
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_DEBUG, "PS:%p END_WORKER:%p", pollset, worker);
+  }
   if (worker_hdl != NULL) *worker_hdl = NULL;
-  worker->kick_state = KICKED;
+  /* Make sure we appear kicked */
+  SET_KICK_STATE(worker, KICKED);
   grpc_closure_list_move(&worker->schedule_on_end_work,
                          &exec_ctx->closure_list);
   if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) {
     if (worker->next != worker && worker->next->kick_state == UNKICKED) {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_DEBUG, " .. choose next poller to be peer %p", worker);
+      }
       GPR_ASSERT(worker->next->initialized_cv);
       gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next);
-      worker->next->kick_state = DESIGNATED_POLLER;
+      SET_KICK_STATE(worker->next, DESIGNATED_POLLER);
       gpr_cv_signal(&worker->next->cv);
       if (grpc_exec_ctx_has_work(exec_ctx)) {
         gpr_mu_unlock(&pollset->mu);
@@ -646,9 +865,9 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
     } else {
       gpr_atm_no_barrier_store(&g_active_poller, 0);
-      gpr_mu_unlock(&pollset->mu);
       size_t poller_neighbourhood_idx =
           (size_t)(pollset->neighbourhood - g_neighbourhoods);
+      gpr_mu_unlock(&pollset->mu);
       bool found_worker = false;
       bool scan_state[MAX_NEIGHBOURHOODS];
       for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) {
@@ -684,86 +903,216 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (worker->initialized_cv) {
     gpr_cv_destroy(&worker->cv);
   }
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_DEBUG, " .. remove worker");
+  }
   if (EMPTIED == worker_remove(pollset, worker)) {
     pollset_maybe_finish_shutdown(exec_ctx, pollset);
   }
   GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker);
+  GPR_TIMER_END("end_worker", 0);
 }
 
 /* pollset->po.mu lock must be held by the caller before calling this.
    The function pollset_work() may temporarily release the lock (pollset->po.mu)
    during the course of its execution but it will always re-acquire the lock and
    ensure that it is held by the time the function returns */
-static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
                                 grpc_pollset_worker **worker_hdl,
                                 gpr_timespec now, gpr_timespec deadline) {
   grpc_pollset_worker worker;
   grpc_error *error = GRPC_ERROR_NONE;
   static const char *err_desc = "pollset_work";
-  if (pollset->kicked_without_poller) {
-    pollset->kicked_without_poller = false;
+  GPR_TIMER_BEGIN("pollset_work", 0);
+  if (ps->kicked_without_poller) {
+    ps->kicked_without_poller = false;
+    GPR_TIMER_END("pollset_work", 0);
     return GRPC_ERROR_NONE;
   }
-  gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
-  if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) {
+
+  if (begin_worker(ps, &worker, worker_hdl, &now, deadline)) {
+    gpr_tls_set(&g_current_thread_pollset, (intptr_t)ps);
     gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
-    GPR_ASSERT(!pollset->shutdown_closure);
-    GPR_ASSERT(!pollset->seen_inactive);
-    gpr_mu_unlock(&pollset->mu);
-    append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline),
-                 err_desc);
-    gpr_mu_lock(&pollset->mu);
+    GPR_ASSERT(!ps->shutting_down);
+    GPR_ASSERT(!ps->seen_inactive);
+
+    gpr_mu_unlock(&ps->mu); /* unlock */
+    /* This is the designated polling thread at this point and should ideally do
+       polling. However, if there are unprocessed events left from a previous
+       call to do_epoll_wait(), skip calling epoll_wait() in this iteration and
+       process the pending epoll events.
+
+       The reason for decoupling do_epoll_wait and process_epoll_events is to
+       better distrubute the work (i.e handling epoll events) across multiple
+       threads
+
+       process_epoll_events() returns very quickly: It just queues the work on
+       exec_ctx but does not execute it (the actual exectution or more
+       accurately grpc_exec_ctx_flush() happens in end_worker() AFTER selecting
+       a designated poller). So we are not waiting long periods without a
+       designated poller */
+    if (gpr_atm_acq_load(&g_epoll_set.cursor) ==
+        gpr_atm_acq_load(&g_epoll_set.num_events)) {
+      append_error(&error, do_epoll_wait(exec_ctx, ps, now, deadline),
+                   err_desc);
+    }
+    append_error(&error, process_epoll_events(exec_ctx, ps), err_desc);
+
+    gpr_mu_lock(&ps->mu); /* lock */
+
     gpr_tls_set(&g_current_thread_worker, 0);
+  } else {
+    gpr_tls_set(&g_current_thread_pollset, (intptr_t)ps);
   }
-  end_worker(exec_ctx, pollset, &worker, worker_hdl);
+  end_worker(exec_ctx, ps, &worker, worker_hdl);
+
   gpr_tls_set(&g_current_thread_pollset, 0);
+  GPR_TIMER_END("pollset_work", 0);
   return error;
 }
 
 static grpc_error *pollset_kick(grpc_pollset *pollset,
                                 grpc_pollset_worker *specific_worker) {
+  GPR_TIMER_BEGIN("pollset_kick", 0);
+  grpc_error *ret_err = GRPC_ERROR_NONE;
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_strvec log;
+    gpr_strvec_init(&log);
+    char *tmp;
+    gpr_asprintf(
+        &tmp, "PS:%p KICK:%p curps=%p curworker=%p root=%p", pollset,
+        specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset),
+        (void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker);
+    gpr_strvec_add(&log, tmp);
+    if (pollset->root_worker != NULL) {
+      gpr_asprintf(&tmp, " {kick_state=%s next=%p {kick_state=%s}}",
+                   kick_state_string(pollset->root_worker->kick_state),
+                   pollset->root_worker->next,
+                   kick_state_string(pollset->root_worker->next->kick_state));
+      gpr_strvec_add(&log, tmp);
+    }
+    if (specific_worker != NULL) {
+      gpr_asprintf(&tmp, " worker_kick_state=%s",
+                   kick_state_string(specific_worker->kick_state));
+      gpr_strvec_add(&log, tmp);
+    }
+    tmp = gpr_strvec_flatten(&log, NULL);
+    gpr_strvec_destroy(&log);
+    gpr_log(GPR_ERROR, "%s", tmp);
+    gpr_free(tmp);
+  }
   if (specific_worker == NULL) {
     if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) {
       grpc_pollset_worker *root_worker = pollset->root_worker;
       if (root_worker == NULL) {
         pollset->kicked_without_poller = true;
-        return GRPC_ERROR_NONE;
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. kicked_without_poller");
+        }
+        goto done;
       }
       grpc_pollset_worker *next_worker = root_worker->next;
-      if (root_worker == next_worker &&
-          root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(
-                             &g_active_poller)) {
-        root_worker->kick_state = KICKED;
-        return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+      if (root_worker->kick_state == KICKED) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. already kicked %p", root_worker);
+        }
+        SET_KICK_STATE(root_worker, KICKED);
+        goto done;
+      } else if (next_worker->kick_state == KICKED) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. already kicked %p", next_worker);
+        }
+        SET_KICK_STATE(next_worker, KICKED);
+        goto done;
+      } else if (root_worker ==
+                     next_worker &&  // only try and wake up a poller if
+                                     // there is no next worker
+                 root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(
+                                    &g_active_poller)) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. kicked %p", root_worker);
+        }
+        SET_KICK_STATE(root_worker, KICKED);
+        ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+        goto done;
       } else if (next_worker->kick_state == UNKICKED) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. kicked %p", next_worker);
+        }
         GPR_ASSERT(next_worker->initialized_cv);
-        next_worker->kick_state = KICKED;
+        SET_KICK_STATE(next_worker, KICKED);
         gpr_cv_signal(&next_worker->cv);
-        return GRPC_ERROR_NONE;
+        goto done;
+      } else if (next_worker->kick_state == DESIGNATED_POLLER) {
+        if (root_worker->kick_state != DESIGNATED_POLLER) {
+          if (GRPC_TRACER_ON(grpc_polling_trace)) {
+            gpr_log(
+                GPR_ERROR,
+                " .. kicked root non-poller %p (initialized_cv=%d) (poller=%p)",
+                root_worker, root_worker->initialized_cv, next_worker);
+          }
+          SET_KICK_STATE(root_worker, KICKED);
+          if (root_worker->initialized_cv) {
+            gpr_cv_signal(&root_worker->cv);
+          }
+          goto done;
+        } else {
+          if (GRPC_TRACER_ON(grpc_polling_trace)) {
+            gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker,
+                    root_worker);
+          }
+          SET_KICK_STATE(next_worker, KICKED);
+          ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+          goto done;
+        }
       } else {
-        return GRPC_ERROR_NONE;
+        GPR_ASSERT(next_worker->kick_state == KICKED);
+        SET_KICK_STATE(next_worker, KICKED);
+        goto done;
       }
     } else {
-      return GRPC_ERROR_NONE;
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_ERROR, " .. kicked while waking up");
+      }
+      goto done;
     }
   } else if (specific_worker->kick_state == KICKED) {
-    return GRPC_ERROR_NONE;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. specific worker already kicked");
+    }
+    goto done;
   } else if (gpr_tls_get(&g_current_thread_worker) ==
              (intptr_t)specific_worker) {
-    specific_worker->kick_state = KICKED;
-    return GRPC_ERROR_NONE;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker);
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
+    goto done;
   } else if (specific_worker ==
              (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) {
-    specific_worker->kick_state = KICKED;
-    return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. kick active poller");
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
+    ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+    goto done;
   } else if (specific_worker->initialized_cv) {
-    specific_worker->kick_state = KICKED;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. kick waiting worker");
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
     gpr_cv_signal(&specific_worker->cv);
-    return GRPC_ERROR_NONE;
+    goto done;
   } else {
-    specific_worker->kick_state = KICKED;
-    return GRPC_ERROR_NONE;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. kick non-waiting worker");
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
+    goto done;
   }
+done:
+  GPR_TIMER_END("pollset_kick", 0);
+  return ret_err;
 }
 
 static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -807,6 +1156,7 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
 static void shutdown_engine(void) {
   fd_global_shutdown();
   pollset_global_shutdown();
+  epoll_set_shutdown();
 }
 
 static const grpc_event_engine_vtable vtable = {
@@ -841,26 +1191,26 @@ static const grpc_event_engine_vtable vtable = {
 };
 
 /* It is possible that GLIBC has epoll but the underlying kernel doesn't.
- * Create a dummy epoll_fd to make sure epoll support is available */
+ * Create epoll_fd (epoll_set_init() takes care of that) to make sure epoll
+ * support is available */
 const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) {
-  /* TODO(ctiller): temporary, until this stabilizes */
-  if (!explicit_request) return NULL;
+  if (!explicit_request) {
+    return NULL;
+  }
 
   if (!grpc_has_wakeup_fd()) {
     return NULL;
   }
 
-  g_epfd = epoll_create1(EPOLL_CLOEXEC);
-  if (g_epfd < 0) {
-    gpr_log(GPR_ERROR, "epoll unavailable");
+  if (!epoll_set_init()) {
     return NULL;
   }
 
   fd_global_init();
 
   if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) {
-    close(g_epfd);
     fd_global_shutdown();
+    epoll_set_shutdown();
     return NULL;
   }
 
diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
index 5f293a74eb..e2e3cd9003 100644
--- a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
@@ -932,7 +932,7 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
@@ -953,8 +953,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
        before doing this.) */
   if (fd->po.pi != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, false /* is_fd_closed */,
-                                    &error);
+    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
@@ -1008,12 +1007,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -1224,7 +1223,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 
   /* Note, it is possible that fd_become_readable might be called twice with
      different 'notifier's when an fd becomes readable and it is in two epoll
@@ -1236,7 +1235,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx,
@@ -1288,7 +1287,8 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
 }
 
 /* NOTE: This function may modify 'now' */
-static bool acquire_polling_lease(grpc_pollset_worker *worker,
+static bool acquire_polling_lease(grpc_exec_ctx *exec_ctx,
+                                  grpc_pollset_worker *worker,
                                   polling_island *pi, gpr_timespec deadline,
                                   gpr_timespec *now) {
   bool is_lease_acquired = false;
@@ -1381,7 +1381,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd,
                                    sigset_t *sig_mask, grpc_error **error) {
   /* Only g_max_pollers_per_pi threads can be doing polling in parallel.
      If we cannot get a lease, we cannot continue to do epoll_pwait() */
-  if (!acquire_polling_lease(worker, pi, deadline, &now)) {
+  if (!acquire_polling_lease(exec_ctx, worker, pi, deadline, &now)) {
     return;
   }
 
diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
index a307c4e709..ddb8c74d2d 100644
--- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
@@ -494,8 +494,8 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
+  bool is_fd_closed = already_closed;
   grpc_error *error = GRPC_ERROR_NONE;
   epoll_set *unref_eps = NULL;
 
@@ -506,7 +506,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!is_fd_closed) {
     close(fd->fd);
     is_fd_closed = true;
   }
@@ -561,12 +561,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -697,11 +697,11 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
 }
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c
index 3560e5de30..0e42f76af3 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.c
+++ b/src/core/lib/iomgr/ev_epollex_linux.c
@@ -50,7 +50,7 @@
 #include "src/core/lib/support/spinlock.h"
 
 /*******************************************************************************
- * Pollset-set sibling link
+ * Polling object
  */
 
 typedef enum {
@@ -381,8 +381,8 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
+  bool is_fd_closed = already_closed;
   grpc_error *error = GRPC_ERROR_NONE;
 
   gpr_mu_lock(&fd->pollable.po.mu);
@@ -393,7 +393,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!is_fd_closed) {
     close(fd->fd);
     is_fd_closed = true;
   }
@@ -439,12 +439,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -711,7 +711,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 
   /* Note, it is possible that fd_become_readable might be called twice with
      different 'notifier's when an fd becomes readable and it is in two epoll
@@ -723,7 +723,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static grpc_error *fd_become_pollable_locked(grpc_fd *fd) {
@@ -1051,8 +1051,8 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx,
     /* Introduce a spurious completion.
        If we do not, then it may be that the fd-specific epoll set consumed
        a completion without being polled, leading to a missed edge going up. */
-    grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure);
-    grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure);
+    grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read");
+    grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write");
     pollset_kick_all(exec_ctx, pollset);
     pollset->current_pollable = &pollset->pollable;
     if (append_error(&error, pollable_materialize(&pollset->pollable),
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c
index d278d34092..16f2d21237 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.c
+++ b/src/core/lib/iomgr/ev_epollsig_linux.c
@@ -855,7 +855,7 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
@@ -876,8 +876,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
        before doing this.) */
   if (fd->po.pi != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, false /* is_fd_closed */,
-                                    &error);
+    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
@@ -934,12 +933,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -1116,7 +1115,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 
   /* Note, it is possible that fd_become_readable might be called twice with
      different 'notifier's when an fd becomes readable and it is in two epoll
@@ -1128,7 +1127,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index e1d758ec00..fbd265f3ce 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -43,6 +43,7 @@
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/murmur_hash.h"
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
 
@@ -240,22 +241,43 @@ struct grpc_pollset_set {
  * condition variable polling definitions
  */
 
+#define POLLCV_THREAD_GRACE_MS 1000
 #define CV_POLL_PERIOD_MS 1000
 #define CV_DEFAULT_TABLE_SIZE 16
 
-typedef enum poll_status_t { INPROGRESS, COMPLETED, CANCELLED } poll_status_t;
-
-typedef struct poll_args {
+typedef struct poll_result {
   gpr_refcount refcount;
-  gpr_cv *cv;
+  cv_node *watchers;
+  int watchcount;
   struct pollfd *fds;
   nfds_t nfds;
-  int timeout;
   int retval;
   int err;
-  gpr_atm status;
+  int completed;
+} poll_result;
+
+typedef struct poll_args {
+  gpr_cv trigger;
+  int trigger_set;
+  struct pollfd *fds;
+  nfds_t nfds;
+  poll_result *result;
+  struct poll_args *next;
+  struct poll_args *prev;
 } poll_args;
 
+// This is a 2-tiered cache, we mantain a hash table
+// of active poll calls, so we can wait on the result
+// of that call.  We also maintain a freelist of inactive
+// poll threads.
+typedef struct poll_hash_table {
+  poll_args *free_pollers;
+  poll_args **active_pollers;
+  unsigned int size;
+  unsigned int count;
+} poll_hash_table;
+
+poll_hash_table poll_cache;
 cv_fd_table g_cvfds;
 
 /*******************************************************************************
@@ -399,11 +421,14 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   fd->on_done_closure = on_done;
   fd->released = release_fd != NULL;
-  if (fd->released) {
+  if (release_fd != NULL) {
     *release_fd = fd->fd;
+    fd->released = true;
+  } else if (already_closed) {
+    fd->released = true;
   }
   gpr_mu_lock(&fd->mu);
   REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
@@ -1276,43 +1301,205 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
  * Condition Variable polling extensions
  */
 
-static void decref_poll_args(poll_args *args) {
-  if (gpr_unref(&args->refcount)) {
-    gpr_free(args->fds);
-    gpr_cv_destroy(args->cv);
-    gpr_free(args->cv);
-    gpr_free(args);
+static void run_poll(void *args);
+static void cache_poller_locked(poll_args *args);
+
+static void cache_insert_locked(poll_args *args) {
+  uint32_t key = gpr_murmur_hash3(args->fds, args->nfds * sizeof(struct pollfd),
+                                  0xDEADBEEF);
+  key = key % poll_cache.size;
+  if (poll_cache.active_pollers[key]) {
+    poll_cache.active_pollers[key]->prev = args;
   }
+  args->next = poll_cache.active_pollers[key];
+  args->prev = NULL;
+  poll_cache.active_pollers[key] = args;
+  poll_cache.count++;
 }
 
-// Poll in a background thread
-static void run_poll(void *arg) {
-  int timeout, retval;
-  poll_args *pargs = (poll_args *)arg;
-  while (gpr_atm_no_barrier_load(&pargs->status) == INPROGRESS) {
-    if (pargs->timeout < 0) {
-      timeout = CV_POLL_PERIOD_MS;
-    } else {
-      timeout = GPR_MIN(CV_POLL_PERIOD_MS, pargs->timeout);
-      pargs->timeout -= timeout;
+static void init_result(poll_args *pargs) {
+  pargs->result = gpr_malloc(sizeof(poll_result));
+  gpr_ref_init(&pargs->result->refcount, 1);
+  pargs->result->watchers = NULL;
+  pargs->result->watchcount = 0;
+  pargs->result->fds = gpr_malloc(sizeof(struct pollfd) * pargs->nfds);
+  memcpy(pargs->result->fds, pargs->fds, sizeof(struct pollfd) * pargs->nfds);
+  pargs->result->nfds = pargs->nfds;
+  pargs->result->retval = 0;
+  pargs->result->err = 0;
+  pargs->result->completed = 0;
+}
+
+// Creates a poll_args object for a given arguments to poll().
+// This object may return a poll_args in the cache.
+static poll_args *get_poller_locked(struct pollfd *fds, nfds_t count) {
+  uint32_t key =
+      gpr_murmur_hash3(fds, count * sizeof(struct pollfd), 0xDEADBEEF);
+  key = key % poll_cache.size;
+  poll_args *curr = poll_cache.active_pollers[key];
+  while (curr) {
+    if (curr->nfds == count &&
+        memcmp(curr->fds, fds, count * sizeof(struct pollfd)) == 0) {
+      gpr_free(fds);
+      return curr;
     }
-    retval = g_cvfds.poll(pargs->fds, pargs->nfds, timeout);
-    if (retval != 0 || pargs->timeout == 0) {
-      pargs->retval = retval;
-      pargs->err = errno;
-      break;
+    curr = curr->next;
+  }
+
+  if (poll_cache.free_pollers) {
+    poll_args *pargs = poll_cache.free_pollers;
+    poll_cache.free_pollers = pargs->next;
+    if (poll_cache.free_pollers) {
+      poll_cache.free_pollers->prev = NULL;
     }
+    pargs->fds = fds;
+    pargs->nfds = count;
+    pargs->next = NULL;
+    pargs->prev = NULL;
+    init_result(pargs);
+    cache_poller_locked(pargs);
+    return pargs;
+  }
+
+  poll_args *pargs = gpr_malloc(sizeof(struct poll_args));
+  gpr_cv_init(&pargs->trigger);
+  pargs->fds = fds;
+  pargs->nfds = count;
+  pargs->next = NULL;
+  pargs->prev = NULL;
+  pargs->trigger_set = 0;
+  init_result(pargs);
+  cache_poller_locked(pargs);
+  gpr_thd_id t_id;
+  gpr_thd_options opt = gpr_thd_options_default();
+  gpr_ref(&g_cvfds.pollcount);
+  gpr_thd_options_set_detached(&opt);
+  GPR_ASSERT(gpr_thd_new(&t_id, &run_poll, pargs, &opt));
+  return pargs;
+}
+
+static void cache_delete_locked(poll_args *args) {
+  if (!args->prev) {
+    uint32_t key = gpr_murmur_hash3(
+        args->fds, args->nfds * sizeof(struct pollfd), 0xDEADBEEF);
+    key = key % poll_cache.size;
+    GPR_ASSERT(poll_cache.active_pollers[key] == args);
+    poll_cache.active_pollers[key] = args->next;
+  } else {
+    args->prev->next = args->next;
   }
-  gpr_mu_lock(&g_cvfds.mu);
-  if (gpr_atm_no_barrier_load(&pargs->status) == INPROGRESS) {
-    // Signal main thread that the poll completed
-    gpr_atm_no_barrier_store(&pargs->status, COMPLETED);
-    gpr_cv_signal(pargs->cv);
+
+  if (args->next) {
+    args->next->prev = args->prev;
   }
-  decref_poll_args(pargs);
-  g_cvfds.pollcount--;
-  if (g_cvfds.shutdown && g_cvfds.pollcount == 0) {
-    gpr_cv_signal(&g_cvfds.shutdown_complete);
+
+  poll_cache.count--;
+  if (poll_cache.free_pollers) {
+    poll_cache.free_pollers->prev = args;
+  }
+  args->prev = NULL;
+  args->next = poll_cache.free_pollers;
+  gpr_free(args->fds);
+  poll_cache.free_pollers = args;
+}
+
+static void cache_poller_locked(poll_args *args) {
+  if (poll_cache.count + 1 > poll_cache.size / 2) {
+    poll_args **old_active_pollers = poll_cache.active_pollers;
+    poll_cache.size = poll_cache.size * 2;
+    poll_cache.count = 0;
+    poll_cache.active_pollers = gpr_malloc(sizeof(void *) * poll_cache.size);
+    for (unsigned int i = 0; i < poll_cache.size; i++) {
+      poll_cache.active_pollers[i] = NULL;
+    }
+    for (unsigned int i = 0; i < poll_cache.size / 2; i++) {
+      poll_args *curr = old_active_pollers[i];
+      poll_args *next = NULL;
+      while (curr) {
+        next = curr->next;
+        cache_insert_locked(curr);
+        curr = next;
+      }
+    }
+    gpr_free(old_active_pollers);
+  }
+
+  cache_insert_locked(args);
+}
+
+static void cache_destroy_locked(poll_args *args) {
+  if (args->next) {
+    args->next->prev = args->prev;
+  }
+
+  if (args->prev) {
+    args->prev->next = args->next;
+  } else {
+    poll_cache.free_pollers = args->next;
+  }
+
+  gpr_free(args);
+}
+
+static void decref_poll_result(poll_result *res) {
+  if (gpr_unref(&res->refcount)) {
+    GPR_ASSERT(!res->watchers);
+    gpr_free(res->fds);
+    gpr_free(res);
+  }
+}
+
+void remove_cvn(cv_node **head, cv_node *target) {
+  if (target->next) {
+    target->next->prev = target->prev;
+  }
+
+  if (target->prev) {
+    target->prev->next = target->next;
+  } else {
+    *head = target->next;
+  }
+}
+
+gpr_timespec thread_grace;
+
+// Poll in a background thread
+static void run_poll(void *args) {
+  poll_args *pargs = (poll_args *)args;
+  while (1) {
+    poll_result *result = pargs->result;
+    int retval = g_cvfds.poll(result->fds, result->nfds, CV_POLL_PERIOD_MS);
+    gpr_mu_lock(&g_cvfds.mu);
+    if (retval != 0) {
+      result->completed = 1;
+      result->retval = retval;
+      result->err = errno;
+      cv_node *watcher = result->watchers;
+      while (watcher) {
+        gpr_cv_signal(watcher->cv);
+        watcher = watcher->next;
+      }
+    }
+    if (result->watchcount == 0 || result->completed) {
+      cache_delete_locked(pargs);
+      decref_poll_result(result);
+      // Leave this polling thread alive for a grace period to do another poll()
+      // op
+      gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME);
+      deadline = gpr_time_add(deadline, thread_grace);
+      pargs->trigger_set = 0;
+      gpr_cv_wait(&pargs->trigger, &g_cvfds.mu, deadline);
+      if (!pargs->trigger_set) {
+        cache_destroy_locked(pargs);
+        break;
+      }
+    }
+    gpr_mu_unlock(&g_cvfds.mu);
+  }
+
+  // We still have the lock here
+  if (gpr_unref(&g_cvfds.pollcount)) {
+    gpr_cv_signal(&g_cvfds.shutdown_cv);
   }
   gpr_mu_unlock(&g_cvfds.mu);
 }
@@ -1321,24 +1508,29 @@ static void run_poll(void *arg) {
 static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
   unsigned int i;
   int res, idx;
-  gpr_cv *pollcv;
-  cv_node *cvn, *prev;
+  cv_node *pollcv;
   int skip_poll = 0;
   nfds_t nsockfds = 0;
-  gpr_thd_id t_id;
-  gpr_thd_options opt;
-  poll_args *pargs = NULL;
+  poll_result *result = NULL;
   gpr_mu_lock(&g_cvfds.mu);
-  pollcv = gpr_malloc(sizeof(gpr_cv));
-  gpr_cv_init(pollcv);
+  pollcv = gpr_malloc(sizeof(cv_node));
+  pollcv->next = NULL;
+  gpr_cv pollcv_cv;
+  gpr_cv_init(&pollcv_cv);
+  pollcv->cv = &pollcv_cv;
+  cv_node *fd_cvs = gpr_malloc(nfds * sizeof(cv_node));
+
   for (i = 0; i < nfds; i++) {
     fds[i].revents = 0;
     if (fds[i].fd < 0 && (fds[i].events & POLLIN)) {
       idx = FD_TO_IDX(fds[i].fd);
-      cvn = gpr_malloc(sizeof(cv_node));
-      cvn->cv = pollcv;
-      cvn->next = g_cvfds.cvfds[idx].cvs;
-      g_cvfds.cvfds[idx].cvs = cvn;
+      fd_cvs[i].cv = &pollcv_cv;
+      fd_cvs[i].prev = NULL;
+      fd_cvs[i].next = g_cvfds.cvfds[idx].cvs;
+      if (g_cvfds.cvfds[idx].cvs) {
+        g_cvfds.cvfds[idx].cvs->prev = &(fd_cvs[i]);
+      }
+      g_cvfds.cvfds[idx].cvs = &(fd_cvs[i]);
       // Don't bother polling if a wakeup fd is ready
       if (g_cvfds.cvfds[idx].is_set) {
         skip_poll = 1;
@@ -1348,81 +1540,68 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
     }
   }
 
+  gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME);
+  if (timeout < 0) {
+    deadline = gpr_inf_future(GPR_CLOCK_REALTIME);
+  } else {
+    deadline =
+        gpr_time_add(deadline, gpr_time_from_millis(timeout, GPR_TIMESPAN));
+  }
+
   res = 0;
   if (!skip_poll && nsockfds > 0) {
-    pargs = gpr_malloc(sizeof(struct poll_args));
-    // Both the main thread and calling thread get a reference
-    gpr_ref_init(&pargs->refcount, 2);
-    pargs->cv = pollcv;
-    pargs->fds = gpr_malloc(sizeof(struct pollfd) * nsockfds);
-    pargs->nfds = nsockfds;
-    pargs->timeout = timeout;
-    pargs->retval = 0;
-    pargs->err = 0;
-    gpr_atm_no_barrier_store(&pargs->status, INPROGRESS);
+    struct pollfd *pollfds = gpr_malloc(sizeof(struct pollfd) * nsockfds);
     idx = 0;
     for (i = 0; i < nfds; i++) {
       if (fds[i].fd >= 0) {
-        pargs->fds[idx].fd = fds[i].fd;
-        pargs->fds[idx].events = fds[i].events;
-        pargs->fds[idx].revents = 0;
+        pollfds[idx].fd = fds[i].fd;
+        pollfds[idx].events = fds[i].events;
+        pollfds[idx].revents = 0;
         idx++;
       }
     }
-    g_cvfds.pollcount++;
-    opt = gpr_thd_options_default();
-    gpr_thd_options_set_detached(&opt);
-    GPR_ASSERT(gpr_thd_new(&t_id, &run_poll, pargs, &opt));
-    // We want the poll() thread to trigger the deadline, so wait forever here
-    gpr_cv_wait(pollcv, &g_cvfds.mu, gpr_inf_future(GPR_CLOCK_MONOTONIC));
-    if (gpr_atm_no_barrier_load(&pargs->status) == COMPLETED) {
-      res = pargs->retval;
-      errno = pargs->err;
-    } else {
-      errno = 0;
-      gpr_atm_no_barrier_store(&pargs->status, CANCELLED);
+    poll_args *pargs = get_poller_locked(pollfds, nsockfds);
+    result = pargs->result;
+    pollcv->next = result->watchers;
+    pollcv->prev = NULL;
+    if (result->watchers) {
+      result->watchers->prev = pollcv;
     }
+    result->watchers = pollcv;
+    result->watchcount++;
+    gpr_ref(&result->refcount);
+
+    pargs->trigger_set = 1;
+    gpr_cv_signal(&pargs->trigger);
+    gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline);
+    res = result->retval;
+    errno = result->err;
+    result->watchcount--;
+    remove_cvn(&result->watchers, pollcv);
   } else if (!skip_poll) {
-    gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME);
-    deadline =
-        gpr_time_add(deadline, gpr_time_from_millis(timeout, GPR_TIMESPAN));
-    gpr_cv_wait(pollcv, &g_cvfds.mu, deadline);
+    gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline);
   }
 
   idx = 0;
   for (i = 0; i < nfds; i++) {
     if (fds[i].fd < 0 && (fds[i].events & POLLIN)) {
-      cvn = g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs;
-      prev = NULL;
-      while (cvn->cv != pollcv) {
-        prev = cvn;
-        cvn = cvn->next;
-        GPR_ASSERT(cvn);
-      }
-      if (!prev) {
-        g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs = cvn->next;
-      } else {
-        prev->next = cvn->next;
-      }
-      gpr_free(cvn);
-
+      remove_cvn(&g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i]));
       if (g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].is_set) {
         fds[i].revents = POLLIN;
         if (res >= 0) res++;
       }
-    } else if (!skip_poll && fds[i].fd >= 0 &&
-               gpr_atm_no_barrier_load(&pargs->status) == COMPLETED) {
-      fds[i].revents = pargs->fds[idx].revents;
+    } else if (!skip_poll && fds[i].fd >= 0 && result->completed) {
+      fds[i].revents = result->fds[idx].revents;
       idx++;
     }
   }
 
-  if (pargs) {
-    decref_poll_args(pargs);
-  } else {
-    gpr_cv_destroy(pollcv);
-    gpr_free(pollcv);
+  gpr_free(fd_cvs);
+  gpr_free(pollcv);
+  if (result) {
+    decref_poll_result(result);
   }
+
   gpr_mu_unlock(&g_cvfds.mu);
 
   return res;
@@ -1431,12 +1610,12 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
 static void global_cv_fd_table_init() {
   gpr_mu_init(&g_cvfds.mu);
   gpr_mu_lock(&g_cvfds.mu);
-  gpr_cv_init(&g_cvfds.shutdown_complete);
-  g_cvfds.shutdown = 0;
-  g_cvfds.pollcount = 0;
+  gpr_cv_init(&g_cvfds.shutdown_cv);
+  gpr_ref_init(&g_cvfds.pollcount, 1);
   g_cvfds.size = CV_DEFAULT_TABLE_SIZE;
   g_cvfds.cvfds = gpr_malloc(sizeof(fd_node) * CV_DEFAULT_TABLE_SIZE);
   g_cvfds.free_fds = NULL;
+  thread_grace = gpr_time_from_millis(POLLCV_THREAD_GRACE_MS, GPR_TIMESPAN);
   for (int i = 0; i < CV_DEFAULT_TABLE_SIZE; i++) {
     g_cvfds.cvfds[i].is_set = 0;
     g_cvfds.cvfds[i].cvs = NULL;
@@ -1446,23 +1625,35 @@ static void global_cv_fd_table_init() {
   // Override the poll function with one that supports cvfds
   g_cvfds.poll = grpc_poll_function;
   grpc_poll_function = &cvfd_poll;
+
+  // Initialize the cache
+  poll_cache.size = 32;
+  poll_cache.count = 0;
+  poll_cache.free_pollers = NULL;
+  poll_cache.active_pollers = gpr_malloc(sizeof(void *) * 32);
+  for (unsigned int i = 0; i < poll_cache.size; i++) {
+    poll_cache.active_pollers[i] = NULL;
+  }
+
   gpr_mu_unlock(&g_cvfds.mu);
 }
 
 static void global_cv_fd_table_shutdown() {
   gpr_mu_lock(&g_cvfds.mu);
-  g_cvfds.shutdown = 1;
   // Attempt to wait for all abandoned poll() threads to terminate
   // Not doing so will result in reported memory leaks
-  if (g_cvfds.pollcount > 0) {
-    int res = gpr_cv_wait(&g_cvfds.shutdown_complete, &g_cvfds.mu,
+  if (!gpr_unref(&g_cvfds.pollcount)) {
+    int res = gpr_cv_wait(&g_cvfds.shutdown_cv, &g_cvfds.mu,
                           gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
                                        gpr_time_from_seconds(3, GPR_TIMESPAN)));
     GPR_ASSERT(res == 0);
   }
-  gpr_cv_destroy(&g_cvfds.shutdown_complete);
+  gpr_cv_destroy(&g_cvfds.shutdown_cv);
   grpc_poll_function = g_cvfds.poll;
   gpr_free(g_cvfds.cvfds);
+
+  gpr_free(poll_cache.active_pollers);
+
   gpr_mu_unlock(&g_cvfds.mu);
   gpr_mu_destroy(&g_cvfds.mu);
 }
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index cff77e641c..91f8cd5482 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -170,8 +170,9 @@ int grpc_fd_wrapped_fd(grpc_fd *fd) {
 }
 
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason) {
-  g_event_engine->fd_orphan(exec_ctx, fd, on_done, release_fd, reason);
+                    int *release_fd, bool already_closed, const char *reason) {
+  g_event_engine->fd_orphan(exec_ctx, fd, on_done, release_fd, already_closed,
+                            reason);
 }
 
 void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 0de7333843..1108e46ef8 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -37,7 +37,7 @@ typedef struct grpc_event_engine_vtable {
   grpc_fd *(*fd_create)(int fd, const char *name);
   int (*fd_wrapped_fd)(grpc_fd *fd);
   void (*fd_orphan)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason);
+                    int *release_fd, bool already_closed, const char *reason);
   void (*fd_shutdown)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why);
   void (*fd_notify_on_read)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                             grpc_closure *closure);
@@ -104,7 +104,7 @@ int grpc_fd_wrapped_fd(grpc_fd *fd);
    notify_on_write.
    MUST NOT be called with a pollset lock taken */
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason);
+                    int *release_fd, bool already_closed, const char *reason);
 
 /* Has grpc_fd_shutdown been called on an fd? */
 bool grpc_fd_is_shutdown(grpc_fd *fd);
diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c
index 833170ceed..41c69add17 100644
--- a/src/core/lib/iomgr/exec_ctx.c
+++ b/src/core/lib/iomgr/exec_ctx.c
@@ -51,33 +51,6 @@ bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) {
          !grpc_closure_list_empty(exec_ctx->closure_list);
 }
 
-bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
-  bool did_something = 0;
-  GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);
-  for (;;) {
-    if (!grpc_closure_list_empty(exec_ctx->closure_list)) {
-      grpc_closure *c = exec_ctx->closure_list.head;
-      exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL;
-      while (c != NULL) {
-        grpc_closure *next = c->next_data.next;
-        grpc_error *error = c->error_data.error;
-        did_something = true;
-#ifndef NDEBUG
-        c->scheduled = false;
-#endif
-        c->cb(exec_ctx, c->cb_arg, error);
-        GRPC_ERROR_UNREF(error);
-        c = next;
-      }
-    } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) {
-      break;
-    }
-  }
-  GPR_ASSERT(exec_ctx->active_combiner == NULL);
-  GPR_TIMER_END("grpc_exec_ctx_flush", 0);
-  return did_something;
-}
-
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {
   exec_ctx->flags |= GRPC_EXEC_CTX_FLAG_IS_FINISHED;
   grpc_exec_ctx_flush(exec_ctx);
@@ -103,6 +76,29 @@ static void exec_ctx_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
   GRPC_ERROR_UNREF(error);
 }
 
+bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
+  bool did_something = 0;
+  GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);
+  for (;;) {
+    if (!grpc_closure_list_empty(exec_ctx->closure_list)) {
+      grpc_closure *c = exec_ctx->closure_list.head;
+      exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL;
+      while (c != NULL) {
+        grpc_closure *next = c->next_data.next;
+        grpc_error *error = c->error_data.error;
+        did_something = true;
+        exec_ctx_run(exec_ctx, c, error);
+        c = next;
+      }
+    } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) {
+      break;
+    }
+  }
+  GPR_ASSERT(exec_ctx->active_combiner == NULL);
+  GPR_TIMER_END("grpc_exec_ctx_flush", 0);
+  return did_something;
+}
+
 static void exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                            grpc_error *error) {
   grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
diff --git a/src/core/lib/iomgr/gethostname.h b/src/core/lib/iomgr/gethostname.h
new file mode 100644
index 0000000000..9c6b9d8d42
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname.h
@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H
+#define GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H
+
+// Returns the hostname of the local machine.
+// Caller takes ownership of result.
+char *grpc_gethostname();
+
+#endif /* GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H */
diff --git a/src/core/lib/support/thd_internal.h b/src/core/lib/iomgr/gethostname_fallback.c
index cc468c7846..6229461568 100644
--- a/src/core/lib/support/thd_internal.h
+++ b/src/core/lib/iomgr/gethostname_fallback.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015 gRPC authors.
+ * Copyright 2017 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H
-#define GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H
+#include "src/core/lib/iomgr/port.h"
 
-/* Internal interfaces between modules within the gpr support library.  */
+#ifdef GRPC_GETHOSTNAME_FALLBACK
 
-#endif /* GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H */
+#include <stddef.h>
+
+char *grpc_gethostname() { return NULL; }
+
+#endif  // GRPC_GETHOSTNAME_FALLBACK
diff --git a/src/core/lib/iomgr/gethostname_host_name_max.c b/src/core/lib/iomgr/gethostname_host_name_max.c
new file mode 100644
index 0000000000..4d0511412e
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname_host_name_max.c
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_POSIX_HOST_NAME_MAX
+
+#include <limits.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+
+char *grpc_gethostname() {
+  char *hostname = (char *)gpr_malloc(HOST_NAME_MAX);
+  if (gethostname(hostname, HOST_NAME_MAX) != 0) {
+    gpr_free(hostname);
+    return NULL;
+  }
+  return hostname;
+}
+
+#endif  // GRPC_POSIX_HOST_NAME_MAX
diff --git a/src/core/lib/iomgr/gethostname_sysconf.c b/src/core/lib/iomgr/gethostname_sysconf.c
new file mode 100644
index 0000000000..51bac5d69d
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname_sysconf.c
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_POSIX_SYSCONF
+
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+
+char *grpc_gethostname() {
+  size_t host_name_max = (size_t)sysconf(_SC_HOST_NAME_MAX);
+  char *hostname = (char *)gpr_malloc(host_name_max);
+  if (gethostname(hostname, host_name_max) != 0) {
+    gpr_free(hostname);
+    return NULL;
+  }
+  return hostname;
+}
+
+#endif  // GRPC_POSIX_SYSCONF
diff --git a/src/core/lib/iomgr/iomgr_uv.c b/src/core/lib/iomgr/iomgr_uv.c
index 8b1245c640..df5d23af3b 100644
--- a/src/core/lib/iomgr/iomgr_uv.c
+++ b/src/core/lib/iomgr/iomgr_uv.c
@@ -21,12 +21,20 @@
 #ifdef GRPC_UV
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/pollset_uv.h"
 #include "src/core/lib/iomgr/tcp_uv.h"
 
+gpr_thd_id g_init_thread;
+
 void grpc_iomgr_platform_init(void) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_pollset_global_init();
   grpc_register_tracer(&grpc_tcp_trace);
+  grpc_executor_set_threading(&exec_ctx, false);
+  g_init_thread = gpr_thd_currentid();
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 void grpc_iomgr_platform_flush(void) {}
 void grpc_iomgr_platform_shutdown(void) { grpc_pollset_global_shutdown(); }
diff --git a/src/core/lib/iomgr/iomgr_uv.h b/src/core/lib/iomgr/iomgr_uv.h
new file mode 100644
index 0000000000..3b4daaa73b
--- /dev/null
+++ b/src/core/lib/iomgr/iomgr_uv.h
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_IOMGR_UV_H
+#define GRPC_CORE_LIB_IOMGR_IOMGR_UV_H
+
+#include "src/core/lib/iomgr/iomgr_internal.h"
+
+#include <grpc/support/thd.h>
+
+/* The thread ID of the thread on which grpc was initialized. Used to verify
+ * that all calls into libuv are made on that same thread */
+extern gpr_thd_id g_init_thread;
+
+#ifdef GRPC_UV_THREAD_CHECK
+#define GRPC_UV_ASSERT_SAME_THREAD() \
+  GPR_ASSERT(gpr_thd_currentid() == g_init_thread)
+#else
+#define GRPC_UV_ASSERT_SAME_THREAD()
+#endif /* GRPC_UV_THREAD_CHECK */
+
+#endif /* GRPC_CORE_LIB_IOMGR_IOMGR_UV_H */
diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c
index c2ceecb3c5..f967b22ba9 100644
--- a/src/core/lib/iomgr/lockfree_event.c
+++ b/src/core/lib/iomgr/lockfree_event.c
@@ -79,12 +79,12 @@ bool grpc_lfev_is_shutdown(gpr_atm *state) {
 }
 
 void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state,
-                         grpc_closure *closure) {
+                         grpc_closure *closure, const char *variable) {
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
-      gpr_log(GPR_DEBUG, "lfev_notify_on: %p curr=%p closure=%p", state,
-              (void *)curr, closure);
+      gpr_log(GPR_ERROR, "lfev_notify_on[%s]: %p curr=%p closure=%p", variable,
+              state, (void *)curr, closure);
     }
     switch (curr) {
       case CLOSURE_NOT_READY: {
@@ -149,7 +149,7 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
-      gpr_log(GPR_DEBUG, "lfev_set_shutdown: %p curr=%p err=%s", state,
+      gpr_log(GPR_ERROR, "lfev_set_shutdown: %p curr=%p err=%s", state,
               (void *)curr, grpc_error_string(shutdown_err));
     }
     switch (curr) {
@@ -193,12 +193,14 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
   GPR_UNREACHABLE_CODE(return false);
 }
 
-void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) {
+void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state,
+                         const char *variable) {
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
 
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
-      gpr_log(GPR_DEBUG, "lfev_set_ready: %p curr=%p", state, (void *)curr);
+      gpr_log(GPR_ERROR, "lfev_set_ready[%s]: %p curr=%p", variable, state,
+              (void *)curr);
     }
 
     switch (curr) {
diff --git a/src/core/lib/iomgr/lockfree_event.h b/src/core/lib/iomgr/lockfree_event.h
index ef3844a07a..6a14a0f3b2 100644
--- a/src/core/lib/iomgr/lockfree_event.h
+++ b/src/core/lib/iomgr/lockfree_event.h
@@ -30,10 +30,11 @@ void grpc_lfev_destroy(gpr_atm *state);
 bool grpc_lfev_is_shutdown(gpr_atm *state);
 
 void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state,
-                         grpc_closure *closure);
+                         grpc_closure *closure, const char *variable);
 /* Returns true on first successful shutdown */
 bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
                             grpc_error *shutdown_err);
-void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state);
+void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state,
+                         const char *variable);
 
 #endif /* GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H */
diff --git a/src/core/lib/iomgr/nameser.h b/src/core/lib/iomgr/nameser.h
new file mode 100644
index 0000000000..daed6de518
--- /dev/null
+++ b/src/core/lib/iomgr/nameser.h
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_NAMESER_H
+#define GRPC_CORE_LIB_IOMGR_NAMESER_H
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_HAVE_ARPA_NAMESER
+
+#include <arpa/nameser.h>
+
+#else /* GRPC_HAVE_ARPA_NAMESER */
+
+typedef enum __ns_class {
+  ns_c_invalid = 0, /* Cookie. */
+  ns_c_in = 1,      /* Internet. */
+  ns_c_2 = 2,       /* unallocated/unsupported. */
+  ns_c_chaos = 3,   /* MIT Chaos-net. */
+  ns_c_hs = 4,      /* MIT Hesiod. */
+  /* Query class values which do not appear in resource records */
+  ns_c_none = 254, /* for prereq. sections in update requests */
+  ns_c_any = 255,  /* Wildcard match. */
+  ns_c_max = 65536
+} ns_class;
+
+typedef enum __ns_type {
+  ns_t_invalid = 0,   /* Cookie. */
+  ns_t_a = 1,         /* Host address. */
+  ns_t_ns = 2,        /* Authoritative server. */
+  ns_t_md = 3,        /* Mail destination. */
+  ns_t_mf = 4,        /* Mail forwarder. */
+  ns_t_cname = 5,     /* Canonical name. */
+  ns_t_soa = 6,       /* Start of authority zone. */
+  ns_t_mb = 7,        /* Mailbox domain name. */
+  ns_t_mg = 8,        /* Mail group member. */
+  ns_t_mr = 9,        /* Mail rename name. */
+  ns_t_null = 10,     /* Null resource record. */
+  ns_t_wks = 11,      /* Well known service. */
+  ns_t_ptr = 12,      /* Domain name pointer. */
+  ns_t_hinfo = 13,    /* Host information. */
+  ns_t_minfo = 14,    /* Mailbox information. */
+  ns_t_mx = 15,       /* Mail routing information. */
+  ns_t_txt = 16,      /* Text strings. */
+  ns_t_rp = 17,       /* Responsible person. */
+  ns_t_afsdb = 18,    /* AFS cell database. */
+  ns_t_x25 = 19,      /* X_25 calling address. */
+  ns_t_isdn = 20,     /* ISDN calling address. */
+  ns_t_rt = 21,       /* Router. */
+  ns_t_nsap = 22,     /* NSAP address. */
+  ns_t_nsap_ptr = 23, /* Reverse NSAP lookup (deprecated). */
+  ns_t_sig = 24,      /* Security signature. */
+  ns_t_key = 25,      /* Security key. */
+  ns_t_px = 26,       /* X.400 mail mapping. */
+  ns_t_gpos = 27,     /* Geographical position (withdrawn). */
+  ns_t_aaaa = 28,     /* Ip6 Address. */
+  ns_t_loc = 29,      /* Location Information. */
+  ns_t_nxt = 30,      /* Next domain (security). */
+  ns_t_eid = 31,      /* Endpoint identifier. */
+  ns_t_nimloc = 32,   /* Nimrod Locator. */
+  ns_t_srv = 33,      /* Server Selection. */
+  ns_t_atma = 34,     /* ATM Address */
+  ns_t_naptr = 35,    /* Naming Authority PoinTeR */
+  ns_t_kx = 36,       /* Key Exchange */
+  ns_t_cert = 37,     /* Certification record */
+  ns_t_a6 = 38,       /* IPv6 address (deprecates AAAA) */
+  ns_t_dname = 39,    /* Non-terminal DNAME (for IPv6) */
+  ns_t_sink = 40,     /* Kitchen sink (experimentatl) */
+  ns_t_opt = 41,      /* EDNS0 option (meta-RR) */
+  ns_t_apl = 42,      /* Address prefix list (RFC3123) */
+  ns_t_ds = 43,       /* Delegation Signer (RFC4034) */
+  ns_t_sshfp = 44,    /* SSH Key Fingerprint (RFC4255) */
+  ns_t_rrsig = 46,    /* Resource Record Signature (RFC4034) */
+  ns_t_nsec = 47,     /* Next Secure (RFC4034) */
+  ns_t_dnskey = 48,   /* DNS Public Key (RFC4034) */
+  ns_t_tkey = 249,    /* Transaction key */
+  ns_t_tsig = 250,    /* Transaction signature. */
+  ns_t_ixfr = 251,    /* Incremental zone transfer. */
+  ns_t_axfr = 252,    /* Transfer zone of authority. */
+  ns_t_mailb = 253,   /* Transfer mailbox records. */
+  ns_t_maila = 254,   /* Transfer mail agent records. */
+  ns_t_any = 255,     /* Wildcard match. */
+  ns_t_zxfr = 256,    /* BIND-specific, nonstandard. */
+  ns_t_max = 65536
+} ns_type;
+
+#endif /* GRPC_HAVE_ARPA_NAMESER */
+
+#endif /* GRPC_CORE_LIB_IOMGR_NAMESER_H */
diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c
index 946f0c8c60..a79fe89d3e 100644
--- a/src/core/lib/iomgr/pollset_uv.c
+++ b/src/core/lib/iomgr/pollset_uv.c
@@ -28,6 +28,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_uv.h"
 
@@ -70,6 +71,7 @@ void grpc_pollset_global_init(void) {
 }
 
 void grpc_pollset_global_shutdown(void) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   gpr_mu_destroy(&grpc_polling_mu);
   uv_close((uv_handle_t *)dummy_uv_handle, dummy_handle_close_cb);
 }
@@ -79,6 +81,7 @@ static void timer_run_cb(uv_timer_t *timer) {}
 static void timer_close_cb(uv_handle_t *handle) { handle->data = (void *)1; }
 
 void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   *mu = &grpc_polling_mu;
   uv_timer_init(uv_default_loop(), &pollset->timer);
   pollset->shutting_down = 0;
@@ -87,6 +90,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
 void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                            grpc_closure *closure) {
   GPR_ASSERT(!pollset->shutting_down);
+  GRPC_UV_ASSERT_SAME_THREAD();
   pollset->shutting_down = 1;
   if (grpc_pollset_work_run_loop) {
     // Drain any pending UV callbacks without blocking
@@ -99,6 +103,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 }
 
 void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   uv_close((uv_handle_t *)&pollset->timer, timer_close_cb);
   // timer.data is a boolean indicating that the timer has finished closing
   pollset->timer.data = (void *)0;
@@ -113,6 +118,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               grpc_pollset_worker **worker_hdl,
                               gpr_timespec now, gpr_timespec deadline) {
   uint64_t timeout;
+  GRPC_UV_ASSERT_SAME_THREAD();
   gpr_mu_unlock(&grpc_polling_mu);
   if (grpc_pollset_work_run_loop) {
     if (gpr_time_cmp(deadline, now) >= 0) {
@@ -141,6 +147,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
 grpc_error *grpc_pollset_kick(grpc_pollset *pollset,
                               grpc_pollset_worker *specific_worker) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0);
   return GRPC_ERROR_NONE;
 }
diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h
index f5d15b4850..42033d0ba4 100644
--- a/src/core/lib/iomgr/port.h
+++ b/src/core/lib/iomgr/port.h
@@ -24,6 +24,7 @@
 #if defined(GRPC_UV)
 // Do nothing
 #elif defined(GPR_MANYLINUX1)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
 #define GRPC_HAVE_IP_PKTINFO 1
@@ -51,12 +52,14 @@
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_LINUX)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
 #define GRPC_HAVE_IP_PKTINFO 1
 #define GRPC_HAVE_MSG_NOSIGNAL 1
 #define GRPC_HAVE_UNIX_SOCKET 1
 #define GRPC_LINUX_MULTIPOLL_WITH_EPOLL 1
+#define GRPC_POSIX_HOST_NAME_MAX 1
 #define GRPC_POSIX_SOCKET 1
 #define GRPC_POSIX_SOCKETADDR 1
 #define GRPC_POSIX_WAKEUP_FD 1
@@ -82,6 +85,7 @@
 #define GRPC_POSIX_SOCKETUTILS
 #endif
 #elif defined(GPR_APPLE)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_SO_NOSIGPIPE 1
 #define GRPC_HAVE_UNIX_SOCKET 1
@@ -90,9 +94,11 @@
 #define GRPC_POSIX_SOCKET 1
 #define GRPC_POSIX_SOCKETADDR 1
 #define GRPC_POSIX_SOCKETUTILS 1
+#define GRPC_POSIX_SYSCONF 1
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_FREEBSD)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
 #define GRPC_HAVE_SO_NOSIGPIPE 1
@@ -104,6 +110,7 @@
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_NACL)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1
 #define GRPC_POSIX_SOCKET 1
 #define GRPC_POSIX_SOCKETADDR 1
@@ -120,4 +127,11 @@
 #error Must define exactly one of GRPC_POSIX_SOCKET, GRPC_WINSOCK_SOCKET, GPR_CUSTOM_SOCKET
 #endif
 
+#if defined(GRPC_POSIX_HOST_NAME_MAX) && defined(GRPC_POSIX_SYSCONF)
+#error "Cannot define both GRPC_POSIX_HOST_NAME_MAX and GRPC_POSIX_SYSCONF"
+#endif
+#if !defined(GRPC_POSIX_HOST_NAME_MAX) && !defined(GRPC_POSIX_SYSCONF)
+#define GRPC_GETHOSTNAME_FALLBACK 1
+#endif
+
 #endif /* GRPC_CORE_LIB_IOMGR_PORT_H */
diff --git a/src/core/lib/iomgr/resolve_address_uv.c b/src/core/lib/iomgr/resolve_address_uv.c
index a98b8e62db..2d438e8b48 100644
--- a/src/core/lib/iomgr/resolve_address_uv.c
+++ b/src/core/lib/iomgr/resolve_address_uv.c
@@ -30,6 +30,7 @@
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -114,11 +115,14 @@ static void getaddrinfo_callback(uv_getaddrinfo_t *req, int status,
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_error *error;
   int retry_status;
+  char *port = r->port;
 
   gpr_free(req);
   retry_status = retry_named_port_failure(status, r, getaddrinfo_callback);
   if (retry_status == 0) {
-    // The request is being retried. Nothing should be done here
+    /* The request is being retried. It is using its own port string, so we free
+     * the original one */
+    gpr_free(port);
     return;
   }
   /* Either no retry was attempted, or the retry failed. Either way, the
@@ -171,6 +175,8 @@ static grpc_error *blocking_resolve_address_impl(
   grpc_error *err;
   int retry_status;
 
+  GRPC_UV_ASSERT_SAME_THREAD();
+
   req.addrinfo = NULL;
 
   err = try_split_host_port(name, default_port, &host, &port);
@@ -218,16 +224,19 @@ static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name,
                                  grpc_pollset_set *interested_parties,
                                  grpc_closure *on_done,
                                  grpc_resolved_addresses **addrs) {
-  uv_getaddrinfo_t *req;
-  request *r;
-  struct addrinfo *hints;
-  char *host;
-  char *port;
+  uv_getaddrinfo_t *req = NULL;
+  request *r = NULL;
+  struct addrinfo *hints = NULL;
+  char *host = NULL;
+  char *port = NULL;
   grpc_error *err;
   int s;
+  GRPC_UV_ASSERT_SAME_THREAD();
   err = try_split_host_port(name, default_port, &host, &port);
   if (err != GRPC_ERROR_NONE) {
     GRPC_CLOSURE_SCHED(exec_ctx, on_done, err);
+    gpr_free(host);
+    gpr_free(port);
     return;
   }
   r = gpr_malloc(sizeof(request));
diff --git a/src/core/lib/iomgr/sockaddr_utils.c b/src/core/lib/iomgr/sockaddr_utils.c
index 99dc2f1c78..3f4145d104 100644
--- a/src/core/lib/iomgr/sockaddr_utils.c
+++ b/src/core/lib/iomgr/sockaddr_utils.c
@@ -220,6 +220,11 @@ const char *grpc_sockaddr_get_uri_scheme(
   return NULL;
 }
 
+int grpc_sockaddr_get_family(const grpc_resolved_address *resolved_addr) {
+  const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr;
+  return addr->sa_family;
+}
+
 int grpc_sockaddr_get_port(const grpc_resolved_address *resolved_addr) {
   const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr;
   switch (addr->sa_family) {
diff --git a/src/core/lib/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h
index 7692b969f2..a589a19705 100644
--- a/src/core/lib/iomgr/sockaddr_utils.h
+++ b/src/core/lib/iomgr/sockaddr_utils.h
@@ -75,4 +75,6 @@ char *grpc_sockaddr_to_uri(const grpc_resolved_address *addr);
 /* Returns the URI scheme corresponding to \a addr */
 const char *grpc_sockaddr_get_uri_scheme(const grpc_resolved_address *addr);
 
+int grpc_sockaddr_get_family(const grpc_resolved_address *resolved_addr);
+
 #endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H */
diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index 21e320a6e7..a25fba4527 100644
--- a/src/core/lib/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -209,7 +209,8 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
 finish:
   if (fd != NULL) {
     grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd);
-    grpc_fd_orphan(exec_ctx, fd, NULL, NULL, "tcp_client_orphan");
+    grpc_fd_orphan(exec_ctx, fd, NULL, NULL, false /* already_closed */,
+                   "tcp_client_orphan");
     fd = NULL;
   }
   done = (--ac->refs == 0);
@@ -295,7 +296,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   }
 
   if (errno != EWOULDBLOCK && errno != EINPROGRESS) {
-    grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error");
+    grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, false /* already_closed */,
+                   "tcp_client_connect_error");
     GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"));
     goto done;
   }
diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c
index 2f1d237c07..786c456b73 100644
--- a/src/core/lib/iomgr/tcp_client_uv.c
+++ b/src/core/lib/iomgr/tcp_client_uv.c
@@ -26,6 +26,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/iomgr/tcp_uv.h"
@@ -124,6 +125,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   (void)channel_args;
   (void)interested_parties;
 
+  GRPC_UV_ASSERT_SAME_THREAD();
+
   if (channel_args != NULL) {
     for (size_t i = 0; i < channel_args->num_args; i++) {
       if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index 668f6bb447..7e789cc5b5 100644
--- a/src/core/lib/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -157,7 +157,7 @@ static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
 static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   grpc_fd_orphan(exec_ctx, tcp->em_fd, tcp->release_fd_cb, tcp->release_fd,
-                 "tcp_unref_orphan");
+                 false /* already_closed */, "tcp_unref_orphan");
   grpc_slice_buffer_destroy_internal(exec_ctx, &tcp->last_read_buffer);
   grpc_resource_user_unref(exec_ctx, tcp->resource_user);
   gpr_free(tcp->peer_string);
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index f304642951..0fc5c0fd86 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -166,7 +166,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
       GRPC_CLOSURE_INIT(&sp->destroyed_closure, destroyed_port, s,
                         grpc_schedule_on_exec_ctx);
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
-                     "tcp_listener_shutdown");
+                     false /* already_closed */, "tcp_listener_shutdown");
     }
     gpr_mu_unlock(&s->mu);
   } else {
diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_common.c b/src/core/lib/iomgr/tcp_server_utils_posix_common.c
index dbb43186bd..ad535bc43e 100644
--- a/src/core/lib/iomgr/tcp_server_utils_posix_common.c
+++ b/src/core/lib/iomgr/tcp_server_utils_posix_common.c
@@ -39,7 +39,7 @@
 
 #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100
 
-static gpr_once s_init_max_accept_queue_size;
+static gpr_once s_init_max_accept_queue_size = GPR_ONCE_INIT;
 static int s_max_accept_queue_size;
 
 /* get max listen queue size on linux */
diff --git a/src/core/lib/iomgr/tcp_server_uv.c b/src/core/lib/iomgr/tcp_server_uv.c
index 2ab836cc34..3b9332321f 100644
--- a/src/core/lib/iomgr/tcp_server_uv.c
+++ b/src/core/lib/iomgr/tcp_server_uv.c
@@ -20,6 +20,7 @@
 
 #ifdef GRPC_UV
 
+#include <assert.h>
 #include <string.h>
 
 #include <grpc/support/alloc.h>
@@ -27,6 +28,7 @@
 
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/tcp_server.h"
@@ -43,6 +45,8 @@ struct grpc_tcp_listener {
   struct grpc_tcp_listener *next;
 
   bool closed;
+
+  bool has_pending_connection;
 };
 
 struct grpc_tcp_server {
@@ -104,6 +108,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
 }
 
 grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   gpr_ref(&s->refs);
   return s;
 }
@@ -168,6 +173,7 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 }
 
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   if (gpr_unref(&s->refs)) {
     /* Complete shutdown_starting work before destroying. */
     grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -183,18 +189,49 @@ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   }
 }
 
-static void accepted_connection_close_cb(uv_handle_t *handle) {
-  gpr_free(handle);
-}
-
-static void on_connect(uv_stream_t *server, int status) {
-  grpc_tcp_listener *sp = (grpc_tcp_listener *)server->data;
+static void finish_accept(grpc_exec_ctx *exec_ctx, grpc_tcp_listener *sp) {
+  grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
   uv_tcp_t *client;
   grpc_endpoint *ep = NULL;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_resolved_address peer_name;
   char *peer_name_string;
   int err;
+  uv_tcp_t *server = sp->handle;
+
+  client = gpr_malloc(sizeof(uv_tcp_t));
+  uv_tcp_init(uv_default_loop(), client);
+  // UV documentation says this is guaranteed to succeed
+  uv_accept((uv_stream_t *)server, (uv_stream_t *)client);
+  peer_name_string = NULL;
+  memset(&peer_name, 0, sizeof(grpc_resolved_address));
+  peer_name.len = sizeof(struct sockaddr_storage);
+  err = uv_tcp_getpeername(client, (struct sockaddr *)&peer_name.addr,
+                           (int *)&peer_name.len);
+  if (err == 0) {
+    peer_name_string = grpc_sockaddr_to_uri(&peer_name);
+  } else {
+    gpr_log(GPR_INFO, "uv_tcp_getpeername error: %s", uv_strerror(err));
+  }
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    if (peer_name_string) {
+      gpr_log(GPR_DEBUG, "SERVER_CONNECT: %p accepted connection: %s",
+              sp->server, peer_name_string);
+    } else {
+      gpr_log(GPR_DEBUG, "SERVER_CONNECT: %p accepted connection", sp->server);
+    }
+  }
+  ep = grpc_tcp_create(client, sp->server->resource_quota, peer_name_string);
+  acceptor->from_server = sp->server;
+  acceptor->port_index = sp->port_index;
+  acceptor->fd_index = 0;
+  sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
+                           acceptor);
+  gpr_free(peer_name_string);
+}
+
+static void on_connect(uv_stream_t *server, int status) {
+  grpc_tcp_listener *sp = (grpc_tcp_listener *)server->data;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
   if (status < 0) {
     switch (status) {
@@ -207,35 +244,19 @@ static void on_connect(uv_stream_t *server, int status) {
     }
   }
 
-  client = gpr_malloc(sizeof(uv_tcp_t));
-  uv_tcp_init(uv_default_loop(), client);
-  // UV documentation says this is guaranteed to succeed
-  uv_accept((uv_stream_t *)server, (uv_stream_t *)client);
-  // If the server has not been started, we discard incoming connections
-  if (sp->server->on_accept_cb == NULL) {
-    uv_close((uv_handle_t *)client, accepted_connection_close_cb);
+  GPR_ASSERT(!sp->has_pending_connection);
+
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    gpr_log(GPR_DEBUG, "SERVER_CONNECT: %p incoming connection", sp->server);
+  }
+
+  // Create acceptor.
+  if (sp->server->on_accept_cb) {
+    finish_accept(&exec_ctx, sp);
   } else {
-    peer_name_string = NULL;
-    memset(&peer_name, 0, sizeof(grpc_resolved_address));
-    peer_name.len = sizeof(struct sockaddr_storage);
-    err = uv_tcp_getpeername(client, (struct sockaddr *)&peer_name.addr,
-                             (int *)&peer_name.len);
-    if (err == 0) {
-      peer_name_string = grpc_sockaddr_to_uri(&peer_name);
-    } else {
-      gpr_log(GPR_INFO, "uv_tcp_getpeername error: %s", uv_strerror(status));
-    }
-    ep = grpc_tcp_create(client, sp->server->resource_quota, peer_name_string);
-    // Create acceptor.
-    grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
-    acceptor->from_server = sp->server;
-    acceptor->port_index = sp->port_index;
-    acceptor->fd_index = 0;
-    sp->server->on_accept_cb(&exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
-                             acceptor);
-    grpc_exec_ctx_finish(&exec_ctx);
-    gpr_free(peer_name_string);
+    sp->has_pending_connection = true;
   }
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static grpc_error *add_socket_to_server(grpc_tcp_server *s, uv_tcp_t *handle,
@@ -282,7 +303,7 @@ static grpc_error *add_socket_to_server(grpc_tcp_server *s, uv_tcp_t *handle,
 
   GPR_ASSERT(port >= 0);
   GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server");
-  sp = gpr_malloc(sizeof(grpc_tcp_listener));
+  sp = gpr_zalloc(sizeof(grpc_tcp_listener));
   sp->next = NULL;
   if (s->head == NULL) {
     s->head = sp;
@@ -316,6 +337,9 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
   unsigned port_index = 0;
   int status;
   grpc_error *error = GRPC_ERROR_NONE;
+  int family;
+
+  GRPC_UV_ASSERT_SAME_THREAD();
 
   if (s->tail != NULL) {
     port_index = s->tail->port_index + 1;
@@ -353,7 +377,18 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
   }
 
   handle = gpr_malloc(sizeof(uv_tcp_t));
-  status = uv_tcp_init(uv_default_loop(), handle);
+
+  family = grpc_sockaddr_get_family(addr);
+  status = uv_tcp_init_ex(uv_default_loop(), handle, (unsigned int)family);
+#if defined(GPR_LINUX) && defined(SO_REUSEPORT)
+  if (family == AF_INET || family == AF_INET6) {
+    int fd;
+    uv_fileno((uv_handle_t *)handle, &fd);
+    int enable = 1;
+    setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(enable));
+  }
+#endif /* GPR_LINUX && SO_REUSEPORT */
+
   if (status == 0) {
     error = add_socket_to_server(s, handle, addr, port_index, &sp);
   } else {
@@ -366,6 +401,18 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
 
   gpr_free(allocated_addr);
 
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    char *port_string;
+    grpc_sockaddr_to_string(&port_string, addr, 0);
+    const char *str = grpc_error_string(error);
+    if (port_string) {
+      gpr_log(GPR_DEBUG, "SERVER %p add_port %s error=%s", s, port_string, str);
+      gpr_free(port_string);
+    } else {
+      gpr_log(GPR_DEBUG, "SERVER %p add_port error=%s", s, str);
+    }
+  }
+
   if (error != GRPC_ERROR_NONE) {
     grpc_error *error_out = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
         "Failed to add port to server", &error, 1);
@@ -385,13 +432,19 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
   grpc_tcp_listener *sp;
   (void)pollsets;
   (void)pollset_count;
+  GRPC_UV_ASSERT_SAME_THREAD();
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    gpr_log(GPR_DEBUG, "SERVER_START %p", server);
+  }
   GPR_ASSERT(on_accept_cb);
   GPR_ASSERT(!server->on_accept_cb);
   server->on_accept_cb = on_accept_cb;
   server->on_accept_cb_arg = cb_arg;
   for (sp = server->head; sp; sp = sp->next) {
-    GPR_ASSERT(uv_listen((uv_stream_t *)sp->handle, SOMAXCONN, on_connect) ==
-               0);
+    if (sp->has_pending_connection) {
+      finish_accept(exec_ctx, sp);
+      sp->has_pending_connection = false;
+    }
   }
 }
 
diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c
index 7c6a9b85f5..a05c19b4ac 100644
--- a/src/core/lib/iomgr/tcp_uv.c
+++ b/src/core/lib/iomgr/tcp_uv.c
@@ -30,6 +30,7 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/resource_quota.h"
 #include "src/core/lib/iomgr/tcp_uv.h"
@@ -183,6 +184,7 @@ static void uv_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   grpc_tcp *tcp = (grpc_tcp *)ep;
   int status;
   grpc_error *error = GRPC_ERROR_NONE;
+  GRPC_UV_ASSERT_SAME_THREAD();
   GPR_ASSERT(tcp->read_cb == NULL);
   tcp->read_cb = cb;
   tcp->read_slices = read_slices;
@@ -236,6 +238,7 @@ static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   unsigned int i;
   grpc_slice *slice;
   uv_write_t *write_req;
+  GRPC_UV_ASSERT_SAME_THREAD();
 
   if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     size_t j;
@@ -307,6 +310,10 @@ static void uv_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                                  grpc_error *why) {
   grpc_tcp *tcp = (grpc_tcp *)ep;
   if (!tcp->shutting_down) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+      const char *str = grpc_error_string(why);
+      gpr_log(GPR_DEBUG, "TCP %p shutdown why=%s", tcp->handle, str);
+    }
     tcp->shutting_down = true;
     uv_shutdown_t *req = &tcp->shutdown_req;
     uv_shutdown(req, (uv_stream_t *)tcp->handle, shutdown_callback);
diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c
index 1ab82ef1d5..70f49bcbe8 100644
--- a/src/core/lib/iomgr/timer_uv.c
+++ b/src/core/lib/iomgr/timer_uv.c
@@ -24,6 +24,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/timer.h"
 
 #include <uv.h>
@@ -43,6 +44,7 @@ static void stop_uv_timer(uv_timer_t *handle) {
 void run_expired_timer(uv_timer_t *handle) {
   grpc_timer *timer = (grpc_timer *)handle->data;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GRPC_UV_ASSERT_SAME_THREAD();
   GPR_ASSERT(timer->pending);
   timer->pending = 0;
   GRPC_CLOSURE_SCHED(&exec_ctx, timer->closure, GRPC_ERROR_NONE);
@@ -55,6 +57,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
                      gpr_timespec now) {
   uint64_t timeout;
   uv_timer_t *uv_timer;
+  GRPC_UV_ASSERT_SAME_THREAD();
   timer->closure = closure;
   if (gpr_time_cmp(deadline, now) <= 0) {
     timer->pending = 0;
@@ -75,6 +78,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 }
 
 void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   if (timer->pending) {
     timer->pending = 0;
     GRPC_CLOSURE_SCHED(exec_ctx, timer->closure, GRPC_ERROR_CANCELLED);
diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c
index 54e7f417a7..88fa34cb7a 100644
--- a/src/core/lib/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@@ -214,7 +214,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
                       sp->server->user_data);
       }
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
-                     "udp_listener_shutdown");
+                     false /* already_closed */, "udp_listener_shutdown");
     }
     gpr_mu_unlock(&s->mu);
   } else {
diff --git a/src/core/lib/iomgr/wakeup_fd_cv.h b/src/core/lib/iomgr/wakeup_fd_cv.h
index c5dcdc9746..46e84f5843 100644
--- a/src/core/lib/iomgr/wakeup_fd_cv.h
+++ b/src/core/lib/iomgr/wakeup_fd_cv.h
@@ -43,6 +43,7 @@
 typedef struct cv_node {
   gpr_cv* cv;
   struct cv_node* next;
+  struct cv_node* prev;
 } cv_node;
 
 typedef struct fd_node {
@@ -53,9 +54,8 @@ typedef struct fd_node {
 
 typedef struct cv_fd_table {
   gpr_mu mu;
-  int pollcount;
-  int shutdown;
-  gpr_cv shutdown_complete;
+  gpr_refcount pollcount;
+  gpr_cv shutdown_cv;
   fd_node* cvfds;
   fd_node* free_fds;
   unsigned int size;
diff --git a/src/core/lib/json/json_reader.c b/src/core/lib/json/json_reader.c
index 75f59e0912..094a35176c 100644
--- a/src/core/lib/json/json_reader.c
+++ b/src/core/lib/json/json_reader.c
@@ -178,6 +178,7 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
             json_reader_string_clear(reader);
             reader->state = GRPC_JSON_STATE_VALUE_END;
           /* The missing break here is intentional. */
+          /* fallthrough */
 
           case GRPC_JSON_STATE_VALUE_END:
           case GRPC_JSON_STATE_OBJECT_KEY_BEGIN:
diff --git a/src/core/lib/profiling/timers.h b/src/core/lib/profiling/timers.h
index 58e6659e6d..7f02b4bf84 100644
--- a/src/core/lib/profiling/timers.h
+++ b/src/core/lib/profiling/timers.h
@@ -37,7 +37,8 @@ void gpr_timers_set_log_filename(const char *filename);
 
 void gpr_timer_set_enabled(int enabled);
 
-#if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER))
+#if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER) + \
+      defined(GRPC_CUSTOM_PROFILER))
 /* No profiling. No-op all the things. */
 #define GPR_TIMER_MARK(tag, important) \
   do {                                 \
@@ -56,6 +57,12 @@ void gpr_timer_set_enabled(int enabled);
 #if defined(GRPC_STAP_PROFILER) && defined(GRPC_BASIC_PROFILER)
 #error "GRPC_STAP_PROFILER and GRPC_BASIC_PROFILER are mutually exclusive."
 #endif
+#if defined(GRPC_STAP_PROFILER) && defined(GRPC_CUSTOM_PROFILER)
+#error "GRPC_STAP_PROFILER and GRPC_CUSTOM_PROFILER are mutually exclusive."
+#endif
+#if defined(GRPC_CUSTOM_PROFILER) && defined(GRPC_BASIC_PROFILER)
+#error "GRPC_CUSTOM_PROFILER and GRPC_BASIC_PROFILER are mutually exclusive."
+#endif
 
 /* Generic profiling interface. */
 #define GPR_TIMER_MARK(tag, important) \
@@ -80,22 +87,25 @@ void gpr_timer_set_enabled(int enabled);
 #ifdef __cplusplus
 }
 
-#if (defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER))
+#if (defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER) + \
+     defined(GRPC_CUSTOM_PROFILER))
 namespace grpc {
 class ProfileScope {
  public:
-  ProfileScope(const char *desc, bool important) : desc_(desc) {
-    GPR_TIMER_BEGIN(desc_, important ? 1 : 0);
+  ProfileScope(const char *desc, bool important, const char *file, int line)
+      : desc_(desc) {
+    gpr_timer_begin(desc_, important ? 1 : 0, file, line);
   }
-  ~ProfileScope() { GPR_TIMER_END(desc_, 0); }
+  ~ProfileScope() { gpr_timer_end(desc_, 0, "n/a", 0); }
 
  private:
   const char *const desc_;
 };
-}
+}  // namespace grpc
 
-#define GPR_TIMER_SCOPE(tag, important) \
-  ::grpc::ProfileScope _profile_scope_##__LINE__((tag), (important))
+#define GPR_TIMER_SCOPE(tag, important)                                        \
+  ::grpc::ProfileScope _profile_scope_##__LINE__((tag), (important), __FILE__, \
+                                                 __LINE__)
 #else
 #define GPR_TIMER_SCOPE(tag, important) \
   do {                                  \
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
index 77d7b04627..09fd60a12c 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.c
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -32,88 +32,98 @@
 typedef struct {
   grpc_composite_call_credentials *composite_creds;
   size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
   grpc_polling_entity *pollent;
-  grpc_credentials_metadata_cb cb;
+  grpc_auth_metadata_context auth_md_context;
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  grpc_closure internal_on_request_metadata;
 } grpc_composite_call_credentials_metadata_context;
 
 static void composite_call_destruct(grpc_exec_ctx *exec_ctx,
                                     grpc_call_credentials *creds) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
+  for (size_t i = 0; i < c->inner.num_creds; i++) {
     grpc_call_credentials_unref(exec_ctx, c->inner.creds_array[i]);
   }
   gpr_free(c->inner.creds_array);
 }
 
-static void composite_call_md_context_destroy(
-    grpc_exec_ctx *exec_ctx,
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(exec_ctx, ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status,
-                                       const char *error_details) {
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *arg,
+                                       grpc_error *error) {
   grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status, error_details);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
+      (grpc_composite_call_credentials_metadata_context *)arg;
+  if (error == GRPC_ERROR_NONE) {
+    /* See if we need to get some more metadata. */
+    if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+      grpc_call_credentials *inner_creds =
+          ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+      if (grpc_call_credentials_get_request_metadata(
+              exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+              ctx->md_array, &ctx->internal_on_request_metadata, &error)) {
+        // Synchronous response, so call ourselves recursively.
+        composite_call_metadata_cb(exec_ctx, arg, error);
+        GRPC_ERROR_UNREF(error);
+      }
+      return;
     }
+    // We're done!
   }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(
-        exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
-        composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK, NULL);
-  composite_call_md_context_destroy(exec_ctx, ctx);
+  GRPC_CLOSURE_SCHED(exec_ctx, ctx->on_request_metadata, GRPC_ERROR_REF(error));
+  gpr_free(ctx);
 }
 
-static void composite_call_get_request_metadata(
+static bool composite_call_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context auth_md_context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
   grpc_composite_call_credentials_metadata_context *ctx;
-
   ctx = gpr_zalloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
   ctx->composite_creds = c;
   ctx->pollent = pollent;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pollent,
-      auth_md_context, composite_call_metadata_cb, ctx);
+  ctx->auth_md_context = auth_md_context;
+  ctx->md_array = md_array;
+  ctx->on_request_metadata = on_request_metadata;
+  GRPC_CLOSURE_INIT(&ctx->internal_on_request_metadata,
+                    composite_call_metadata_cb, ctx, grpc_schedule_on_exec_ctx);
+  while (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    if (grpc_call_credentials_get_request_metadata(
+            exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+            ctx->md_array, &ctx->internal_on_request_metadata, error)) {
+      if (*error != GRPC_ERROR_NONE) break;
+    } else {
+      break;
+    }
+  }
+  // If we got through all creds synchronously or we got a synchronous
+  // error on one of them, return synchronously.
+  if (ctx->creds_index == ctx->composite_creds->inner.num_creds ||
+      *error != GRPC_ERROR_NONE) {
+    gpr_free(ctx);
+    return true;
+  }
+  // At least one inner cred is returning asynchronously, so we'll
+  // return asynchronously as well.
+  return false;
+}
+
+static void composite_call_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  for (size_t i = 0; i < c->inner.num_creds; ++i) {
+    grpc_call_credentials_cancel_get_request_metadata(
+        exec_ctx, c->inner.creds_array[i], md_array, GRPC_ERROR_REF(error));
+  }
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
+    composite_call_destruct, composite_call_get_request_metadata,
+    composite_call_cancel_get_request_metadata};
 
 static grpc_call_credentials_array get_creds_array(
     grpc_call_credentials **creds_addr) {
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
index b1f1e82076..8a67c9865b 100644
--- a/src/core/lib/security/credentials/credentials.c
+++ b/src/core/lib/security/credentials/credentials.c
@@ -38,13 +38,10 @@
 /* -- Common. -- */
 
 grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
-    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
-    void *user_data) {
+    grpc_call_credentials *creds) {
   grpc_credentials_metadata_request *r =
       gpr_zalloc(sizeof(grpc_credentials_metadata_request));
   r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
   return r;
 }
 
@@ -104,18 +101,25 @@ void grpc_call_credentials_release(grpc_call_credentials *creds) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-void grpc_call_credentials_get_request_metadata(
+bool grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
-    }
+    return true;
+  }
+  return creds->vtable->get_request_metadata(
+      exec_ctx, creds, pollent, context, md_array, on_request_metadata, error);
+}
+
+void grpc_call_credentials_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  if (creds == NULL || creds->vtable->cancel_get_request_metadata == NULL) {
     return;
   }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pollent, context, cb,
-                                      user_data);
+  creds->vtable->cancel_get_request_metadata(exec_ctx, creds, md_array, error);
 }
 
 grpc_security_status grpc_channel_credentials_create_security_connector(
diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h
index c45c2e957c..04a54b0ca8 100644
--- a/src/core/lib/security/credentials/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -138,48 +138,39 @@ grpc_channel_credentials *grpc_channel_credentials_from_arg(
 grpc_channel_credentials *grpc_channel_credentials_find_in_args(
     const grpc_channel_args *args);
 
-/* --- grpc_credentials_md. --- */
+/* --- grpc_credentials_mdelem_array. --- */
 
 typedef struct {
-  grpc_slice key;
-  grpc_slice value;
-} grpc_credentials_md;
+  grpc_mdelem *md;
+  size_t size;
+} grpc_credentials_mdelem_array;
 
-typedef struct {
-  grpc_credentials_md *entries;
-  size_t num_entries;
-  size_t allocated;
-  gpr_refcount refcount;
-} grpc_credentials_md_store;
+/// Takes a new ref to \a md.
+void grpc_credentials_mdelem_array_add(grpc_credentials_mdelem_array *list,
+                                       grpc_mdelem md);
 
-grpc_credentials_md_store *grpc_credentials_md_store_create(
-    size_t initial_capacity);
+/// Appends all elements from \a src to \a dst, taking a new ref to each one.
+void grpc_credentials_mdelem_array_append(grpc_credentials_mdelem_array *dst,
+                                          grpc_credentials_mdelem_array *src);
 
-/* Will ref key and value. */
-void grpc_credentials_md_store_add(grpc_credentials_md_store *store,
-                                   grpc_slice key, grpc_slice value);
-void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store,
-                                            const char *key, const char *value);
-grpc_credentials_md_store *grpc_credentials_md_store_ref(
-    grpc_credentials_md_store *store);
-void grpc_credentials_md_store_unref(grpc_exec_ctx *exec_ctx,
-                                     grpc_credentials_md_store *store);
+void grpc_credentials_mdelem_array_destroy(grpc_exec_ctx *exec_ctx,
+                                           grpc_credentials_mdelem_array *list);
 
 /* --- grpc_call_credentials. --- */
 
-/* error_details must be NULL if status is GRPC_CREDENTIALS_OK. */
-typedef void (*grpc_credentials_metadata_cb)(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details);
-
 typedef struct {
   void (*destruct)(grpc_exec_ctx *exec_ctx, grpc_call_credentials *c);
-  void (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
+  bool (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
                                grpc_call_credentials *c,
                                grpc_polling_entity *pollent,
                                grpc_auth_metadata_context context,
-                               grpc_credentials_metadata_cb cb,
-                               void *user_data);
+                               grpc_credentials_mdelem_array *md_array,
+                               grpc_closure *on_request_metadata,
+                               grpc_error **error);
+  void (*cancel_get_request_metadata)(grpc_exec_ctx *exec_ctx,
+                                      grpc_call_credentials *c,
+                                      grpc_credentials_mdelem_array *md_array,
+                                      grpc_error *error);
 } grpc_call_credentials_vtable;
 
 struct grpc_call_credentials {
@@ -191,15 +182,29 @@ struct grpc_call_credentials {
 grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds);
 void grpc_call_credentials_unref(grpc_exec_ctx *exec_ctx,
                                  grpc_call_credentials *creds);
-void grpc_call_credentials_get_request_metadata(
+
+/// Returns true if completed synchronously, in which case \a error will
+/// be set to indicate the result.  Otherwise, \a on_request_metadata will
+/// be invoked asynchronously when complete.  \a md_array will be populated
+/// with the resulting metadata once complete.
+bool grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data);
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error);
+
+/// Cancels a pending asynchronous operation started by
+/// grpc_call_credentials_get_request_metadata() with the corresponding
+/// value of \a md_array.
+void grpc_call_credentials_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error);
 
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async);
+    grpc_exec_ctx *exec_ctx, const char *md_key, const char *md_value,
+    bool is_async);
 
 /* --- grpc_server_credentials. --- */
 
@@ -238,14 +243,11 @@ grpc_server_credentials *grpc_find_server_credentials_in_args(
 
 typedef struct {
   grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
   grpc_http_response response;
-  void *user_data;
 } grpc_credentials_metadata_request;
 
 grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
-    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
-    void *user_data);
+    grpc_call_credentials *creds);
 
 void grpc_credentials_metadata_request_destroy(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *r);
diff --git a/src/core/lib/security/credentials/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
index fcfdd52d20..ccd39e610f 100644
--- a/src/core/lib/security/credentials/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -24,65 +24,36 @@
 
 #include "src/core/lib/slice/slice_internal.h"
 
-static void store_ensure_capacity(grpc_credentials_md_store *store) {
-  if (store->num_entries == store->allocated) {
-    store->allocated = (store->allocated == 0) ? 1 : store->allocated * 2;
-    store->entries = gpr_realloc(
-        store->entries, store->allocated * sizeof(grpc_credentials_md));
+static void mdelem_list_ensure_capacity(grpc_credentials_mdelem_array *list,
+                                        size_t additional_space_needed) {
+  size_t target_size = list->size + additional_space_needed;
+  // Find the next power of two greater than the target size (i.e.,
+  // whenever we add more space, we double what we already have).
+  size_t new_size = 2;
+  while (new_size < target_size) {
+    new_size *= 2;
   }
+  list->md = gpr_realloc(list->md, sizeof(grpc_mdelem) * new_size);
 }
 
-grpc_credentials_md_store *grpc_credentials_md_store_create(
-    size_t initial_capacity) {
-  grpc_credentials_md_store *store =
-      gpr_zalloc(sizeof(grpc_credentials_md_store));
-  if (initial_capacity > 0) {
-    store->entries = gpr_malloc(initial_capacity * sizeof(grpc_credentials_md));
-    store->allocated = initial_capacity;
-  }
-  gpr_ref_init(&store->refcount, 1);
-  return store;
-}
-
-void grpc_credentials_md_store_add(grpc_credentials_md_store *store,
-                                   grpc_slice key, grpc_slice value) {
-  if (store == NULL) return;
-  store_ensure_capacity(store);
-  store->entries[store->num_entries].key = grpc_slice_ref_internal(key);
-  store->entries[store->num_entries].value = grpc_slice_ref_internal(value);
-  store->num_entries++;
+void grpc_credentials_mdelem_array_add(grpc_credentials_mdelem_array *list,
+                                       grpc_mdelem md) {
+  mdelem_list_ensure_capacity(list, 1);
+  list->md[list->size++] = GRPC_MDELEM_REF(md);
 }
 
-void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store,
-                                            const char *key,
-                                            const char *value) {
-  if (store == NULL) return;
-  store_ensure_capacity(store);
-  store->entries[store->num_entries].key = grpc_slice_from_copied_string(key);
-  store->entries[store->num_entries].value =
-      grpc_slice_from_copied_string(value);
-  store->num_entries++;
-}
-
-grpc_credentials_md_store *grpc_credentials_md_store_ref(
-    grpc_credentials_md_store *store) {
-  if (store == NULL) return NULL;
-  gpr_ref(&store->refcount);
-  return store;
+void grpc_credentials_mdelem_array_append(grpc_credentials_mdelem_array *dst,
+                                          grpc_credentials_mdelem_array *src) {
+  mdelem_list_ensure_capacity(dst, src->size);
+  for (size_t i = 0; i < src->size; ++i) {
+    dst->md[dst->size++] = GRPC_MDELEM_REF(src->md[i]);
+  }
 }
 
-void grpc_credentials_md_store_unref(grpc_exec_ctx *exec_ctx,
-                                     grpc_credentials_md_store *store) {
-  if (store == NULL) return;
-  if (gpr_unref(&store->refcount)) {
-    if (store->entries != NULL) {
-      size_t i;
-      for (i = 0; i < store->num_entries; i++) {
-        grpc_slice_unref_internal(exec_ctx, store->entries[i].key);
-        grpc_slice_unref_internal(exec_ctx, store->entries[i].value);
-      }
-      gpr_free(store->entries);
-    }
-    gpr_free(store);
+void grpc_credentials_mdelem_array_destroy(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_mdelem_array *list) {
+  for (size_t i = 0; i < list->size; ++i) {
+    GRPC_MDELEM_UNREF(exec_ctx, list->md[i]);
   }
+  gpr_free(list->md);
 }
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
index 67e74f7b92..ac9017850f 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.c
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -98,49 +98,44 @@ const char *grpc_fake_transport_get_expected_targets(
 static void md_only_test_destruct(grpc_exec_ctx *exec_ctx,
                                   grpc_call_credentials *creds) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->md_store);
+  GRPC_MDELEM_UNREF(exec_ctx, c->md);
 }
 
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, grpc_error *error) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK, NULL);
-  grpc_credentials_metadata_request_destroy(exec_ctx, r);
-}
-
-static void md_only_test_get_request_metadata(
+static bool md_only_test_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
+  grpc_credentials_mdelem_array_add(md_array, c->md);
   if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    GRPC_CLOSURE_SCHED(exec_ctx,
-                       GRPC_CLOSURE_CREATE(on_simulated_token_fetch_done,
-                                           cb_arg, grpc_executor_scheduler),
-                       GRPC_ERROR_NONE);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK, NULL);
+    GRPC_CLOSURE_SCHED(exec_ctx, on_request_metadata, GRPC_ERROR_NONE);
+    return false;
   }
+  return true;
+}
+
+static void md_only_test_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
+    md_only_test_destruct, md_only_test_get_request_metadata,
+    md_only_test_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
+    grpc_exec_ctx *exec_ctx, const char *md_key, const char *md_value,
+    bool is_async) {
   grpc_md_only_test_credentials *c =
       gpr_zalloc(sizeof(grpc_md_only_test_credentials));
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
   c->base.vtable = &md_only_test_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->md =
+      grpc_mdelem_from_slices(exec_ctx, grpc_slice_from_copied_string(md_key),
+                              grpc_slice_from_copied_string(md_value));
   c->is_async = is_async;
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
index fae7a6a9c4..aa0f3b6e20 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.h
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -52,8 +52,8 @@ const char *grpc_fake_transport_get_expected_targets(
 
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
+  grpc_mdelem md;
+  bool is_async;
 } grpc_md_only_test_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
index 4b32c5a047..3de8319d98 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.c
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -30,26 +30,33 @@
 static void iam_destruct(grpc_exec_ctx *exec_ctx,
                          grpc_call_credentials *creds) {
   grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->iam_md);
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &c->md_array);
 }
 
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
                                      grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
+                                     grpc_credentials_mdelem_array *md_array,
+                                     grpc_closure *on_request_metadata,
+                                     grpc_error **error) {
   grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK, NULL);
+  grpc_credentials_mdelem_array_append(md_array, &c->md_array);
+  return true;
 }
 
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
+static void iam_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {
+    iam_destruct, iam_get_request_metadata, iam_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_google_iam_credentials_create(
     const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   GRPC_API_TRACE(
       "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
       "reserved=%p)",
@@ -57,14 +64,22 @@ grpc_call_credentials *grpc_google_iam_credentials_create(
   GPR_ASSERT(reserved == NULL);
   GPR_ASSERT(token != NULL);
   GPR_ASSERT(authority_selector != NULL);
-  c = gpr_zalloc(sizeof(grpc_google_iam_credentials));
+  grpc_google_iam_credentials *c = gpr_zalloc(sizeof(*c));
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
   c->base.vtable = &iam_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  grpc_mdelem md = grpc_mdelem_from_slices(
+      &exec_ctx,
+      grpc_slice_from_static_string(GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY),
+      grpc_slice_from_copied_string(token));
+  grpc_credentials_mdelem_array_add(&c->md_array, md);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  md = grpc_mdelem_from_slices(
+      &exec_ctx,
+      grpc_slice_from_static_string(GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY),
+      grpc_slice_from_copied_string(authority_selector));
+  grpc_credentials_mdelem_array_add(&c->md_array, md);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  grpc_exec_ctx_finish(&exec_ctx);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h
index fff3c6d2ca..5e3cf65bad 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.h
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -23,7 +23,7 @@
 
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
+  grpc_credentials_mdelem_array md_array;
 } grpc_google_iam_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
index 4357657def..02c82e99ba 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.c
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -29,10 +29,8 @@
 
 static void jwt_reset_cache(grpc_exec_ctx *exec_ctx,
                             grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(exec_ctx, c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
+  GRPC_MDELEM_UNREF(exec_ctx, c->cached.jwt_md);
+  c->cached.jwt_md = GRPC_MDNULL;
   if (c->cached.service_url != NULL) {
     gpr_free(c->cached.service_url);
     c->cached.service_url = NULL;
@@ -49,33 +47,34 @@ static void jwt_destruct(grpc_exec_ctx *exec_ctx,
   gpr_mu_destroy(&c->cache_mu);
 }
 
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
                                      grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
+                                     grpc_credentials_mdelem_array *md_array,
+                                     grpc_closure *on_request_metadata,
+                                     grpc_error **error) {
   grpc_service_account_jwt_access_credentials *c =
       (grpc_service_account_jwt_access_credentials *)creds;
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
 
   /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
+  grpc_mdelem jwt_md = GRPC_MDNULL;
   {
     gpr_mu_lock(&c->cache_mu);
     if (c->cached.service_url != NULL &&
         strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
+        !GRPC_MDISNULL(c->cached.jwt_md) &&
         (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
                                    gpr_now(GPR_CLOCK_REALTIME)),
                       refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+      jwt_md = GRPC_MDELEM_REF(c->cached.jwt_md);
     }
     gpr_mu_unlock(&c->cache_mu);
   }
 
-  if (jwt_md == NULL) {
+  if (GRPC_MDISNULL(jwt_md)) {
     char *jwt = NULL;
     /* Generate a new jwt. */
     gpr_mu_lock(&c->cache_mu);
@@ -89,27 +88,33 @@ static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
       c->cached.jwt_expiration =
           gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
       c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      c->cached.jwt_md = grpc_mdelem_from_slices(
+          exec_ctx,
+          grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+          grpc_slice_from_copied_string(md_value));
       gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+      jwt_md = GRPC_MDELEM_REF(c->cached.jwt_md);
     }
     gpr_mu_unlock(&c->cache_mu);
   }
 
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK, NULL);
-    grpc_credentials_md_store_unref(exec_ctx, jwt_md);
+  if (!GRPC_MDISNULL(jwt_md)) {
+    grpc_credentials_mdelem_array_add(md_array, jwt_md);
+    GRPC_MDELEM_UNREF(exec_ctx, jwt_md);
   } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-       "Could not generate JWT.");
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Could not generate JWT.");
   }
+  return true;
+}
+
+static void jwt_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
+static grpc_call_credentials_vtable jwt_vtable = {
+    jwt_destruct, jwt_get_request_metadata, jwt_cancel_get_request_metadata};
 
 grpc_call_credentials *
 grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
index 6e461f1715..07f4022669 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.h
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -29,7 +29,7 @@ typedef struct {
   // the service_url for a more sophisticated one.
   gpr_mu cache_mu;
   struct {
-    grpc_credentials_md_store *jwt_md;
+    grpc_mdelem jwt_md;
     char *service_url;
     gpr_timespec jwt_expiration;
   } cached;
diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
index 6cd558d123..a27284bc50 100644
--- a/src/core/lib/security/credentials/jwt/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -442,7 +442,7 @@ static EVP_PKEY *extract_pkey_from_x509(const char *x509_str) {
 
 end:
   BIO_free(bio);
-  if (x509 != NULL) X509_free(x509);
+  X509_free(x509);
   return result;
 }
 
@@ -496,6 +496,8 @@ static EVP_PKEY *pkey_from_jwk(grpc_exec_ctx *exec_ctx, const grpc_json *json,
   const grpc_json *key_prop;
   RSA *rsa = NULL;
   EVP_PKEY *result = NULL;
+  BIGNUM *tmp_n = NULL;
+  BIGNUM *tmp_e = NULL;
 
   GPR_ASSERT(kty != NULL && json != NULL);
   if (strcmp(kty, "RSA") != 0) {
@@ -507,8 +509,6 @@ static EVP_PKEY *pkey_from_jwk(grpc_exec_ctx *exec_ctx, const grpc_json *json,
     gpr_log(GPR_ERROR, "Could not create rsa key.");
     goto end;
   }
-  BIGNUM *tmp_n = NULL;
-  BIGNUM *tmp_e = NULL;
   for (key_prop = json->child; key_prop != NULL; key_prop = key_prop->next) {
     if (strcmp(key_prop->key, "n") == 0) {
       tmp_n =
@@ -528,11 +528,16 @@ static EVP_PKEY *pkey_from_jwk(grpc_exec_ctx *exec_ctx, const grpc_json *json,
     gpr_log(GPR_ERROR, "Cannot set RSA key from inputs.");
     goto end;
   }
+  /* RSA_set0_key takes ownership on success. */
+  tmp_n = NULL;
+  tmp_e = NULL;
   result = EVP_PKEY_new();
   EVP_PKEY_set1_RSA(result, rsa); /* uprefs rsa. */
 
 end:
-  if (rsa != NULL) RSA_free(rsa);
+  RSA_free(rsa);
+  BN_free(tmp_n);
+  BN_free(tmp_e);
   return result;
 }
 
@@ -618,7 +623,7 @@ static int verify_jwt_signature(EVP_PKEY *key, const char *alg,
   result = 1;
 
 end:
-  if (md_ctx != NULL) EVP_MD_CTX_destroy(md_ctx);
+  EVP_MD_CTX_destroy(md_ctx);
   return result;
 }
 
@@ -658,7 +663,7 @@ static void on_keys_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
 
 end:
   if (json != NULL) grpc_json_destroy(json);
-  if (verification_key != NULL) EVP_PKEY_free(verification_key);
+  EVP_PKEY_free(verification_key);
   ctx->user_cb(exec_ctx, ctx->user_data, status, claims);
   verifier_cb_ctx_destroy(exec_ctx, ctx);
 }
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
index 9de561b310..10b270c49c 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -107,15 +107,17 @@ static void oauth2_token_fetcher_destruct(grpc_exec_ctx *exec_ctx,
                                           grpc_call_credentials *creds) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->access_token_md);
+  GRPC_MDELEM_UNREF(exec_ctx, c->access_token_md);
   gpr_mu_destroy(&c->mu);
+  grpc_pollset_set_destroy(exec_ctx,
+                           grpc_polling_entity_pollset_set(&c->pollent));
   grpc_httpcli_context_destroy(exec_ctx, &c->httpcli_context);
 }
 
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_exec_ctx *exec_ctx, const grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime) {
+    grpc_mdelem *token_md, gpr_timespec *token_lifetime) {
   char *null_terminated_body = NULL;
   char *new_access_token = NULL;
   grpc_credentials_status status = GRPC_CREDENTIALS_OK;
@@ -184,17 +186,18 @@ grpc_oauth2_token_fetcher_credentials_parse_server_response(
     token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
     token_lifetime->tv_nsec = 0;
     token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(exec_ctx, *token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    if (!GRPC_MDISNULL(*token_md)) GRPC_MDELEM_UNREF(exec_ctx, *token_md);
+    *token_md = grpc_mdelem_from_slices(
+        exec_ctx,
+        grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+        grpc_slice_from_copied_string(new_access_token));
     status = GRPC_CREDENTIALS_OK;
   }
 
 end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(exec_ctx, *token_md);
-    *token_md = NULL;
+  if (status != GRPC_CREDENTIALS_OK && !GRPC_MDISNULL(*token_md)) {
+    GRPC_MDELEM_UNREF(exec_ctx, *token_md);
+    *token_md = GRPC_MDNULL;
   }
   if (null_terminated_body != NULL) gpr_free(null_terminated_body);
   if (new_access_token != NULL) gpr_free(new_access_token);
@@ -205,63 +208,130 @@ end:
 static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx,
                                                   void *user_data,
                                                   grpc_error *error) {
+  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
   grpc_credentials_metadata_request *r =
       (grpc_credentials_metadata_request *)user_data;
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  grpc_mdelem access_token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
-
+  grpc_credentials_status status =
+      grpc_oauth2_token_fetcher_credentials_parse_server_response(
+          exec_ctx, &r->response, &access_token_md, &token_lifetime);
+  // Update cache and grab list of pending requests.
   gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      exec_ctx, &r->response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status,
-          "Error occured when fetching oauth2 token.");
-  }
+  c->token_fetch_pending = false;
+  c->access_token_md = GRPC_MDELEM_REF(access_token_md);
+  c->token_expiration =
+      status == GRPC_CREDENTIALS_OK
+          ? gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime)
+          : gpr_inf_past(GPR_CLOCK_REALTIME);
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      c->pending_requests;
+  c->pending_requests = NULL;
   gpr_mu_unlock(&c->mu);
+  // Invoke callbacks for all pending requests.
+  while (pending_request != NULL) {
+    if (status == GRPC_CREDENTIALS_OK) {
+      grpc_credentials_mdelem_array_add(pending_request->md_array,
+                                        access_token_md);
+    } else {
+      error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+          "Error occured when fetching oauth2 token.", &error, 1);
+    }
+    GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata, error);
+    grpc_polling_entity_del_from_pollset_set(
+        exec_ctx, pending_request->pollent,
+        grpc_polling_entity_pollset_set(&c->pollent));
+    grpc_oauth2_pending_get_request_metadata *prev = pending_request;
+    pending_request = pending_request->next;
+    gpr_free(prev);
+  }
+  GRPC_MDELEM_UNREF(exec_ctx, access_token_md);
+  grpc_call_credentials_unref(exec_ctx, r->creds);
   grpc_credentials_metadata_request_destroy(exec_ctx, r);
 }
 
-static void oauth2_token_fetcher_get_request_metadata(
+static bool oauth2_token_fetcher_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
+  // Check if we can use the cached token.
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
+  grpc_mdelem cached_access_token_md = GRPC_MDNULL;
+  gpr_mu_lock(&c->mu);
+  if (!GRPC_MDISNULL(c->access_token_md) &&
+      (gpr_time_cmp(
+           gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+           refresh_threshold) > 0)) {
+    cached_access_token_md = GRPC_MDELEM_REF(c->access_token_md);
+  }
+  if (!GRPC_MDISNULL(cached_access_token_md)) {
     gpr_mu_unlock(&c->mu);
+    grpc_credentials_mdelem_array_add(md_array, cached_access_token_md);
+    GRPC_MDELEM_UNREF(exec_ctx, cached_access_token_md);
+    return true;
   }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
-    grpc_credentials_md_store_unref(exec_ctx, cached_access_token_md);
-  } else {
+  // Couldn't get the token from the cache.
+  // Add request to c->pending_requests and start a new fetch if needed.
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      (grpc_oauth2_pending_get_request_metadata *)gpr_malloc(
+          sizeof(*pending_request));
+  pending_request->md_array = md_array;
+  pending_request->on_request_metadata = on_request_metadata;
+  pending_request->pollent = pollent;
+  grpc_polling_entity_add_to_pollset_set(
+      exec_ctx, pollent, grpc_polling_entity_pollset_set(&c->pollent));
+  pending_request->next = c->pending_requests;
+  c->pending_requests = pending_request;
+  bool start_fetch = false;
+  if (!c->token_fetch_pending) {
+    c->token_fetch_pending = true;
+    start_fetch = true;
+  }
+  gpr_mu_unlock(&c->mu);
+  if (start_fetch) {
+    grpc_call_credentials_ref(creds);
     c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pollent, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+        exec_ctx, grpc_credentials_metadata_request_create(creds),
+        &c->httpcli_context, &c->pollent, on_oauth2_token_fetcher_http_response,
+        gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), refresh_threshold));
   }
+  return false;
+}
+
+static void oauth2_token_fetcher_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_mu_lock(&c->mu);
+  grpc_oauth2_pending_get_request_metadata *prev = NULL;
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      c->pending_requests;
+  while (pending_request != NULL) {
+    if (pending_request->md_array == md_array) {
+      // Remove matching pending request from the list.
+      if (prev != NULL) {
+        prev->next = pending_request->next;
+      } else {
+        c->pending_requests = pending_request->next;
+      }
+      // Invoke the callback immediately with an error.
+      GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
+                         GRPC_ERROR_REF(error));
+      gpr_free(pending_request);
+      break;
+    }
+    prev = pending_request;
+    pending_request = pending_request->next;
+  }
+  gpr_mu_unlock(&c->mu);
+  GRPC_ERROR_UNREF(error);
 }
 
 static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
@@ -272,6 +342,8 @@ static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
   gpr_mu_init(&c->mu);
   c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
   c->fetch_func = fetch_func;
+  c->pollent =
+      grpc_polling_entity_create_from_pollset_set(grpc_pollset_set_create());
   grpc_httpcli_context_init(&c->httpcli_context);
 }
 
@@ -280,7 +352,8 @@ static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
 //
 
 static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata,
+    oauth2_token_fetcher_cancel_get_request_metadata};
 
 static void compute_engine_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
@@ -301,7 +374,6 @@ static void compute_engine_fetch_oauth2(
   grpc_httpcli_get(
       exec_ctx, httpcli_context, pollent, resource_quota, &request, deadline,
       GRPC_CLOSURE_CREATE(response_cb, metadata_req, grpc_schedule_on_exec_ctx),
-
       &metadata_req->response);
   grpc_resource_quota_unref_internal(exec_ctx, resource_quota);
 }
@@ -331,7 +403,8 @@ static void refresh_token_destruct(grpc_exec_ctx *exec_ctx,
 }
 
 static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata,
+    oauth2_token_fetcher_cancel_get_request_metadata};
 
 static void refresh_token_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
@@ -416,26 +489,33 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create(
 static void access_token_destruct(grpc_exec_ctx *exec_ctx,
                                   grpc_call_credentials *creds) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->access_token_md);
+  GRPC_MDELEM_UNREF(exec_ctx, c->access_token_md);
 }
 
-static void access_token_get_request_metadata(
+static bool access_token_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK,
-     NULL);
+  grpc_credentials_mdelem_array_add(md_array, c->access_token_md);
+  return true;
+}
+
+static void access_token_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
+    access_token_destruct, access_token_get_request_metadata,
+    access_token_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_access_token_credentials_create(
     const char *access_token, void *reserved) {
   grpc_access_token_credentials *c =
       gpr_zalloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
   GRPC_API_TRACE(
       "grpc_access_token_credentials_create(access_token=<redacted>, "
       "reserved=%p)",
@@ -444,10 +524,13 @@ grpc_call_credentials *grpc_access_token_credentials_create(
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
   c->base.vtable = &access_token_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
+  char *token_md_value;
   gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  c->access_token_md = grpc_mdelem_from_slices(
+      &exec_ctx, grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+      grpc_slice_from_copied_string(token_md_value));
+  grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(token_md_value);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
index 72093afe9e..d9ad6691b8 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -58,13 +58,24 @@ typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
                                        grpc_polling_entity *pollent,
                                        grpc_iomgr_cb_func cb,
                                        gpr_timespec deadline);
+
+typedef struct grpc_oauth2_pending_get_request_metadata {
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  grpc_polling_entity *pollent;
+  struct grpc_oauth2_pending_get_request_metadata *next;
+} grpc_oauth2_pending_get_request_metadata;
+
 typedef struct {
   grpc_call_credentials base;
   gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
+  grpc_mdelem access_token_md;
   gpr_timespec token_expiration;
+  bool token_fetch_pending;
+  grpc_oauth2_pending_get_request_metadata *pending_requests;
   grpc_httpcli_context httpcli_context;
   grpc_fetch_oauth2_func fetch_func;
+  grpc_polling_entity pollent;
 } grpc_oauth2_token_fetcher_credentials;
 
 // Google refresh token credentials.
@@ -76,7 +87,7 @@ typedef struct {
 // Access token credentials.
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
+  grpc_mdelem access_token_md;
 } grpc_access_token_credentials;
 
 // Private constructor for refresh token credentials from an already parsed
@@ -89,6 +100,6 @@ grpc_refresh_token_credentials_create_from_auth_refresh_token(
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_exec_ctx *exec_ctx, const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+    grpc_mdelem *token_md, gpr_timespec *token_lifetime);
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
index 96ebfb4d0d..73e0c23e0f 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.c
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -31,19 +31,28 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/validate_metadata.h"
 
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
 static void plugin_destruct(grpc_exec_ctx *exec_ctx,
                             grpc_call_credentials *creds) {
   grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  gpr_mu_destroy(&c->mu);
   if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
     c->plugin.destroy(c->plugin.state);
   }
 }
 
+static void pending_request_remove_locked(
+    grpc_plugin_credentials *c,
+    grpc_plugin_credentials_pending_request *pending_request) {
+  if (pending_request->prev == NULL) {
+    c->pending_requests = pending_request->next;
+  } else {
+    pending_request->prev->next = pending_request->next;
+  }
+  if (pending_request->next != NULL) {
+    pending_request->next->prev = pending_request->prev;
+  }
+}
+
 static void plugin_md_request_metadata_ready(void *request,
                                              const grpc_metadata *md,
                                              size_t num_md,
@@ -53,76 +62,117 @@ static void plugin_md_request_metadata_ready(void *request,
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(
       GRPC_EXEC_CTX_FLAG_IS_FINISHED | GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP,
       NULL, NULL);
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-          error_details);
-  } else {
-    size_t i;
-    bool seen_illegal_header = false;
-    grpc_credentials_md *md_array = NULL;
-    for (i = 0; i < num_md; i++) {
-      if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
-                             grpc_validate_header_key_is_legal(md[i].key))) {
-        seen_illegal_header = true;
-        break;
-      } else if (!grpc_is_binary_header(md[i].key) &&
-                 !GRPC_LOG_IF_ERROR(
-                     "validate_metadata_from_plugin",
-                     grpc_validate_header_nonbin_value_is_legal(md[i].value))) {
-        gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
-        seen_illegal_header = true;
-        break;
+  grpc_plugin_credentials_pending_request *r =
+      (grpc_plugin_credentials_pending_request *)request;
+  // Check if the request has been cancelled.
+  // If not, remove it from the pending list, so that it cannot be
+  // cancelled out from under us.
+  gpr_mu_lock(&r->creds->mu);
+  if (!r->cancelled) pending_request_remove_locked(r->creds, r);
+  gpr_mu_unlock(&r->creds->mu);
+  grpc_call_credentials_unref(&exec_ctx, &r->creds->base);
+  // If it has not been cancelled, process it.
+  if (!r->cancelled) {
+    if (status != GRPC_STATUS_OK) {
+      char *msg;
+      gpr_asprintf(&msg, "Getting metadata from plugin failed with error: %s",
+                   error_details);
+      GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata,
+                         GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
+      gpr_free(msg);
+    } else {
+      bool seen_illegal_header = false;
+      for (size_t i = 0; i < num_md; ++i) {
+        if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
+                               grpc_validate_header_key_is_legal(md[i].key))) {
+          seen_illegal_header = true;
+          break;
+        } else if (!grpc_is_binary_header(md[i].key) &&
+                   !GRPC_LOG_IF_ERROR(
+                       "validate_metadata_from_plugin",
+                       grpc_validate_header_nonbin_value_is_legal(
+                           md[i].value))) {
+          gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
+          seen_illegal_header = true;
+          break;
+        }
       }
-    }
-    if (seen_illegal_header) {
-      r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-            "Illegal metadata");
-    } else if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = grpc_slice_ref_internal(md[i].key);
-        md_array[i].value = grpc_slice_ref_internal(md[i].value);
+      if (seen_illegal_header) {
+        GRPC_CLOSURE_SCHED(
+            &exec_ctx, r->on_request_metadata,
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal metadata"));
+      } else {
+        for (size_t i = 0; i < num_md; ++i) {
+          grpc_mdelem mdelem = grpc_mdelem_from_slices(
+              &exec_ctx, grpc_slice_ref_internal(md[i].key),
+              grpc_slice_ref_internal(md[i].value));
+          grpc_credentials_mdelem_array_add(r->md_array, mdelem);
+          GRPC_MDELEM_UNREF(&exec_ctx, mdelem);
+        }
+        GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata, GRPC_ERROR_NONE);
       }
-      r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK,
-            NULL);
-      for (i = 0; i < num_md; i++) {
-        grpc_slice_unref_internal(&exec_ctx, md_array[i].key);
-        grpc_slice_unref_internal(&exec_ctx, md_array[i].value);
-      }
-      gpr_free(md_array);
-    } else if (num_md == 0) {
-      r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
     }
   }
   gpr_free(r);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                         grpc_call_credentials *creds,
                                         grpc_polling_entity *pollent,
                                         grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
+                                        grpc_credentials_mdelem_array *md_array,
+                                        grpc_closure *on_request_metadata,
+                                        grpc_error **error) {
   grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
   if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_zalloc(sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
+    // Create pending_request object.
+    grpc_plugin_credentials_pending_request *pending_request =
+        (grpc_plugin_credentials_pending_request *)gpr_zalloc(
+            sizeof(*pending_request));
+    pending_request->creds = c;
+    pending_request->md_array = md_array;
+    pending_request->on_request_metadata = on_request_metadata;
+    // Add it to the pending list.
+    gpr_mu_lock(&c->mu);
+    if (c->pending_requests != NULL) {
+      c->pending_requests->prev = pending_request;
+    }
+    pending_request->next = c->pending_requests;
+    c->pending_requests = pending_request;
+    gpr_mu_unlock(&c->mu);
+    // Invoke the plugin.  The callback holds a ref to us.
+    grpc_call_credentials_ref(creds);
     c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
+                           plugin_md_request_metadata_ready, pending_request);
+    return false;
+  }
+  return true;
+}
+
+static void plugin_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  gpr_mu_lock(&c->mu);
+  for (grpc_plugin_credentials_pending_request *pending_request =
+           c->pending_requests;
+       pending_request != NULL; pending_request = pending_request->next) {
+    if (pending_request->md_array == md_array) {
+      pending_request->cancelled = true;
+      GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
+                         GRPC_ERROR_REF(error));
+      pending_request_remove_locked(c, pending_request);
+      break;
+    }
   }
+  gpr_mu_unlock(&c->mu);
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
+    plugin_destruct, plugin_get_request_metadata,
+    plugin_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
     grpc_metadata_credentials_plugin plugin, void *reserved) {
@@ -134,5 +184,6 @@ grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
   c->base.vtable = &plugin_vtable;
   gpr_ref_init(&c->base.refcount, 1);
   c->plugin = plugin;
+  gpr_mu_init(&c->mu);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
index ba3dd76859..57266d589a 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.h
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -21,10 +21,22 @@
 
 #include "src/core/lib/security/credentials/credentials.h"
 
-typedef struct {
+struct grpc_plugin_credentials;
+
+typedef struct grpc_plugin_credentials_pending_request {
+  bool cancelled;
+  struct grpc_plugin_credentials *creds;
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  struct grpc_plugin_credentials_pending_request *prev;
+  struct grpc_plugin_credentials_pending_request *next;
+} grpc_plugin_credentials_pending_request;
+
+typedef struct grpc_plugin_credentials {
   grpc_call_credentials base;
   grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
+  gpr_mu mu;
+  grpc_plugin_credentials_pending_request *pending_requests;
 } grpc_plugin_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index 50a51b31cd..e3f0163a6c 100644
--- a/src/core/lib/security/transport/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -39,6 +39,7 @@
 
 /* We can have a per-call credentials. */
 typedef struct {
+  grpc_call_combiner *call_combiner;
   grpc_call_credentials *creds;
   bool have_host;
   bool have_method;
@@ -49,10 +50,11 @@ typedef struct {
      pollset_set so that work can progress when this call wants work to progress
   */
   grpc_polling_entity *pollent;
-  gpr_atm security_context_set;
-  gpr_mu security_context_mu;
+  grpc_credentials_mdelem_array md_array;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
   grpc_auth_metadata_context auth_md_context;
+  grpc_closure async_cancel_closure;
+  grpc_closure async_result_closure;
 } call_data;
 
 /* We can have a per-channel credentials. */
@@ -86,42 +88,32 @@ static void add_error(grpc_error **combined, grpc_error *error) {
   *combined = grpc_error_add_child(*combined, error);
 }
 
-static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data,
-                                    grpc_credentials_md *md_elems,
-                                    size_t num_md,
-                                    grpc_credentials_status status,
-                                    const char *error_details) {
-  grpc_transport_stream_op_batch *batch =
-      (grpc_transport_stream_op_batch *)user_data;
+static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                    grpc_error *input_error) {
+  grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
   grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
+  grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL);
   reset_auth_metadata_context(&calld->auth_md_context);
-  grpc_error *error = GRPC_ERROR_NONE;
-  if (status != GRPC_CREDENTIALS_OK) {
-    error = grpc_error_set_int(
-        GRPC_ERROR_CREATE_FROM_COPIED_STRING(
-            error_details != NULL && strlen(error_details) > 0
-                ? error_details
-                : "Credentials failed to get metadata."),
-        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED);
-  } else {
-    GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT);
+  grpc_error *error = GRPC_ERROR_REF(input_error);
+  if (error == GRPC_ERROR_NONE) {
+    GPR_ASSERT(calld->md_array.size <= MAX_CREDENTIALS_METADATA_COUNT);
     GPR_ASSERT(batch->send_initial_metadata);
     grpc_metadata_batch *mdb =
         batch->payload->send_initial_metadata.send_initial_metadata;
-    for (size_t i = 0; i < num_md; i++) {
-      add_error(&error,
-                grpc_metadata_batch_add_tail(
-                    exec_ctx, mdb, &calld->md_links[i],
-                    grpc_mdelem_from_slices(
-                        exec_ctx, grpc_slice_ref_internal(md_elems[i].key),
-                        grpc_slice_ref_internal(md_elems[i].value))));
+    for (size_t i = 0; i < calld->md_array.size; ++i) {
+      add_error(&error, grpc_metadata_batch_add_tail(
+                            exec_ctx, mdb, &calld->md_links[i],
+                            GRPC_MDELEM_REF(calld->md_array.md[i])));
     }
   }
   if (error == GRPC_ERROR_NONE) {
     grpc_call_next_op(exec_ctx, elem, batch);
   } else {
-    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error);
+    error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS,
+                               GRPC_STATUS_UNAUTHENTICATED);
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error,
+                                                       calld->call_combiner);
   }
 }
 
@@ -155,6 +147,14 @@ void build_auth_metadata_context(grpc_security_connector *sc,
   gpr_free(host);
 }
 
+static void cancel_get_request_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  grpc_call_credentials_cancel_get_request_metadata(
+      exec_ctx, calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
+}
+
 static void send_security_metadata(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem,
                                    grpc_transport_stream_op_batch *batch) {
@@ -183,7 +183,8 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
           grpc_error_set_int(
               GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                   "Incompatible credentials set on channel and call."),
-              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED));
+              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED),
+          calld->call_combiner);
       return;
     }
   } else {
@@ -193,20 +194,34 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
 
   build_auth_metadata_context(&chand->security_connector->base,
                               chand->auth_context, calld);
+
   GPR_ASSERT(calld->pollent != NULL);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
-      on_credentials_metadata, batch);
+
+  GRPC_CLOSURE_INIT(&calld->async_result_closure, on_credentials_metadata,
+                    batch, grpc_schedule_on_exec_ctx);
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (grpc_call_credentials_get_request_metadata(
+          exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
+          &calld->md_array, &calld->async_result_closure, &error)) {
+    // Synchronous return; invoke on_credentials_metadata() directly.
+    on_credentials_metadata(exec_ctx, batch, error);
+    GRPC_ERROR_UNREF(error);
+  } else {
+    // Async return; register cancellation closure with call combiner.
+    GRPC_CLOSURE_INIT(&calld->async_cancel_closure, cancel_get_request_metadata,
+                      elem, grpc_schedule_on_exec_ctx);
+    grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
+                                            &calld->async_cancel_closure);
+  }
 }
 
-static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
-                            grpc_security_status status) {
-  grpc_transport_stream_op_batch *batch =
-      (grpc_transport_stream_op_batch *)user_data;
+static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
   grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
-
-  if (status == GRPC_SECURITY_OK) {
+  grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL);
+  if (error == GRPC_ERROR_NONE) {
     send_security_metadata(exec_ctx, elem, batch);
   } else {
     char *error_msg;
@@ -218,11 +233,22 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
         exec_ctx, batch,
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg),
                            GRPC_ERROR_INT_GRPC_STATUS,
-                           GRPC_STATUS_UNAUTHENTICATED));
+                           GRPC_STATUS_UNAUTHENTICATED),
+        calld->call_combiner);
     gpr_free(error_msg);
   }
 }
 
+static void cancel_check_call_host(grpc_exec_ctx *exec_ctx, void *arg,
+                                   grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  grpc_channel_security_connector_cancel_check_call_host(
+      exec_ctx, chand->security_connector, &calld->async_result_closure,
+      GRPC_ERROR_REF(error));
+}
+
 static void auth_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *batch) {
@@ -233,26 +259,18 @@ static void auth_start_transport_stream_op_batch(
   channel_data *chand = elem->channel_data;
 
   if (!batch->cancel_stream) {
-    /* double checked lock over security context to ensure it's set once */
-    if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
-      gpr_mu_lock(&calld->security_context_mu);
-      if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
-        GPR_ASSERT(batch->payload->context != NULL);
-        if (batch->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) {
-          batch->payload->context[GRPC_CONTEXT_SECURITY].value =
-              grpc_client_security_context_create();
-          batch->payload->context[GRPC_CONTEXT_SECURITY].destroy =
-              grpc_client_security_context_destroy;
-        }
-        grpc_client_security_context *sec_ctx =
-            batch->payload->context[GRPC_CONTEXT_SECURITY].value;
-        GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter");
-        sec_ctx->auth_context =
-            GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter");
-        gpr_atm_rel_store(&calld->security_context_set, 1);
-      }
-      gpr_mu_unlock(&calld->security_context_mu);
+    GPR_ASSERT(batch->payload->context != NULL);
+    if (batch->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) {
+      batch->payload->context[GRPC_CONTEXT_SECURITY].value =
+          grpc_client_security_context_create();
+      batch->payload->context[GRPC_CONTEXT_SECURITY].destroy =
+          grpc_client_security_context_destroy;
     }
+    grpc_client_security_context *sec_ctx =
+        batch->payload->context[GRPC_CONTEXT_SECURITY].value;
+    GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter");
+    sec_ctx->auth_context =
+        GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter");
   }
 
   if (batch->send_initial_metadata) {
@@ -277,11 +295,24 @@ static void auth_start_transport_stream_op_batch(
       }
     }
     if (calld->have_host) {
-      char *call_host = grpc_slice_to_c_string(calld->host);
       batch->handler_private.extra_arg = elem;
-      grpc_channel_security_connector_check_call_host(
-          exec_ctx, chand->security_connector, call_host, chand->auth_context,
-          on_host_checked, batch);
+      GRPC_CLOSURE_INIT(&calld->async_result_closure, on_host_checked, batch,
+                        grpc_schedule_on_exec_ctx);
+      char *call_host = grpc_slice_to_c_string(calld->host);
+      grpc_error *error = GRPC_ERROR_NONE;
+      if (grpc_channel_security_connector_check_call_host(
+              exec_ctx, chand->security_connector, call_host,
+              chand->auth_context, &calld->async_result_closure, &error)) {
+        // Synchronous return; invoke on_host_checked() directly.
+        on_host_checked(exec_ctx, batch, error);
+        GRPC_ERROR_UNREF(error);
+      } else {
+        // Async return; register cancellation closure with call combiner.
+        GRPC_CLOSURE_INIT(&calld->async_cancel_closure, cancel_check_call_host,
+                          elem, grpc_schedule_on_exec_ctx);
+        grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
+                                                &calld->async_cancel_closure);
+      }
       gpr_free(call_host);
       GPR_TIMER_END("auth_start_transport_stream_op_batch", 0);
       return; /* early exit */
@@ -298,8 +329,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
-  memset(calld, 0, sizeof(*calld));
-  gpr_mu_init(&calld->security_context_mu);
+  calld->call_combiner = args->call_combiner;
   return GRPC_ERROR_NONE;
 }
 
@@ -315,6 +345,7 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
                               grpc_closure *ignored) {
   call_data *calld = elem->call_data;
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &calld->md_array);
   grpc_call_credentials_unref(exec_ctx, calld->creds);
   if (calld->have_host) {
     grpc_slice_unref_internal(exec_ctx, calld->host);
@@ -323,7 +354,6 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_slice_unref_internal(exec_ctx, calld->method);
   }
   reset_auth_metadata_context(&calld->auth_md_context);
-  gpr_mu_destroy(&calld->security_context_mu);
 }
 
 /* Constructor for channel_data */
@@ -382,6 +412,5 @@ const grpc_channel_filter grpc_client_auth_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "client-auth"};
diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 6788126769..a7568b995f 100644
--- a/src/core/lib/security/transport/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -136,15 +136,27 @@ void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_channel_security_connector_check_call_host(
+bool grpc_channel_security_connector_check_call_host(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
     const char *host, grpc_auth_context *auth_context,
-    grpc_security_call_host_check_cb cb, void *user_data) {
+    grpc_closure *on_call_host_checked, grpc_error **error) {
   if (sc == NULL || sc->check_call_host == NULL) {
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR);
-  } else {
-    sc->check_call_host(exec_ctx, sc, host, auth_context, cb, user_data);
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "cannot check call host -- no security connector");
+    return true;
   }
+  return sc->check_call_host(exec_ctx, sc, host, auth_context,
+                             on_call_host_checked, error);
+}
+
+void grpc_channel_security_connector_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  if (sc == NULL || sc->cancel_check_call_host == NULL) {
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
+  sc->cancel_check_call_host(exec_ctx, sc, on_call_host_checked, error);
 }
 
 #ifndef NDEBUG
@@ -368,13 +380,19 @@ static void fake_server_check_peer(grpc_exec_ctx *exec_ctx,
   fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked);
 }
 
-static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
+static bool fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
                                          grpc_channel_security_connector *sc,
                                          const char *host,
                                          grpc_auth_context *auth_context,
-                                         grpc_security_call_host_check_cb cb,
-                                         void *user_data) {
-  cb(exec_ctx, user_data, GRPC_SECURITY_OK);
+                                         grpc_closure *on_call_host_checked,
+                                         grpc_error **error) {
+  return true;
+}
+
+static void fake_channel_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static void fake_channel_add_handshakers(
@@ -413,6 +431,7 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = fake_channel_check_call_host;
+  c->base.cancel_check_call_host = fake_channel_cancel_check_call_host;
   c->base.add_handshakers = fake_channel_add_handshakers;
   c->target = gpr_strdup(target);
   const char *expected_targets = grpc_fake_transport_get_expected_targets(args);
@@ -663,26 +682,35 @@ void tsi_shallow_peer_destruct(tsi_peer *peer) {
   if (peer->properties != NULL) gpr_free(peer->properties);
 }
 
-static void ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
+static bool ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
                                         grpc_channel_security_connector *sc,
                                         const char *host,
                                         grpc_auth_context *auth_context,
-                                        grpc_security_call_host_check_cb cb,
-                                        void *user_data) {
+                                        grpc_closure *on_call_host_checked,
+                                        grpc_error **error) {
   grpc_ssl_channel_security_connector *c =
       (grpc_ssl_channel_security_connector *)sc;
   grpc_security_status status = GRPC_SECURITY_ERROR;
   tsi_peer peer = tsi_shallow_peer_from_ssl_auth_context(auth_context);
   if (ssl_host_matches_name(&peer, host)) status = GRPC_SECURITY_OK;
-
   /* If the target name was overridden, then the original target_name was
      'checked' transitively during the previous peer check at the end of the
      handshake. */
   if (c->overridden_target_name != NULL && strcmp(host, c->target_name) == 0) {
     status = GRPC_SECURITY_OK;
   }
-  cb(exec_ctx, user_data, status);
+  if (status != GRPC_SECURITY_OK) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "call host does not match SSL server name");
+  }
   tsi_shallow_peer_destruct(&peer);
+  return true;
+}
+
+static void ssl_channel_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_security_connector_vtable ssl_channel_vtable = {
@@ -811,6 +839,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = ssl_channel_check_call_host;
+  c->base.cancel_check_call_host = ssl_channel_cancel_check_call_host;
   c->base.add_handshakers = ssl_channel_add_handshakers;
   gpr_split_host_port(target_name, &c->target_name, &port);
   gpr_free(port);
diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 1c0fe40045..4f9b63ad20 100644
--- a/src/core/lib/security/transport/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -117,27 +117,38 @@ grpc_security_connector *grpc_security_connector_find_in_args(
 
 typedef struct grpc_channel_security_connector grpc_channel_security_connector;
 
-typedef void (*grpc_security_call_host_check_cb)(grpc_exec_ctx *exec_ctx,
-                                                 void *user_data,
-                                                 grpc_security_status status);
-
 struct grpc_channel_security_connector {
   grpc_security_connector base;
   grpc_call_credentials *request_metadata_creds;
-  void (*check_call_host)(grpc_exec_ctx *exec_ctx,
+  bool (*check_call_host)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc, const char *host,
                           grpc_auth_context *auth_context,
-                          grpc_security_call_host_check_cb cb, void *user_data);
+                          grpc_closure *on_call_host_checked,
+                          grpc_error **error);
+  void (*cancel_check_call_host)(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_security_connector *sc,
+                                 grpc_closure *on_call_host_checked,
+                                 grpc_error *error);
   void (*add_handshakers)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc,
                           grpc_handshake_manager *handshake_mgr);
 };
 
-/* Checks that the host that will be set for a call is acceptable. */
-void grpc_channel_security_connector_check_call_host(
+/// Checks that the host that will be set for a call is acceptable.
+/// Returns true if completed synchronously, in which case \a error will
+/// be set to indicate the result.  Otherwise, \a on_call_host_checked
+/// will be invoked when complete.
+bool grpc_channel_security_connector_check_call_host(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
     const char *host, grpc_auth_context *auth_context,
-    grpc_security_call_host_check_cb cb, void *user_data);
+    grpc_closure *on_call_host_checked, grpc_error **error);
+
+/// Cancels a pending asychronous call to
+/// grpc_channel_security_connector_check_call_host() with
+/// \a on_call_host_checked as its callback.
+void grpc_channel_security_connector_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error);
 
 /* Registers handshakers with \a handshake_mgr. */
 void grpc_channel_security_connector_add_handshakers(
diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c
index 239a211c0b..fc9c9f980f 100644
--- a/src/core/lib/security/transport/security_handshaker.c
+++ b/src/core/lib/security/transport/security_handshaker.c
@@ -147,7 +147,7 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg,
     goto done;
   }
   // Get unused bytes.
-  unsigned char *unused_bytes = NULL;
+  const unsigned char *unused_bytes = NULL;
   size_t unused_bytes_size = 0;
   result = tsi_handshaker_result_get_unused_bytes(
       h->handshaker_result, &unused_bytes, &unused_bytes_size);
@@ -261,7 +261,7 @@ static grpc_error *do_handshaker_next_locked(
     grpc_exec_ctx *exec_ctx, security_handshaker *h,
     const unsigned char *bytes_received, size_t bytes_received_size) {
   // Invoke TSI handshaker.
-  unsigned char *bytes_to_send = NULL;
+  const unsigned char *bytes_to_send = NULL;
   size_t bytes_to_send_size = 0;
   tsi_handshaker_result *handshaker_result = NULL;
   tsi_result result = tsi_handshaker_next(
diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
index 9bf3f0ca0f..b721ce4a22 100644
--- a/src/core/lib/security/transport/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -26,7 +26,15 @@
 #include "src/core/lib/security/transport/auth_filters.h"
 #include "src/core/lib/slice/slice_internal.h"
 
+typedef enum {
+  STATE_INIT = 0,
+  STATE_DONE,
+  STATE_CANCELLED,
+} async_state;
+
 typedef struct call_data {
+  grpc_call_combiner *call_combiner;
+  grpc_call_stack *owning_call;
   grpc_transport_stream_op_batch *recv_initial_metadata_batch;
   grpc_closure *original_recv_initial_metadata_ready;
   grpc_closure recv_initial_metadata_ready;
@@ -34,6 +42,8 @@ typedef struct call_data {
   const grpc_metadata *consumed_md;
   size_t num_consumed_md;
   grpc_auth_context *auth_context;
+  grpc_closure cancel_closure;
+  gpr_atm state;  // async_state
 } call_data;
 
 typedef struct channel_data {
@@ -78,54 +88,92 @@ static grpc_filtered_mdelem remove_consumed_md(grpc_exec_ctx *exec_ctx,
   return GRPC_FILTERED_MDELEM(md);
 }
 
-/* called from application code */
-static void on_md_processing_done(
-    void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md,
-    const grpc_metadata *response_md, size_t num_response_md,
-    grpc_status_code status, const char *error_details) {
-  grpc_call_element *elem = user_data;
+static void on_md_processing_done_inner(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_element *elem,
+                                        const grpc_metadata *consumed_md,
+                                        size_t num_consumed_md,
+                                        const grpc_metadata *response_md,
+                                        size_t num_response_md,
+                                        grpc_error *error) {
   call_data *calld = elem->call_data;
   grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL);
   /* TODO(jboeuf): Implement support for response_md. */
   if (response_md != NULL && num_response_md > 0) {
     gpr_log(GPR_INFO,
             "response_md in auth metadata processing not supported for now. "
             "Ignoring...");
   }
-  grpc_error *error = GRPC_ERROR_NONE;
-  if (status == GRPC_STATUS_OK) {
+  if (error == GRPC_ERROR_NONE) {
     calld->consumed_md = consumed_md;
     calld->num_consumed_md = num_consumed_md;
     error = grpc_metadata_batch_filter(
-        &exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata,
+        exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata,
         remove_consumed_md, elem, "Response metadata filtering error");
-  } else {
-    if (error_details == NULL) {
-      error_details = "Authentication metadata processing failed.";
+  }
+  GRPC_CLOSURE_SCHED(exec_ctx, calld->original_recv_initial_metadata_ready,
+                     error);
+}
+
+// Called from application code.
+static void on_md_processing_done(
+    void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md,
+    const grpc_metadata *response_md, size_t num_response_md,
+    grpc_status_code status, const char *error_details) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  // If the call was not cancelled while we were in flight, process the result.
+  if (gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT,
+                       (gpr_atm)STATE_DONE)) {
+    grpc_error *error = GRPC_ERROR_NONE;
+    if (status != GRPC_STATUS_OK) {
+      if (error_details == NULL) {
+        error_details = "Authentication metadata processing failed.";
+      }
+      error = grpc_error_set_int(
+          GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details),
+          GRPC_ERROR_INT_GRPC_STATUS, status);
     }
-    error =
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details),
-                           GRPC_ERROR_INT_GRPC_STATUS, status);
+    on_md_processing_done_inner(&exec_ctx, elem, consumed_md, num_consumed_md,
+                                response_md, num_response_md, error);
   }
+  // Clean up.
   for (size_t i = 0; i < calld->md.count; i++) {
     grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].key);
     grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].value);
   }
   grpc_metadata_array_destroy(&calld->md);
-  GRPC_CLOSURE_SCHED(&exec_ctx, calld->original_recv_initial_metadata_ready,
-                     error);
+  GRPC_CALL_STACK_UNREF(&exec_ctx, calld->owning_call, "server_auth_metadata");
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
+static void cancel_call(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = elem->call_data;
+  // If the result was not already processed, invoke the callback now.
+  if (gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT,
+                       (gpr_atm)STATE_CANCELLED)) {
+    on_md_processing_done_inner(exec_ctx, elem, NULL, 0, NULL, 0,
+                                GRPC_ERROR_REF(error));
+  }
+}
+
 static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *arg,
                                         grpc_error *error) {
-  grpc_call_element *elem = arg;
+  grpc_call_element *elem = (grpc_call_element *)arg;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch;
   if (error == GRPC_ERROR_NONE) {
     if (chand->creds != NULL && chand->creds->processor.process != NULL) {
+      // We're calling out to the application, so we need to make sure
+      // to drop the call combiner early if we get cancelled.
+      GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem,
+                        grpc_schedule_on_exec_ctx);
+      grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
+                                              &calld->cancel_closure);
+      GRPC_CALL_STACK_REF(calld->owning_call, "server_auth_metadata");
       calld->md = metadata_batch_to_md_array(
           batch->payload->recv_initial_metadata.recv_initial_metadata);
       chand->creds->processor.process(
@@ -159,6 +207,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
+  calld->call_combiner = args->call_combiner;
+  calld->owning_call = args->call_stack;
   GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
                     recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
@@ -218,6 +268,5 @@ const grpc_channel_filter grpc_server_auth_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "server-auth"};
diff --git a/src/core/lib/support/avl.c b/src/core/lib/support/avl.c
index a6178fdbce..0e28b24c98 100644
--- a/src/core/lib/support/avl.c
+++ b/src/core/lib/support/avl.c
@@ -39,15 +39,16 @@ static gpr_avl_node *ref_node(gpr_avl_node *node) {
   return node;
 }
 
-static void unref_node(const gpr_avl_vtable *vtable, gpr_avl_node *node) {
+static void unref_node(const gpr_avl_vtable *vtable, gpr_avl_node *node,
+                       void *user_data) {
   if (node == NULL) {
     return;
   }
   if (gpr_unref(&node->refs)) {
-    vtable->destroy_key(node->key);
-    vtable->destroy_value(node->value);
-    unref_node(vtable, node->left);
-    unref_node(vtable, node->right);
+    vtable->destroy_key(node->key, user_data);
+    vtable->destroy_value(node->value, user_data);
+    unref_node(vtable, node->left, user_data);
+    unref_node(vtable, node->right, user_data);
     gpr_free(node);
   }
 }
@@ -87,30 +88,30 @@ gpr_avl_node *new_node(void *key, void *value, gpr_avl_node *left,
 }
 
 static gpr_avl_node *get(const gpr_avl_vtable *vtable, gpr_avl_node *node,
-                         void *key) {
+                         void *key, void *user_data) {
   long cmp;
 
   if (node == NULL) {
     return NULL;
   }
 
-  cmp = vtable->compare_keys(node->key, key);
+  cmp = vtable->compare_keys(node->key, key, user_data);
   if (cmp == 0) {
     return node;
   } else if (cmp > 0) {
-    return get(vtable, node->left, key);
+    return get(vtable, node->left, key, user_data);
   } else {
-    return get(vtable, node->right, key);
+    return get(vtable, node->right, key, user_data);
   }
 }
 
-void *gpr_avl_get(gpr_avl avl, void *key) {
-  gpr_avl_node *node = get(avl.vtable, avl.root, key);
+void *gpr_avl_get(gpr_avl avl, void *key, void *user_data) {
+  gpr_avl_node *node = get(avl.vtable, avl.root, key, user_data);
   return node ? node->value : NULL;
 }
 
-int gpr_avl_maybe_get(gpr_avl avl, void *key, void **value) {
-  gpr_avl_node *node = get(avl.vtable, avl.root, key);
+int gpr_avl_maybe_get(gpr_avl avl, void *key, void **value, void *user_data) {
+  gpr_avl_node *node = get(avl.vtable, avl.root, key, user_data);
   if (node != NULL) {
     *value = node->value;
     return 1;
@@ -120,70 +121,75 @@ int gpr_avl_maybe_get(gpr_avl avl, void *key, void **value) {
 
 static gpr_avl_node *rotate_left(const gpr_avl_vtable *vtable, void *key,
                                  void *value, gpr_avl_node *left,
-                                 gpr_avl_node *right) {
-  gpr_avl_node *n =
-      new_node(vtable->copy_key(right->key), vtable->copy_value(right->value),
-               new_node(key, value, left, ref_node(right->left)),
-               ref_node(right->right));
-  unref_node(vtable, right);
+                                 gpr_avl_node *right, void *user_data) {
+  gpr_avl_node *n = new_node(vtable->copy_key(right->key, user_data),
+                             vtable->copy_value(right->value, user_data),
+                             new_node(key, value, left, ref_node(right->left)),
+                             ref_node(right->right));
+  unref_node(vtable, right, user_data);
   return n;
 }
 
 static gpr_avl_node *rotate_right(const gpr_avl_vtable *vtable, void *key,
                                   void *value, gpr_avl_node *left,
-                                  gpr_avl_node *right) {
-  gpr_avl_node *n = new_node(
-      vtable->copy_key(left->key), vtable->copy_value(left->value),
-      ref_node(left->left), new_node(key, value, ref_node(left->right), right));
-  unref_node(vtable, left);
+                                  gpr_avl_node *right, void *user_data) {
+  gpr_avl_node *n =
+      new_node(vtable->copy_key(left->key, user_data),
+               vtable->copy_value(left->value, user_data), ref_node(left->left),
+               new_node(key, value, ref_node(left->right), right));
+  unref_node(vtable, left, user_data);
   return n;
 }
 
 static gpr_avl_node *rotate_left_right(const gpr_avl_vtable *vtable, void *key,
                                        void *value, gpr_avl_node *left,
-                                       gpr_avl_node *right) {
+                                       gpr_avl_node *right, void *user_data) {
   /* rotate_right(..., rotate_left(left), right) */
-  gpr_avl_node *n = new_node(
-      vtable->copy_key(left->right->key),
-      vtable->copy_value(left->right->value),
-      new_node(vtable->copy_key(left->key), vtable->copy_value(left->value),
-               ref_node(left->left), ref_node(left->right->left)),
-      new_node(key, value, ref_node(left->right->right), right));
-  unref_node(vtable, left);
+  gpr_avl_node *n =
+      new_node(vtable->copy_key(left->right->key, user_data),
+               vtable->copy_value(left->right->value, user_data),
+               new_node(vtable->copy_key(left->key, user_data),
+                        vtable->copy_value(left->value, user_data),
+                        ref_node(left->left), ref_node(left->right->left)),
+               new_node(key, value, ref_node(left->right->right), right));
+  unref_node(vtable, left, user_data);
   return n;
 }
 
 static gpr_avl_node *rotate_right_left(const gpr_avl_vtable *vtable, void *key,
                                        void *value, gpr_avl_node *left,
-                                       gpr_avl_node *right) {
+                                       gpr_avl_node *right, void *user_data) {
   /* rotate_left(..., left, rotate_right(right)) */
-  gpr_avl_node *n = new_node(
-      vtable->copy_key(right->left->key),
-      vtable->copy_value(right->left->value),
-      new_node(key, value, left, ref_node(right->left->left)),
-      new_node(vtable->copy_key(right->key), vtable->copy_value(right->value),
-               ref_node(right->left->right), ref_node(right->right)));
-  unref_node(vtable, right);
+  gpr_avl_node *n =
+      new_node(vtable->copy_key(right->left->key, user_data),
+               vtable->copy_value(right->left->value, user_data),
+               new_node(key, value, left, ref_node(right->left->left)),
+               new_node(vtable->copy_key(right->key, user_data),
+                        vtable->copy_value(right->value, user_data),
+                        ref_node(right->left->right), ref_node(right->right)));
+  unref_node(vtable, right, user_data);
   return n;
 }
 
 static gpr_avl_node *rebalance(const gpr_avl_vtable *vtable, void *key,
                                void *value, gpr_avl_node *left,
-                               gpr_avl_node *right) {
+                               gpr_avl_node *right, void *user_data) {
   switch (node_height(left) - node_height(right)) {
     case 2:
       if (node_height(left->left) - node_height(left->right) == -1) {
         return assert_invariants(
-            rotate_left_right(vtable, key, value, left, right));
+            rotate_left_right(vtable, key, value, left, right, user_data));
       } else {
-        return assert_invariants(rotate_right(vtable, key, value, left, right));
+        return assert_invariants(
+            rotate_right(vtable, key, value, left, right, user_data));
       }
     case -2:
       if (node_height(right->left) - node_height(right->right) == 1) {
         return assert_invariants(
-            rotate_right_left(vtable, key, value, left, right));
+            rotate_right_left(vtable, key, value, left, right, user_data));
       } else {
-        return assert_invariants(rotate_left(vtable, key, value, left, right));
+        return assert_invariants(
+            rotate_left(vtable, key, value, left, right, user_data));
       }
     default:
       return assert_invariants(new_node(key, value, left, right));
@@ -191,30 +197,32 @@ static gpr_avl_node *rebalance(const gpr_avl_vtable *vtable, void *key,
 }
 
 static gpr_avl_node *add_key(const gpr_avl_vtable *vtable, gpr_avl_node *node,
-                             void *key, void *value) {
+                             void *key, void *value, void *user_data) {
   long cmp;
   if (node == NULL) {
     return new_node(key, value, NULL, NULL);
   }
-  cmp = vtable->compare_keys(node->key, key);
+  cmp = vtable->compare_keys(node->key, key, user_data);
   if (cmp == 0) {
     return new_node(key, value, ref_node(node->left), ref_node(node->right));
   } else if (cmp > 0) {
-    return rebalance(
-        vtable, vtable->copy_key(node->key), vtable->copy_value(node->value),
-        add_key(vtable, node->left, key, value), ref_node(node->right));
+    return rebalance(vtable, vtable->copy_key(node->key, user_data),
+                     vtable->copy_value(node->value, user_data),
+                     add_key(vtable, node->left, key, value, user_data),
+                     ref_node(node->right), user_data);
   } else {
-    return rebalance(vtable, vtable->copy_key(node->key),
-                     vtable->copy_value(node->value), ref_node(node->left),
-                     add_key(vtable, node->right, key, value));
+    return rebalance(
+        vtable, vtable->copy_key(node->key, user_data),
+        vtable->copy_value(node->value, user_data), ref_node(node->left),
+        add_key(vtable, node->right, key, value, user_data), user_data);
   }
 }
 
-gpr_avl gpr_avl_add(gpr_avl avl, void *key, void *value) {
+gpr_avl gpr_avl_add(gpr_avl avl, void *key, void *value, void *user_data) {
   gpr_avl_node *old_root = avl.root;
-  avl.root = add_key(avl.vtable, avl.root, key, value);
+  avl.root = add_key(avl.vtable, avl.root, key, value, user_data);
   assert_invariants(avl.root);
-  unref_node(avl.vtable, old_root);
+  unref_node(avl.vtable, old_root, user_data);
   return avl;
 }
 
@@ -233,12 +241,13 @@ static gpr_avl_node *in_order_tail(gpr_avl_node *node) {
 }
 
 static gpr_avl_node *remove_key(const gpr_avl_vtable *vtable,
-                                gpr_avl_node *node, void *key) {
+                                gpr_avl_node *node, void *key,
+                                void *user_data) {
   long cmp;
   if (node == NULL) {
     return NULL;
   }
-  cmp = vtable->compare_keys(node->key, key);
+  cmp = vtable->compare_keys(node->key, key, user_data);
   if (cmp == 0) {
     if (node->left == NULL) {
       return ref_node(node->right);
@@ -246,39 +255,45 @@ static gpr_avl_node *remove_key(const gpr_avl_vtable *vtable,
       return ref_node(node->left);
     } else if (node->left->height < node->right->height) {
       gpr_avl_node *h = in_order_head(node->right);
-      return rebalance(vtable, vtable->copy_key(h->key),
-                       vtable->copy_value(h->value), ref_node(node->left),
-                       remove_key(vtable, node->right, h->key));
+      return rebalance(
+          vtable, vtable->copy_key(h->key, user_data),
+          vtable->copy_value(h->value, user_data), ref_node(node->left),
+          remove_key(vtable, node->right, h->key, user_data), user_data);
     } else {
       gpr_avl_node *h = in_order_tail(node->left);
-      return rebalance(
-          vtable, vtable->copy_key(h->key), vtable->copy_value(h->value),
-          remove_key(vtable, node->left, h->key), ref_node(node->right));
+      return rebalance(vtable, vtable->copy_key(h->key, user_data),
+                       vtable->copy_value(h->value, user_data),
+                       remove_key(vtable, node->left, h->key, user_data),
+                       ref_node(node->right), user_data);
     }
   } else if (cmp > 0) {
-    return rebalance(
-        vtable, vtable->copy_key(node->key), vtable->copy_value(node->value),
-        remove_key(vtable, node->left, key), ref_node(node->right));
+    return rebalance(vtable, vtable->copy_key(node->key, user_data),
+                     vtable->copy_value(node->value, user_data),
+                     remove_key(vtable, node->left, key, user_data),
+                     ref_node(node->right), user_data);
   } else {
-    return rebalance(vtable, vtable->copy_key(node->key),
-                     vtable->copy_value(node->value), ref_node(node->left),
-                     remove_key(vtable, node->right, key));
+    return rebalance(
+        vtable, vtable->copy_key(node->key, user_data),
+        vtable->copy_value(node->value, user_data), ref_node(node->left),
+        remove_key(vtable, node->right, key, user_data), user_data);
   }
 }
 
-gpr_avl gpr_avl_remove(gpr_avl avl, void *key) {
+gpr_avl gpr_avl_remove(gpr_avl avl, void *key, void *user_data) {
   gpr_avl_node *old_root = avl.root;
-  avl.root = remove_key(avl.vtable, avl.root, key);
+  avl.root = remove_key(avl.vtable, avl.root, key, user_data);
   assert_invariants(avl.root);
-  unref_node(avl.vtable, old_root);
+  unref_node(avl.vtable, old_root, user_data);
   return avl;
 }
 
-gpr_avl gpr_avl_ref(gpr_avl avl) {
+gpr_avl gpr_avl_ref(gpr_avl avl, void *user_data) {
   ref_node(avl.root);
   return avl;
 }
 
-void gpr_avl_unref(gpr_avl avl) { unref_node(avl.vtable, avl.root); }
+void gpr_avl_unref(gpr_avl avl, void *user_data) {
+  unref_node(avl.vtable, avl.root, user_data);
+}
 
 int gpr_avl_is_empty(gpr_avl avl) { return avl.root == NULL; }
diff --git a/src/core/lib/support/block_annotate.h b/src/core/lib/support/block_annotate.h
index 0a2cb45018..8e3ef7df65 100644
--- a/src/core/lib/support/block_annotate.h
+++ b/src/core/lib/support/block_annotate.h
@@ -19,15 +19,37 @@
 #ifndef GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H
 #define GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void gpr_thd_start_blocking_region();
+void gpr_thd_end_blocking_region();
+
+#ifdef __cplusplus
+}
+#endif
+
 /* These annotations identify the beginning and end of regions where
    the code may block for reasons other than synchronization functions.
    These include poll, epoll, and getaddrinfo. */
 
+#ifdef GRPC_SCHEDULING_MARK_BLOCKING_REGION
+#define GRPC_SCHEDULING_START_BLOCKING_REGION \
+  do {                                        \
+    gpr_thd_start_blocking_region();          \
+  } while (0)
+#define GRPC_SCHEDULING_END_BLOCKING_REGION \
+  do {                                      \
+    gpr_thd_end_blocking_region();          \
+  } while (0)
+#else
 #define GRPC_SCHEDULING_START_BLOCKING_REGION \
   do {                                        \
   } while (0)
 #define GRPC_SCHEDULING_END_BLOCKING_REGION \
   do {                                      \
   } while (0)
+#endif
 
 #endif /* GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H */
diff --git a/src/core/lib/support/murmur_hash.c b/src/core/lib/support/murmur_hash.c
index f329611818..f06b970de7 100644
--- a/src/core/lib/support/murmur_hash.c
+++ b/src/core/lib/support/murmur_hash.c
@@ -62,8 +62,10 @@ uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
   switch (len & 3) {
     case 3:
       k1 ^= ((uint32_t)tail[2]) << 16;
+    /* fallthrough */
     case 2:
       k1 ^= ((uint32_t)tail[1]) << 8;
+    /* fallthrough */
     case 1:
       k1 ^= tail[0];
       k1 *= c1;
diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c
index ef8405cca8..7d60b1de17 100644
--- a/src/core/lib/surface/alarm.c
+++ b/src/core/lib/surface/alarm.c
@@ -15,13 +15,21 @@
  * limitations under the License.
  *
  */
+#include "src/core/lib/surface/alarm_internal.h"
 
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/surface/completion_queue.h"
 
+#ifndef NDEBUG
+grpc_tracer_flag grpc_trace_alarm_refcount =
+    GRPC_TRACER_INITIALIZER(false, "alarm_refcount");
+#endif
+
 struct grpc_alarm {
+  gpr_refcount refs;
   grpc_timer alarm;
   grpc_closure on_alarm;
   grpc_cq_completion completion;
@@ -31,13 +39,58 @@ struct grpc_alarm {
   void *tag;
 };
 
-static void do_nothing_end_completion(grpc_exec_ctx *exec_ctx, void *arg,
-                                      grpc_cq_completion *c) {}
+static void alarm_ref(grpc_alarm *alarm) { gpr_ref(&alarm->refs); }
+
+static void alarm_unref(grpc_alarm *alarm) {
+  if (gpr_unref(&alarm->refs)) {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm");
+    grpc_exec_ctx_finish(&exec_ctx);
+    gpr_free(alarm);
+  }
+}
+
+#ifndef NDEBUG
+static void alarm_ref_dbg(grpc_alarm *alarm, const char *reason,
+                          const char *file, int line) {
+  if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) {
+    gpr_atm val = gpr_atm_no_barrier_load(&alarm->refs.count);
+    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+            "Alarm:%p  ref %" PRIdPTR " -> %" PRIdPTR " %s", alarm, val,
+            val + 1, reason);
+  }
+
+  alarm_ref(alarm);
+}
+
+static void alarm_unref_dbg(grpc_alarm *alarm, const char *reason,
+                            const char *file, int line) {
+  if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) {
+    gpr_atm val = gpr_atm_no_barrier_load(&alarm->refs.count);
+    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+            "Alarm:%p  Unref %" PRIdPTR " -> %" PRIdPTR " %s", alarm, val,
+            val - 1, reason);
+  }
+
+  alarm_unref(alarm);
+}
+#endif
+
+static void alarm_end_completion(grpc_exec_ctx *exec_ctx, void *arg,
+                                 grpc_cq_completion *c) {
+  grpc_alarm *alarm = arg;
+  GRPC_ALARM_UNREF(alarm, "dequeue-end-op");
+}
 
 static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_alarm *alarm = arg;
-  grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, error,
-                 do_nothing_end_completion, NULL, &alarm->completion);
+
+  /* We are queuing an op on completion queue. This means, the alarm's structure
+     cannot be destroyed until the op is dequeued. Adding an extra ref
+     here and unref'ing when the op is dequeued will achieve this */
+  GRPC_ALARM_REF(alarm, "queue-end-op");
+  grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, error, alarm_end_completion,
+                 (void *)alarm, &alarm->completion);
 }
 
 grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline,
@@ -45,11 +98,19 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline,
   grpc_alarm *alarm = gpr_malloc(sizeof(grpc_alarm));
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
+  gpr_ref_init(&alarm->refs, 1);
+
+#ifndef NDEBUG
+  if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) {
+    gpr_log(GPR_DEBUG, "Alarm:%p created (ref: 1)", alarm);
+  }
+#endif
+
   GRPC_CQ_INTERNAL_REF(cq, "alarm");
   alarm->cq = cq;
   alarm->tag = tag;
 
-  grpc_cq_begin_op(cq, tag);
+  GPR_ASSERT(grpc_cq_begin_op(cq, tag));
   GRPC_CLOSURE_INIT(&alarm->on_alarm, alarm_cb, alarm,
                     grpc_schedule_on_exec_ctx);
   grpc_timer_init(&exec_ctx, &alarm->alarm,
@@ -66,9 +127,6 @@ void grpc_alarm_cancel(grpc_alarm *alarm) {
 }
 
 void grpc_alarm_destroy(grpc_alarm *alarm) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_alarm_cancel(alarm);
-  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm");
-  gpr_free(alarm);
-  grpc_exec_ctx_finish(&exec_ctx);
+  GRPC_ALARM_UNREF(alarm, "alarm_destroy");
 }
diff --git a/src/core/lib/surface/alarm_internal.h b/src/core/lib/surface/alarm_internal.h
new file mode 100644
index 0000000000..7f2126c5c9
--- /dev/null
+++ b/src/core/lib/surface/alarm_internal.h
@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2015-2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SURFACE_ALARM_INTERNAL_H
+#define GRPC_CORE_LIB_SURFACE_ALARM_INTERNAL_H
+
+#include <grpc/support/log.h>
+#include "src/core/lib/debug/trace.h"
+
+#ifndef NDEBUG
+
+extern grpc_tracer_flag grpc_trace_alarm_refcount;
+
+#define GRPC_ALARM_REF(a, reason) alarm_ref_dbg(a, reason, __FILE__, __LINE__)
+#define GRPC_ALARM_UNREF(a, reason) \
+  alarm_unref_dbg(a, reason, __FILE__, __LINE__)
+
+#else /* !defined(NDEBUG) */
+
+#define GRPC_ALARM_REF(a, reason) alarm_ref(a)
+#define GRPC_ALARM_UNREF(a, reason) alarm_unref(a)
+
+#endif /* defined(NDEBUG) */
+
+#endif /* GRPC_CORE_LIB_SURFACE_ALARM_INTERNAL_H */
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index 2f42726c7f..03eaaf99ac 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -122,6 +122,7 @@ typedef struct batch_control {
       bool is_closure;
     } notify_tag;
   } completion_data;
+  grpc_closure start_batch;
   grpc_closure finish_batch;
   gpr_refcount steps_to_complete;
 
@@ -145,9 +146,13 @@ typedef struct {
   grpc_call *sibling_prev;
 } child_call;
 
+#define RECV_NONE ((gpr_atm)0)
+#define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1)
+
 struct grpc_call {
   gpr_refcount ext_ref;
   gpr_arena *arena;
+  grpc_call_combiner call_combiner;
   grpc_completion_queue *cq;
   grpc_polling_entity pollent;
   grpc_channel *channel;
@@ -171,9 +176,6 @@ struct grpc_call {
   gpr_atm any_ops_sent_atm;
   gpr_atm received_final_op_atm;
 
-  /* have we received initial metadata */
-  bool has_initial_md_been_received;
-
   batch_control *active_batches[MAX_CONCURRENT_BATCHES];
   grpc_transport_stream_op_batch_payload stream_op_payload;
 
@@ -184,6 +186,11 @@ struct grpc_call {
      Element 0 is initial metadata, element 1 is trailing metadata. */
   grpc_metadata_array *buffered_metadata[2];
 
+  grpc_metadata compression_md;
+
+  // A char* indicating the peer name.
+  gpr_atm peer_string;
+
   /* Packed received call statuses from various sources */
   gpr_atm status[STATUS_SOURCE_COUNT];
 
@@ -193,8 +200,12 @@ struct grpc_call {
 
   /* Compression algorithm for *incoming* data */
   grpc_compression_algorithm incoming_compression_algorithm;
+  /* Stream compression algorithm for *incoming* data */
+  grpc_stream_compression_algorithm incoming_stream_compression_algorithm;
   /* Supported encodings (compression algorithms), a bitset */
   uint32_t encodings_accepted_by_peer;
+  /* Supported stream encodings (stream compression algorithms), a bitset */
+  uint32_t stream_encodings_accepted_by_peer;
 
   /* Contexts for various subsystems (security, tracing, ...). */
   grpc_call_context_element context[GRPC_CONTEXT_COUNT];
@@ -227,7 +238,23 @@ struct grpc_call {
     } server;
   } final_op;
 
-  void *saved_receiving_stream_ready_bctlp;
+  /* recv_state can contain one of the following values:
+     RECV_NONE :                 :  no initial metadata and messages received
+     RECV_INITIAL_METADATA_FIRST :  received initial metadata first
+     a batch_control*            :  received messages first
+
+                 +------1------RECV_NONE------3-----+
+                 |                                  |
+                 |                                  |
+                 v                                  v
+     RECV_INITIAL_METADATA_FIRST        receiving_stream_ready_bctlp
+           |           ^                      |           ^
+           |           |                      |           |
+           +-----2-----+                      +-----4-----+
+
+    For 1, 4: See receiving_initial_metadata_ready() function
+    For 2, 3: See receiving_stream_ready() function */
+  gpr_atm recv_state;
 };
 
 grpc_tracer_flag grpc_call_error_trace =
@@ -242,8 +269,9 @@ grpc_tracer_flag grpc_compression_trace =
 #define CALL_FROM_TOP_ELEM(top_elem) \
   CALL_FROM_CALL_STACK(grpc_call_stack_from_top_element(top_elem))
 
-static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
-                       grpc_transport_stream_op_batch *op);
+static void execute_batch(grpc_exec_ctx *exec_ctx, grpc_call *call,
+                          grpc_transport_stream_op_batch *op,
+                          grpc_closure *start_batch_closure);
 static void cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
                                status_source source, grpc_status_code status,
                                const char *description);
@@ -308,6 +336,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
                          sizeof(grpc_call) + channel_stack->call_stack_size);
   gpr_ref_init(&call->ext_ref, 1);
   call->arena = arena;
+  grpc_call_combiner_init(&call->call_combiner);
   *out_call = call;
   call->channel = args->channel;
   call->cq = args->cq;
@@ -416,7 +445,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
       .path = path,
       .start_time = call->start_time,
       .deadline = send_deadline,
-      .arena = call->arena};
+      .arena = call->arena,
+      .call_combiner = &call->call_combiner};
   add_init_error(&error, grpc_call_stack_init(exec_ctx, channel_stack, 1,
                                               destroy_call, call, &call_args));
   if (error != GRPC_ERROR_NONE) {
@@ -483,6 +513,8 @@ static void release_call(grpc_exec_ctx *exec_ctx, void *call,
                          grpc_error *error) {
   grpc_call *c = call;
   grpc_channel *channel = c->channel;
+  grpc_call_combiner_destroy(&c->call_combiner);
+  gpr_free((char *)c->peer_string);
   grpc_channel_update_call_size_estimate(channel, gpr_arena_destroy(c->arena));
   GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call");
 }
@@ -582,30 +614,37 @@ grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) {
   return GRPC_CALL_OK;
 }
 
-static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
-                       grpc_transport_stream_op_batch *op) {
-  grpc_call_element *elem;
-
-  GPR_TIMER_BEGIN("execute_op", 0);
-  elem = CALL_ELEM_FROM_CALL(call, 0);
-  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op);
-  GPR_TIMER_END("execute_op", 0);
+// This is called via the call combiner to start sending a batch down
+// the filter stack.
+static void execute_batch_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg,
+                                           grpc_error *ignored) {
+  grpc_transport_stream_op_batch *batch = arg;
+  grpc_call *call = batch->handler_private.extra_arg;
+  GPR_TIMER_BEGIN("execute_batch", 0);
+  grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0);
+  GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
+  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, batch);
+  GPR_TIMER_END("execute_batch", 0);
+}
+
+// start_batch_closure points to a caller-allocated closure to be used
+// for entering the call combiner.
+static void execute_batch(grpc_exec_ctx *exec_ctx, grpc_call *call,
+                          grpc_transport_stream_op_batch *batch,
+                          grpc_closure *start_batch_closure) {
+  batch->handler_private.extra_arg = call;
+  GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
+                    grpc_schedule_on_exec_ctx);
+  GRPC_CALL_COMBINER_START(exec_ctx, &call->call_combiner, start_batch_closure,
+                           GRPC_ERROR_NONE, "executing batch");
 }
 
 char *grpc_call_get_peer(grpc_call *call) {
-  grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0);
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  char *result;
-  GRPC_API_TRACE("grpc_call_get_peer(%p)", 1, (call));
-  result = elem->filter->get_peer(&exec_ctx, elem);
-  if (result == NULL) {
-    result = grpc_channel_get_target(call->channel);
-  }
-  if (result == NULL) {
-    result = gpr_strdup("unknown");
-  }
-  grpc_exec_ctx_finish(&exec_ctx);
-  return result;
+  char *peer_string = (char *)gpr_atm_acq_load(&call->peer_string);
+  if (peer_string != NULL) return gpr_strdup(peer_string);
+  peer_string = grpc_channel_get_target(call->channel);
+  if (peer_string != NULL) return peer_string;
+  return gpr_strdup("unknown");
 }
 
 grpc_call *grpc_call_from_top_element(grpc_call_element *elem) {
@@ -632,24 +671,47 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
   return GRPC_CALL_OK;
 }
 
-static void done_termination(grpc_exec_ctx *exec_ctx, void *call,
+typedef struct {
+  grpc_call *call;
+  grpc_closure start_batch;
+  grpc_closure finish_batch;
+} cancel_state;
+
+// The on_complete callback used when sending a cancel_stream batch down
+// the filter stack.  Yields the call combiner when the batch is done.
+static void done_termination(grpc_exec_ctx *exec_ctx, void *arg,
                              grpc_error *error) {
-  GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "termination");
+  cancel_state *state = (cancel_state *)arg;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &state->call->call_combiner,
+                          "on_complete for cancel_stream op");
+  GRPC_CALL_INTERNAL_UNREF(exec_ctx, state->call, "termination");
+  gpr_free(state);
 }
 
 static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c,
                               status_source source, grpc_error *error) {
   GRPC_CALL_INTERNAL_REF(c, "termination");
+  // Inform the call combiner of the cancellation, so that it can cancel
+  // any in-flight asynchronous actions that may be holding the call
+  // combiner.  This ensures that the cancel_stream batch can be sent
+  // down the filter stack in a timely manner.
+  grpc_call_combiner_cancel(exec_ctx, &c->call_combiner, GRPC_ERROR_REF(error));
   set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error));
-  grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op(
-      GRPC_CLOSURE_CREATE(done_termination, c, grpc_schedule_on_exec_ctx));
+  cancel_state *state = (cancel_state *)gpr_malloc(sizeof(*state));
+  state->call = c;
+  GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
+                    grpc_schedule_on_exec_ctx);
+  grpc_transport_stream_op_batch *op =
+      grpc_make_transport_stream_op(&state->finish_batch);
   op->cancel_stream = true;
   op->payload->cancel_stream.cancel_error = error;
-  execute_op(exec_ctx, c, op);
+  execute_batch(exec_ctx, c, op, &state->start_batch);
 }
 
 static grpc_error *error_from_status(grpc_status_code status,
                                      const char *description) {
+  // copying 'description' is needed to ensure the grpc_call_cancel_with_status
+  // guarantee that can be short-lived.
   return grpc_error_set_int(
       grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description),
                          GRPC_ERROR_STR_GRPC_MESSAGE,
@@ -756,6 +818,12 @@ static void set_incoming_compression_algorithm(
   call->incoming_compression_algorithm = algo;
 }
 
+static void set_incoming_stream_compression_algorithm(
+    grpc_call *call, grpc_stream_compression_algorithm algo) {
+  GPR_ASSERT(algo < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT);
+  call->incoming_stream_compression_algorithm = algo;
+}
+
 grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
     grpc_call *call) {
   grpc_compression_algorithm algorithm;
@@ -769,6 +837,13 @@ static grpc_compression_algorithm compression_algorithm_for_level_locked(
                                               call->encodings_accepted_by_peer);
 }
 
+static grpc_stream_compression_algorithm
+stream_compression_algorithm_for_level_locked(
+    grpc_call *call, grpc_stream_compression_level level) {
+  return grpc_stream_compression_algorithm_for_level(
+      level, call->stream_encodings_accepted_by_peer);
+}
+
 uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) {
   uint32_t flags;
   flags = call->test_only_last_message_flags;
@@ -823,13 +898,71 @@ static void set_encodings_accepted_by_peer(grpc_exec_ctx *exec_ctx,
       (void *)(((uintptr_t)call->encodings_accepted_by_peer) + 1));
 }
 
+static void set_stream_encodings_accepted_by_peer(grpc_exec_ctx *exec_ctx,
+                                                  grpc_call *call,
+                                                  grpc_mdelem mdel) {
+  size_t i;
+  grpc_stream_compression_algorithm algorithm;
+  grpc_slice_buffer accept_encoding_parts;
+  grpc_slice accept_encoding_slice;
+  void *accepted_user_data;
+
+  accepted_user_data =
+      grpc_mdelem_get_user_data(mdel, destroy_encodings_accepted_by_peer);
+  if (accepted_user_data != NULL) {
+    call->stream_encodings_accepted_by_peer =
+        (uint32_t)(((uintptr_t)accepted_user_data) - 1);
+    return;
+  }
+
+  accept_encoding_slice = GRPC_MDVALUE(mdel);
+  grpc_slice_buffer_init(&accept_encoding_parts);
+  grpc_slice_split(accept_encoding_slice, ",", &accept_encoding_parts);
+
+  /* Always support no compression */
+  GPR_BITSET(&call->stream_encodings_accepted_by_peer,
+             GRPC_STREAM_COMPRESS_NONE);
+  for (i = 0; i < accept_encoding_parts.count; i++) {
+    grpc_slice accept_encoding_entry_slice = accept_encoding_parts.slices[i];
+    if (grpc_stream_compression_algorithm_parse(accept_encoding_entry_slice,
+                                                &algorithm)) {
+      GPR_BITSET(&call->stream_encodings_accepted_by_peer, algorithm);
+    } else {
+      char *accept_encoding_entry_str =
+          grpc_slice_to_c_string(accept_encoding_entry_slice);
+      gpr_log(GPR_ERROR,
+              "Invalid entry in accept encoding metadata: '%s'. Ignoring.",
+              accept_encoding_entry_str);
+      gpr_free(accept_encoding_entry_str);
+    }
+  }
+
+  grpc_slice_buffer_destroy_internal(exec_ctx, &accept_encoding_parts);
+
+  grpc_mdelem_set_user_data(
+      mdel, destroy_encodings_accepted_by_peer,
+      (void *)(((uintptr_t)call->stream_encodings_accepted_by_peer) + 1));
+}
+
 uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call) {
   uint32_t encodings_accepted_by_peer;
   encodings_accepted_by_peer = call->encodings_accepted_by_peer;
   return encodings_accepted_by_peer;
 }
 
-static grpc_linked_mdelem *linked_from_md(grpc_metadata *md) {
+uint32_t grpc_call_test_only_get_stream_encodings_accepted_by_peer(
+    grpc_call *call) {
+  uint32_t stream_encodings_accepted_by_peer;
+  stream_encodings_accepted_by_peer = call->stream_encodings_accepted_by_peer;
+  return stream_encodings_accepted_by_peer;
+}
+
+grpc_stream_compression_algorithm
+grpc_call_test_only_get_incoming_stream_encodings(grpc_call *call) {
+  return call->incoming_stream_compression_algorithm;
+}
+
+static grpc_linked_mdelem *linked_from_md(const grpc_metadata *md) {
   return (grpc_linked_mdelem *)&md->internal_data;
 }
 
@@ -853,7 +986,7 @@ static int prepare_application_metadata(
   for (i = 0; i < total_count; i++) {
     const grpc_metadata *md =
         get_md_elem(metadata, additional_metadata, i, count);
-    grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
+    grpc_linked_mdelem *l = linked_from_md(md);
     GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data));
     if (!GRPC_LOG_IF_ERROR("validate_metadata",
                            grpc_validate_header_key_is_legal(md->key))) {
@@ -870,7 +1003,7 @@ static int prepare_application_metadata(
     for (int j = 0; j < i; j++) {
       const grpc_metadata *md =
           get_md_elem(metadata, additional_metadata, j, count);
-      grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
+      grpc_linked_mdelem *l = linked_from_md(md);
       GRPC_MDELEM_UNREF(exec_ctx, l->md);
     }
     return 0;
@@ -888,9 +1021,12 @@ static int prepare_application_metadata(
   }
   for (i = 0; i < total_count; i++) {
     grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
-    GRPC_LOG_IF_ERROR(
-        "prepare_application_metadata",
-        grpc_metadata_batch_link_tail(exec_ctx, batch, linked_from_md(md)));
+    grpc_linked_mdelem *l = linked_from_md(md);
+    grpc_error *error = grpc_metadata_batch_link_tail(exec_ctx, batch, l);
+    if (error != GRPC_ERROR_NONE) {
+      GRPC_MDELEM_UNREF(exec_ctx, l->md);
+    }
+    GRPC_LOG_IF_ERROR("prepare_application_metadata", error);
   }
   call->send_extra_metadata_count = 0;
 
@@ -937,6 +1073,22 @@ static grpc_compression_algorithm decode_compression(grpc_mdelem md) {
   return algorithm;
 }
 
+static grpc_stream_compression_algorithm decode_stream_compression(
+    grpc_mdelem md) {
+  grpc_stream_compression_algorithm algorithm =
+      grpc_stream_compression_algorithm_from_slice(GRPC_MDVALUE(md));
+  if (algorithm == GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) {
+    char *md_c_str = grpc_slice_to_c_string(GRPC_MDVALUE(md));
+    gpr_log(GPR_ERROR,
+            "Invalid incoming stream compression algorithm: '%s'. Interpreting "
+            "incoming data as uncompressed.",
+            md_c_str);
+    gpr_free(md_c_str);
+    return GRPC_STREAM_COMPRESS_NONE;
+  }
+  return algorithm;
+}
+
 static void publish_app_metadata(grpc_call *call, grpc_metadata_batch *b,
                                  int is_trailing) {
   if (b->list.count == 0) return;
@@ -961,7 +1113,19 @@ static void publish_app_metadata(grpc_call *call, grpc_metadata_batch *b,
 
 static void recv_initial_filter(grpc_exec_ctx *exec_ctx, grpc_call *call,
                                 grpc_metadata_batch *b) {
-  if (b->idx.named.grpc_encoding != NULL) {
+  if (b->idx.named.content_encoding != NULL) {
+    if (b->idx.named.grpc_encoding != NULL) {
+      gpr_log(GPR_ERROR,
+              "Received both content-encoding and grpc-encoding header. "
+              "Ignoring grpc-encoding.");
+      grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_encoding);
+    }
+    GPR_TIMER_BEGIN("incoming_stream_compression_algorithm", 0);
+    set_incoming_stream_compression_algorithm(
+        call, decode_stream_compression(b->idx.named.content_encoding->md));
+    GPR_TIMER_END("incoming_stream_compression_algorithm", 0);
+    grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.content_encoding);
+  } else if (b->idx.named.grpc_encoding != NULL) {
     GPR_TIMER_BEGIN("incoming_compression_algorithm", 0);
     set_incoming_compression_algorithm(
         call, decode_compression(b->idx.named.grpc_encoding->md));
@@ -975,6 +1139,13 @@ static void recv_initial_filter(grpc_exec_ctx *exec_ctx, grpc_call *call,
     grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_accept_encoding);
     GPR_TIMER_END("encodings_accepted_by_peer", 0);
   }
+  if (b->idx.named.accept_encoding != NULL) {
+    GPR_TIMER_BEGIN("stream_encodings_accepted_by_peer", 0);
+    set_stream_encodings_accepted_by_peer(exec_ctx, call,
+                                          b->idx.named.accept_encoding->md);
+    grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.accept_encoding);
+    GPR_TIMER_END("stream_encodings_accepted_by_peer", 0);
+  }
   publish_app_metadata(call, b, false);
 }
 
@@ -1291,19 +1462,73 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
     cancel_with_error(exec_ctx, call, STATUS_FROM_SURFACE,
                       GRPC_ERROR_REF(error));
   }
-  if (call->has_initial_md_been_received || error != GRPC_ERROR_NONE ||
-      call->receiving_stream == NULL) {
+  /* If recv_state is RECV_NONE, we will save the batch_control
+   * object with rel_cas, and will not use it after the cas. Its corresponding
+   * acq_load is in receiving_initial_metadata_ready() */
+  if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL ||
+      !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) {
     process_data_after_md(exec_ctx, bctlp);
-  } else {
-    call->saved_receiving_stream_ready_bctlp = bctlp;
   }
 }
 
+// The recv_message_ready callback used when sending a batch containing
+// a recv_message op down the filter stack.  Yields the call combiner
+// before processing the received message.
+static void receiving_stream_ready_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                                    void *bctlp,
+                                                    grpc_error *error) {
+  batch_control *bctl = bctlp;
+  grpc_call *call = bctl->call;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, "recv_message_ready");
+  receiving_stream_ready(exec_ctx, bctlp, error);
+}
+
 static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
                                        batch_control *bctl) {
   grpc_call *call = bctl->call;
-  /* validate call->incoming_compression_algorithm */
-  if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) {
+  /* validate compression algorithms */
+  if (call->incoming_stream_compression_algorithm !=
+      GRPC_STREAM_COMPRESS_NONE) {
+    const grpc_stream_compression_algorithm algo =
+        call->incoming_stream_compression_algorithm;
+    char *error_msg = NULL;
+    const grpc_compression_options compression_options =
+        grpc_channel_compression_options(call->channel);
+    if (algo >= GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) {
+      gpr_asprintf(&error_msg,
+                   "Invalid stream compression algorithm value '%d'.", algo);
+      gpr_log(GPR_ERROR, "%s", error_msg);
+      cancel_with_status(exec_ctx, call, STATUS_FROM_SURFACE,
+                         GRPC_STATUS_UNIMPLEMENTED, error_msg);
+    } else if (grpc_compression_options_is_stream_compression_algorithm_enabled(
+                   &compression_options, algo) == 0) {
+      /* check if algorithm is supported by current channel config */
+      char *algo_name = NULL;
+      grpc_stream_compression_algorithm_name(algo, &algo_name);
+      gpr_asprintf(&error_msg, "Stream compression algorithm '%s' is disabled.",
+                   algo_name);
+      gpr_log(GPR_ERROR, "%s", error_msg);
+      cancel_with_status(exec_ctx, call, STATUS_FROM_SURFACE,
+                         GRPC_STATUS_UNIMPLEMENTED, error_msg);
+    }
+    gpr_free(error_msg);
+
+    GPR_ASSERT(call->stream_encodings_accepted_by_peer != 0);
+    if (!GPR_BITGET(call->stream_encodings_accepted_by_peer,
+                    call->incoming_stream_compression_algorithm)) {
+      if (GRPC_TRACER_ON(grpc_compression_trace)) {
+        char *algo_name = NULL;
+        grpc_stream_compression_algorithm_name(
+            call->incoming_stream_compression_algorithm, &algo_name);
+        gpr_log(
+            GPR_ERROR,
+            "Stream compression algorithm (content-encoding = '%s') not "
+            "present in the bitset of accepted encodings (accept-encodings: "
+            "'0x%x')",
+            algo_name, call->stream_encodings_accepted_by_peer);
+      }
+    }
+  } else if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) {
     const grpc_compression_algorithm algo =
         call->incoming_compression_algorithm;
     char *error_msg = NULL;
@@ -1330,22 +1555,20 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
       call->incoming_compression_algorithm = algo;
     }
     gpr_free(error_msg);
-  }
 
-  /* make sure the received grpc-encoding is amongst the ones listed in
-   * grpc-accept-encoding */
-  GPR_ASSERT(call->encodings_accepted_by_peer != 0);
-  if (!GPR_BITGET(call->encodings_accepted_by_peer,
-                  call->incoming_compression_algorithm)) {
-    if (GRPC_TRACER_ON(grpc_compression_trace)) {
-      char *algo_name = NULL;
-      grpc_compression_algorithm_name(call->incoming_compression_algorithm,
-                                      &algo_name);
-      gpr_log(GPR_ERROR,
-              "Compression algorithm (grpc-encoding = '%s') not present in "
-              "the bitset of accepted encodings (grpc-accept-encodings: "
-              "'0x%x')",
-              algo_name, call->encodings_accepted_by_peer);
+    GPR_ASSERT(call->encodings_accepted_by_peer != 0);
+    if (!GPR_BITGET(call->encodings_accepted_by_peer,
+                    call->incoming_compression_algorithm)) {
+      if (GRPC_TRACER_ON(grpc_compression_trace)) {
+        char *algo_name = NULL;
+        grpc_compression_algorithm_name(call->incoming_compression_algorithm,
+                                        &algo_name);
+        gpr_log(GPR_ERROR,
+                "Compression algorithm (grpc-encoding = '%s') not present in "
+                "the bitset of accepted encodings (grpc-accept-encodings: "
+                "'0x%x')",
+                algo_name, call->encodings_accepted_by_peer);
+      }
     }
   }
 }
@@ -1366,6 +1589,9 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
 
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner,
+                          "recv_initial_metadata_ready");
+
   add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false);
   if (error == GRPC_ERROR_NONE) {
     grpc_metadata_batch *md =
@@ -1385,12 +1611,31 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
     }
   }
 
-  call->has_initial_md_been_received = true;
-  if (call->saved_receiving_stream_ready_bctlp != NULL) {
-    grpc_closure *saved_rsr_closure = GRPC_CLOSURE_CREATE(
-        receiving_stream_ready, call->saved_receiving_stream_ready_bctlp,
-        grpc_schedule_on_exec_ctx);
-    call->saved_receiving_stream_ready_bctlp = NULL;
+  grpc_closure *saved_rsr_closure = NULL;
+  while (true) {
+    gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state);
+    /* Should only receive initial metadata once */
+    GPR_ASSERT(rsr_bctlp != 1);
+    if (rsr_bctlp == 0) {
+      /* We haven't seen initial metadata and messages before, thus initial
+       * metadata is received first.
+       * no_barrier_cas is used, as this function won't access the batch_control
+       * object saved by receiving_stream_ready() if the initial metadata is
+       * received first. */
+      if (gpr_atm_no_barrier_cas(&call->recv_state, RECV_NONE,
+                                 RECV_INITIAL_METADATA_FIRST)) {
+        break;
+      }
+    } else {
+      /* Already received messages */
+      saved_rsr_closure = GRPC_CLOSURE_CREATE(receiving_stream_ready,
+                                              (batch_control *)rsr_bctlp,
+                                              grpc_schedule_on_exec_ctx);
+      /* No need to modify recv_state */
+      break;
+    }
+  }
+  if (saved_rsr_closure != NULL) {
     GRPC_CLOSURE_RUN(exec_ctx, saved_rsr_closure, GRPC_ERROR_REF(error));
   }
 
@@ -1400,7 +1645,8 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
 static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp,
                          grpc_error *error) {
   batch_control *bctl = bctlp;
-
+  grpc_call *call = bctl->call;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, "on_complete");
   add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false);
   finish_batch_step(exec_ctx, bctl);
 }
@@ -1420,15 +1666,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
   int num_completion_callbacks_needed = 1;
   grpc_call_error error = GRPC_CALL_OK;
 
-  // sent_initial_metadata guards against variable reuse.
-  grpc_metadata compression_md;
-
   GPR_TIMER_BEGIN("grpc_call_start_batch", 0);
   GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag);
 
   if (nops == 0) {
     if (!is_notify_tag_closure) {
-      grpc_cq_begin_op(call->cq, notify_tag);
+      GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag));
       grpc_cq_end_op(exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE,
                      free_no_op_completion, NULL,
                      gpr_malloc(sizeof(grpc_cq_completion)));
@@ -1470,31 +1713,60 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           goto done_with_error;
         }
         /* process compression level */
-        memset(&compression_md, 0, sizeof(compression_md));
+        memset(&call->compression_md, 0, sizeof(call->compression_md));
         size_t additional_metadata_count = 0;
-        grpc_compression_level effective_compression_level;
+        grpc_compression_level effective_compression_level =
+            GRPC_COMPRESS_LEVEL_NONE;
+        grpc_stream_compression_level effective_stream_compression_level =
+            GRPC_STREAM_COMPRESS_LEVEL_NONE;
         bool level_set = false;
-        if (op->data.send_initial_metadata.maybe_compression_level.is_set) {
+        bool stream_compression = false;
+        if (op->data.send_initial_metadata.maybe_stream_compression_level
+                .is_set) {
+          effective_stream_compression_level =
+              op->data.send_initial_metadata.maybe_stream_compression_level
+                  .level;
+          level_set = true;
+          stream_compression = true;
+        } else if (op->data.send_initial_metadata.maybe_compression_level
+                       .is_set) {
           effective_compression_level =
               op->data.send_initial_metadata.maybe_compression_level.level;
           level_set = true;
         } else {
           const grpc_compression_options copts =
               grpc_channel_compression_options(call->channel);
-          level_set = copts.default_level.is_set;
-          if (level_set) {
+          if (copts.default_stream_compression_level.is_set) {
+            level_set = true;
+            effective_stream_compression_level =
+                copts.default_stream_compression_level.level;
+            stream_compression = true;
+          } else if (copts.default_level.is_set) {
+            level_set = true;
             effective_compression_level = copts.default_level.level;
           }
         }
         if (level_set && !call->is_client) {
-          const grpc_compression_algorithm calgo =
-              compression_algorithm_for_level_locked(
-                  call, effective_compression_level);
-          // the following will be picked up by the compress filter and used as
-          // the call's compression algorithm.
-          compression_md.key = GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST;
-          compression_md.value = grpc_compression_algorithm_slice(calgo);
-          additional_metadata_count++;
+          if (stream_compression) {
+            const grpc_stream_compression_algorithm calgo =
+                stream_compression_algorithm_for_level_locked(
+                    call, effective_stream_compression_level);
+            call->compression_md.key =
+                GRPC_MDSTR_GRPC_INTERNAL_STREAM_ENCODING_REQUEST;
+            call->compression_md.value =
+                grpc_stream_compression_algorithm_slice(calgo);
+          } else {
+            const grpc_compression_algorithm calgo =
+                compression_algorithm_for_level_locked(
+                    call, effective_compression_level);
+            /* the following will be picked up by the compress filter and used
+             * as the call's compression algorithm. */
+            call->compression_md.key =
+                GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST;
+            call->compression_md.value =
+                grpc_compression_algorithm_slice(calgo);
+            additional_metadata_count++;
+          }
         }
 
         if (op->data.send_initial_metadata.count + additional_metadata_count >
@@ -1507,7 +1779,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         if (!prepare_application_metadata(
                 exec_ctx, call, (int)op->data.send_initial_metadata.count,
                 op->data.send_initial_metadata.metadata, 0, call->is_client,
-                &compression_md, (int)additional_metadata_count)) {
+                &call->compression_md, (int)additional_metadata_count)) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1519,6 +1791,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
             &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */];
         stream_op_payload->send_initial_metadata.send_initial_metadata_flags =
             op->flags;
+        if (call->is_client) {
+          stream_op_payload->send_initial_metadata.peer_string =
+              &call->peer_string;
+        }
         break;
       case GRPC_OP_SEND_MESSAGE:
         if (!are_write_flags_valid(op->flags)) {
@@ -1651,6 +1927,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
             &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
         stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
             &call->receiving_initial_metadata_ready;
+        if (!call->is_client) {
+          stream_op_payload->recv_initial_metadata.peer_string =
+              &call->peer_string;
+        }
         num_completion_callbacks_needed++;
         break;
       case GRPC_OP_RECV_MESSAGE:
@@ -1667,8 +1947,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         stream_op->recv_message = true;
         call->receiving_buffer = op->data.recv_message.recv_message;
         stream_op_payload->recv_message.recv_message = &call->receiving_stream;
-        GRPC_CLOSURE_INIT(&call->receiving_stream_ready, receiving_stream_ready,
-                          bctl, grpc_schedule_on_exec_ctx);
+        GRPC_CLOSURE_INIT(&call->receiving_stream_ready,
+                          receiving_stream_ready_in_call_combiner, bctl,
+                          grpc_schedule_on_exec_ctx);
         stream_op_payload->recv_message.recv_message_ready =
             &call->receiving_stream_ready;
         num_completion_callbacks_needed++;
@@ -1729,7 +2010,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
 
   GRPC_CALL_INTERNAL_REF(call, "completion");
   if (!is_notify_tag_closure) {
-    grpc_cq_begin_op(call->cq, notify_tag);
+    GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag));
   }
   gpr_ref_init(&bctl->steps_to_complete, num_completion_callbacks_needed);
 
@@ -1738,7 +2019,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
   stream_op->on_complete = &bctl->finish_batch;
   gpr_atm_rel_store(&call->any_ops_sent_atm, 1);
 
-  execute_op(exec_ctx, call, stream_op);
+  execute_batch(exec_ctx, call, stream_op, &bctl->start_batch);
 
 done:
   GPR_TIMER_END("grpc_call_start_batch", 0);
@@ -1850,6 +2131,8 @@ const char *grpc_call_error_to_string(grpc_call_error error) {
       return "GRPC_CALL_ERROR_PAYLOAD_TYPE_MISMATCH";
     case GRPC_CALL_ERROR_TOO_MANY_OPERATIONS:
       return "GRPC_CALL_ERROR_TOO_MANY_OPERATIONS";
+    case GRPC_CALL_ERROR_COMPLETION_QUEUE_SHUTDOWN:
+      return "GRPC_CALL_ERROR_COMPLETION_QUEUE_SHUTDOWN";
     case GRPC_CALL_OK:
       return "GRPC_CALL_OK";
   }
diff --git a/src/core/lib/surface/call_test_only.h b/src/core/lib/surface/call_test_only.h
index 2f1b80bfd7..a5a01b3679 100644
--- a/src/core/lib/surface/call_test_only.h
+++ b/src/core/lib/surface/call_test_only.h
@@ -42,6 +42,18 @@ uint32_t grpc_call_test_only_get_message_flags(grpc_call *call);
  * To be indexed by grpc_compression_algorithm enum values. */
 uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call);
 
+/** Returns a bitset for the stream encodings (stream compression algorithms)
+ * supported by \a call's peer.
+ *
+ * To be indexed by grpc_stream_compression_algorithm enum values. */
+uint32_t grpc_call_test_only_get_stream_encodings_accepted_by_peer(
+    grpc_call *call);
+
+/** Returns the incoming stream compression algorithm (content-encoding header)
+ * received by a call. */
+grpc_stream_compression_algorithm
+grpc_call_test_only_get_incoming_stream_encodings(grpc_call *call);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c
index 5780a18ce8..850fbe6a69 100644
--- a/src/core/lib/surface/channel.c
+++ b/src/core/lib/surface/channel.c
@@ -142,6 +142,16 @@ grpc_channel *grpc_channel_create_with_builder(
                                      GRPC_COMPRESS_LEVEL_NONE,
                                      GRPC_COMPRESS_LEVEL_COUNT - 1});
     } else if (0 == strcmp(args->args[i].key,
+                           GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) {
+      channel->compression_options.default_stream_compression_level.is_set =
+          true;
+      channel->compression_options.default_stream_compression_level.level =
+          (grpc_stream_compression_level)grpc_channel_arg_get_integer(
+              &args->args[i],
+              (grpc_integer_options){GRPC_STREAM_COMPRESS_LEVEL_NONE,
+                                     GRPC_STREAM_COMPRESS_LEVEL_NONE,
+                                     GRPC_STREAM_COMPRESS_LEVEL_COUNT - 1});
+    } else if (0 == strcmp(args->args[i].key,
                            GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) {
       channel->compression_options.default_algorithm.is_set = true;
       channel->compression_options.default_algorithm.algorithm =
@@ -149,12 +159,31 @@ grpc_channel *grpc_channel_create_with_builder(
               &args->args[i],
               (grpc_integer_options){GRPC_COMPRESS_NONE, GRPC_COMPRESS_NONE,
                                      GRPC_COMPRESS_ALGORITHMS_COUNT - 1});
+    } else if (0 == strcmp(args->args[i].key,
+                           GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) {
+      channel->compression_options.default_stream_compression_algorithm.is_set =
+          true;
+      channel->compression_options.default_stream_compression_algorithm
+          .algorithm =
+          (grpc_stream_compression_algorithm)grpc_channel_arg_get_integer(
+              &args->args[i],
+              (grpc_integer_options){
+                  GRPC_STREAM_COMPRESS_NONE, GRPC_STREAM_COMPRESS_NONE,
+                  GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT - 1});
     } else if (0 ==
                strcmp(args->args[i].key,
                       GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) {
       channel->compression_options.enabled_algorithms_bitset =
           (uint32_t)args->args[i].value.integer |
           0x1; /* always support no compression */
+    } else if (0 ==
+               strcmp(
+                   args->args[i].key,
+                   GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) {
+      channel->compression_options
+          .enabled_stream_compression_algorithms_bitset =
+          (uint32_t)args->args[i].value.integer |
+          0x1; /* always support no compression */
     }
   }
 
diff --git a/src/core/lib/surface/channel_ping.c b/src/core/lib/surface/channel_ping.c
index 80eb80af78..e85b308850 100644
--- a/src/core/lib/surface/channel_ping.c
+++ b/src/core/lib/surface/channel_ping.c
@@ -59,7 +59,7 @@ void grpc_channel_ping(grpc_channel *channel, grpc_completion_queue *cq,
   GRPC_CLOSURE_INIT(&pr->closure, ping_done, pr, grpc_schedule_on_exec_ctx);
   op->send_ping = &pr->closure;
   op->bind_pollset = grpc_cq_pollset(cq);
-  grpc_cq_begin_op(cq, tag);
+  GPR_ASSERT(grpc_cq_begin_op(cq, tag));
   top_elem->filter->start_transport_op(&exec_ctx, top_elem, op);
   grpc_exec_ctx_finish(&exec_ctx);
 }
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index 978d7b4171..10e4e5ab0c 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -196,7 +196,7 @@ typedef struct cq_vtable {
   void (*init)(void *data);
   void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq);
   void (*destroy)(void *data);
-  void (*begin_op)(grpc_completion_queue *cq, void *tag);
+  bool (*begin_op)(grpc_completion_queue *cq, void *tag);
   void (*end_op)(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq, void *tag,
                  grpc_error *error,
                  void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
@@ -235,7 +235,8 @@ typedef struct cq_next_data {
   /* Number of outstanding events (+1 if not shut down) */
   gpr_atm pending_events;
 
-  int shutdown_called;
+  /** 0 initially. 1 once we initiated shutdown */
+  bool shutdown_called;
 } cq_next_data;
 
 typedef struct cq_pluck_data {
@@ -244,15 +245,20 @@ typedef struct cq_pluck_data {
   grpc_cq_completion *completed_tail;
 
   /** Number of pending events (+1 if we're not shutdown) */
-  gpr_refcount pending_events;
+  gpr_atm pending_events;
 
   /** Counter of how many things have ever been queued on this completion queue
       useful for avoiding locks to check the queue */
   gpr_atm things_queued_ever;
 
-  /** 0 initially, 1 once we've begun shutting down */
+  /** 0 initially. 1 once we completed shutting */
+  /* TODO: (sreek) This is not needed since (shutdown == 1) if and only if
+   * (pending_events == 0). So consider removing this in future and use
+   * pending_events */
   gpr_atm shutdown;
-  int shutdown_called;
+
+  /** 0 initially. 1 once we initiated shutdown */
+  bool shutdown_called;
 
   int num_pluckers;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
@@ -288,8 +294,8 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx,
 static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx,
                               grpc_completion_queue *cq);
 
-static void cq_begin_op_for_next(grpc_completion_queue *cq, void *tag);
-static void cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag);
+static bool cq_begin_op_for_next(grpc_completion_queue *cq, void *tag);
+static bool cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag);
 
 static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
                                grpc_completion_queue *cq, void *tag,
@@ -436,7 +442,7 @@ grpc_completion_queue *grpc_completion_queue_create_internal(
 
 static void cq_init_next(void *ptr) {
   cq_next_data *cqd = ptr;
-  /* Initial ref is dropped by grpc_completion_queue_shutdown */
+  /* Initial count is dropped by grpc_completion_queue_shutdown */
   gpr_atm_no_barrier_store(&cqd->pending_events, 1);
   cqd->shutdown_called = false;
   gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
@@ -451,12 +457,12 @@ static void cq_destroy_next(void *ptr) {
 
 static void cq_init_pluck(void *ptr) {
   cq_pluck_data *cqd = ptr;
-  /* Initial ref is dropped by grpc_completion_queue_shutdown */
-  gpr_ref_init(&cqd->pending_events, 1);
+  /* Initial count is dropped by grpc_completion_queue_shutdown */
+  gpr_atm_no_barrier_store(&cqd->pending_events, 1);
   cqd->completed_tail = &cqd->completed_head;
   cqd->completed_head.next = (uintptr_t)cqd->completed_tail;
   gpr_atm_no_barrier_store(&cqd->shutdown, 0);
-  cqd->shutdown_called = 0;
+  cqd->shutdown_called = false;
   cqd->num_pluckers = 0;
   gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
 }
@@ -522,33 +528,6 @@ void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static void cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) {
-  cq_next_data *cqd = DATA_FROM_CQ(cq);
-  GPR_ASSERT(!cqd->shutdown_called);
-  gpr_atm_no_barrier_fetch_add(&cqd->pending_events, 1);
-}
-
-static void cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag) {
-  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
-  GPR_ASSERT(!cqd->shutdown_called);
-  gpr_ref(&cqd->pending_events);
-}
-
-void grpc_cq_begin_op(grpc_completion_queue *cq, void *tag) {
-#ifndef NDEBUG
-  gpr_mu_lock(cq->mu);
-  if (cq->outstanding_tag_count == cq->outstanding_tag_capacity) {
-    cq->outstanding_tag_capacity = GPR_MAX(4, 2 * cq->outstanding_tag_capacity);
-    cq->outstanding_tags =
-        gpr_realloc(cq->outstanding_tags, sizeof(*cq->outstanding_tags) *
-                                              cq->outstanding_tag_capacity);
-  }
-  cq->outstanding_tags[cq->outstanding_tag_count++] = tag;
-  gpr_mu_unlock(cq->mu);
-#endif
-  cq->vtable->begin_op(cq, tag);
-}
-
 #ifndef NDEBUG
 static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {
   int found = 0;
@@ -576,6 +555,49 @@ static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {
 static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {}
 #endif
 
+/* Atomically increments a counter only if the counter is not zero. Returns
+ * true if the increment was successful; false if the counter is zero */
+static bool atm_inc_if_nonzero(gpr_atm *counter) {
+  while (true) {
+    gpr_atm count = gpr_atm_no_barrier_load(counter);
+    /* If zero, we are done. If not, we must to a CAS (instead of an atomic
+     * increment) to maintain the contract: do not increment the counter if it
+     * is zero. */
+    if (count == 0) {
+      return false;
+    } else if (gpr_atm_no_barrier_cas(counter, count, count + 1)) {
+      break;
+    }
+  }
+
+  return true;
+}
+
+static bool cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) {
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
+  return atm_inc_if_nonzero(&cqd->pending_events);
+}
+
+static bool cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag) {
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
+  return atm_inc_if_nonzero(&cqd->pending_events);
+}
+
+bool grpc_cq_begin_op(grpc_completion_queue *cq, void *tag) {
+#ifndef NDEBUG
+  gpr_mu_lock(cq->mu);
+  if (cq->outstanding_tag_count == cq->outstanding_tag_capacity) {
+    cq->outstanding_tag_capacity = GPR_MAX(4, 2 * cq->outstanding_tag_capacity);
+    cq->outstanding_tags =
+        gpr_realloc(cq->outstanding_tags, sizeof(*cq->outstanding_tags) *
+                                              cq->outstanding_tag_capacity);
+  }
+  cq->outstanding_tags[cq->outstanding_tag_count++] = tag;
+  gpr_mu_unlock(cq->mu);
+#endif
+  return cq->vtable->begin_op(cq, tag);
+}
+
 /* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
  * completion
  * type of GRPC_CQ_NEXT) */
@@ -696,8 +718,10 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
       ((uintptr_t)storage) | (1u & (uintptr_t)cqd->completed_tail->next);
   cqd->completed_tail = storage;
 
-  int shutdown = gpr_unref(&cqd->pending_events);
-  if (!shutdown) {
+  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+    cq_finish_shutdown_pluck(exec_ctx, cq);
+    gpr_mu_unlock(cq->mu);
+  } else {
     grpc_pollset_worker *pluck_worker = NULL;
     for (int i = 0; i < cqd->num_pluckers; i++) {
       if (cqd->pluckers[i].tag == tag) {
@@ -717,9 +741,6 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
 
       GRPC_ERROR_UNREF(kick_error);
     }
-  } else {
-    cq_finish_shutdown_pluck(exec_ctx, cq);
-    gpr_mu_unlock(cq->mu);
   }
 
   GPR_TIMER_END("cq_end_op_for_pluck", 0);
@@ -855,8 +876,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline,
          inconsistent state. If it is the latter, we shold do a 0-timeout poll
          so that the thread comes back quickly from poll to make a second
          attempt at popping. Not doing this can potentially deadlock this
-         thread
-         forever (if the deadline is infinity) */
+         thread forever (if the deadline is infinity) */
       if (cq_event_queue_num_items(&cqd->queue) > 0) {
         iteration_deadline = gpr_time_0(GPR_CLOCK_MONOTONIC);
       }
@@ -869,10 +889,8 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline,
       if (cq_event_queue_num_items(&cqd->queue) > 0) {
         /* Go to the beginning of the loop. No point doing a poll because
            (cq->shutdown == true) is only possible when there is no pending
-           work
-           (i.e cq->pending_events == 0) and any outstanding
-           grpc_cq_completion
-           events are already queued on this cq */
+           work (i.e cq->pending_events == 0) and any outstanding completion
+           events should have already been queued on this cq */
         continue;
       }
 
@@ -909,11 +927,6 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline,
     is_finished_arg.first_loop = false;
   }
 
-  GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, &ret);
-  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cq, "next");
-  grpc_exec_ctx_finish(&exec_ctx);
-  GPR_ASSERT(is_finished_arg.stolen_completion == NULL);
-
   if (cq_event_queue_num_items(&cqd->queue) > 0 &&
       gpr_atm_no_barrier_load(&cqd->pending_events) > 0) {
     gpr_mu_lock(cq->mu);
@@ -921,6 +934,11 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline,
     gpr_mu_unlock(cq->mu);
   }
 
+  GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, &ret);
+  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cq, "next");
+  grpc_exec_ctx_finish(&exec_ctx);
+  GPR_ASSERT(is_finished_arg.stolen_completion == NULL);
+
   GPR_TIMER_END("grpc_completion_queue_next", 0);
 
   return ret;
@@ -947,15 +965,20 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx,
                              grpc_completion_queue *cq) {
   cq_next_data *cqd = DATA_FROM_CQ(cq);
 
+  /* Need an extra ref for cq here because:
+   * We call cq_finish_shutdown_next() below, that would call pollset shutdown.
+   * Pollset shutdown decrements the cq ref count which can potentially destroy
+   * the cq (if that happens to be the last ref).
+   * Creating an extra ref here prevents the cq from getting destroyed while
+   * this function is still active */
   GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
   gpr_mu_lock(cq->mu);
   if (cqd->shutdown_called) {
     gpr_mu_unlock(cq->mu);
     GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down");
-    GPR_TIMER_END("grpc_completion_queue_shutdown", 0);
     return;
   }
-  cqd->shutdown_called = 1;
+  cqd->shutdown_called = true;
   if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
     cq_finish_shutdown_next(exec_ctx, cq);
   }
@@ -1167,21 +1190,31 @@ static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx,
                               &cq->pollset_shutdown_done);
 }
 
+/* NOTE: This function is almost exactly identical to cq_shutdown_next() but
+ * merging them is a bit tricky and probably not worth it */
 static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx,
                               grpc_completion_queue *cq) {
   cq_pluck_data *cqd = DATA_FROM_CQ(cq);
 
+  /* Need an extra ref for cq here because:
+   * We call cq_finish_shutdown_pluck() below, that would call pollset shutdown.
+   * Pollset shutdown decrements the cq ref count which can potentially destroy
+   * the cq (if that happens to be the last ref).
+   * Creating an extra ref here prevents the cq from getting destroyed while
+   * this function is still active */
+  GRPC_CQ_INTERNAL_REF(cq, "shutting_down (pluck cq)");
   gpr_mu_lock(cq->mu);
   if (cqd->shutdown_called) {
     gpr_mu_unlock(cq->mu);
-    GPR_TIMER_END("grpc_completion_queue_shutdown", 0);
+    GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down (pluck cq)");
     return;
   }
-  cqd->shutdown_called = 1;
-  if (gpr_unref(&cqd->pending_events)) {
+  cqd->shutdown_called = true;
+  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
     cq_finish_shutdown_pluck(exec_ctx, cq);
   }
   gpr_mu_unlock(cq->mu);
+  GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down (pluck cq)");
 }
 
 /* Shutdown simply drops a ref that we reserved at creation time; if we drop
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index af44482513..69d144bd95 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -72,8 +72,9 @@ void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc);
 
 /* Flag that an operation is beginning: the completion channel will not finish
    shutdown until a corrensponding grpc_cq_end_* call is made.
-   \a tag is currently used only in debug builds. */
-void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag);
+   \a tag is currently used only in debug builds. Return true on success, and
+   false if completion_queue has been shutdown. */
+bool grpc_cq_begin_op(grpc_completion_queue *cc, void *tag);
 
 /* Queue a GRPC_OP_COMPLETED operation; tag must correspond to the tag passed to
    grpc_cq_begin_op */
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
index 822056af5a..280315036f 100644
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@@ -31,12 +31,14 @@
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/iomgr/resource_quota.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/surface/alarm_internal.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel_init.h"
@@ -128,6 +130,7 @@ void grpc_init(void) {
     grpc_register_tracer(&grpc_trace_channel_stack_builder);
     grpc_register_tracer(&grpc_http1_trace);
     grpc_register_tracer(&grpc_cq_pluck_trace);  // default on
+    grpc_register_tracer(&grpc_call_combiner_trace);
     grpc_register_tracer(&grpc_combiner_trace);
     grpc_register_tracer(&grpc_server_channel_trace);
     grpc_register_tracer(&grpc_bdp_estimator_trace);
@@ -137,6 +140,7 @@ void grpc_init(void) {
     grpc_register_tracer(&grpc_call_error_trace);
 #ifndef NDEBUG
     grpc_register_tracer(&grpc_trace_pending_tags);
+    grpc_register_tracer(&grpc_trace_alarm_refcount);
     grpc_register_tracer(&grpc_trace_cq_refcount);
     grpc_register_tracer(&grpc_trace_closure);
     grpc_register_tracer(&grpc_trace_error_refcount);
diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc
index a0791080a9..6286f9159d 100644
--- a/src/core/lib/surface/lame_client.cc
+++ b/src/core/lib/surface/lame_client.cc
@@ -40,6 +40,7 @@ namespace grpc_core {
 namespace {
 
 struct CallData {
+  grpc_call_combiner *call_combiner;
   grpc_linked_mdelem status;
   grpc_linked_mdelem details;
   grpc_core::atomic<bool> filled_metadata;
@@ -52,14 +53,14 @@ struct ChannelData {
 
 static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                           grpc_metadata_batch *mdb) {
-  CallData *calld = static_cast<CallData *>(elem->call_data);
+  CallData *calld = reinterpret_cast<CallData *>(elem->call_data);
   bool expected = false;
   if (!calld->filled_metadata.compare_exchange_strong(
           expected, true, grpc_core::memory_order_relaxed,
           grpc_core::memory_order_relaxed)) {
     return;
   }
-  ChannelData *chand = static_cast<ChannelData *>(elem->channel_data);
+  ChannelData *chand = reinterpret_cast<ChannelData *>(elem->channel_data);
   char tmp[GPR_LTOA_MIN_BUFSIZE];
   gpr_ltoa(chand->error_code, tmp);
   calld->status.md = grpc_mdelem_from_slices(
@@ -79,6 +80,7 @@ static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 static void lame_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *op) {
+  CallData *calld = reinterpret_cast<CallData *>(elem->call_data);
   if (op->recv_initial_metadata) {
     fill_metadata(exec_ctx, elem,
                   op->payload->recv_initial_metadata.recv_initial_metadata);
@@ -87,12 +89,8 @@ static void lame_start_transport_stream_op_batch(
                   op->payload->recv_trailing_metadata.recv_trailing_metadata);
   }
   grpc_transport_stream_op_batch_finish_with_failure(
-      exec_ctx, op,
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel"));
-}
-
-static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  return NULL;
+      exec_ctx, op, GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel"),
+      calld->call_combiner);
 }
 
 static void lame_get_channel_info(grpc_exec_ctx *exec_ctx,
@@ -122,6 +120,8 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
+  CallData *calld = reinterpret_cast<CallData *>(elem->call_data);
+  calld->call_combiner = args->call_combiner;
   return GRPC_ERROR_NONE;
 }
 
@@ -156,7 +156,6 @@ extern "C" const grpc_channel_filter grpc_lame_filter = {
     sizeof(grpc_core::ChannelData),
     grpc_core::init_channel_elem,
     grpc_core::destroy_channel_elem,
-    grpc_core::lame_get_peer,
     grpc_core::lame_get_channel_info,
     "lame-client",
 };
@@ -176,7 +175,7 @@ grpc_channel *grpc_lame_client_channel_create(const char *target,
       "error_message=%s)",
       3, (target, (int)error_code, error_message));
   GPR_ASSERT(elem->filter == &grpc_lame_filter);
-  auto chand = static_cast<grpc_core::ChannelData *>(elem->channel_data);
+  auto chand = reinterpret_cast<grpc_core::ChannelData *>(elem->channel_data);
   chand->error_code = error_code;
   chand->error_message = error_message;
   grpc_exec_ctx_finish(&exec_ctx);
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index fce7f8dca1..8582d826ca 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -789,7 +789,6 @@ static void server_mutate_op(grpc_call_element *elem,
 static void server_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
   server_mutate_op(elem, op);
   grpc_call_next_op(exec_ctx, elem, op);
 }
@@ -962,7 +961,6 @@ const grpc_channel_filter grpc_server_top_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "server",
 };
@@ -1259,7 +1257,7 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
   }
 
   /* stay locked, and gather up some stuff to do */
-  grpc_cq_begin_op(cq, tag);
+  GPR_ASSERT(grpc_cq_begin_op(cq, tag));
   if (server->shutdown_published) {
     grpc_cq_end_op(&exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown,
                    NULL, gpr_malloc(sizeof(grpc_cq_completion)));
@@ -1446,7 +1444,11 @@ grpc_call_error grpc_server_request_call(
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
   }
-  grpc_cq_begin_op(cq_for_notification, tag);
+  if (grpc_cq_begin_op(cq_for_notification, tag) == false) {
+    gpr_free(rc);
+    error = GRPC_CALL_ERROR_COMPLETION_QUEUE_SHUTDOWN;
+    goto done;
+  }
   details->reserved = NULL;
   rc->cq_idx = cq_idx;
   rc->type = BATCH_CALL;
@@ -1496,7 +1498,11 @@ grpc_call_error grpc_server_request_registered_call(
     error = GRPC_CALL_ERROR_PAYLOAD_TYPE_MISMATCH;
     goto done;
   }
-  grpc_cq_begin_op(cq_for_notification, tag);
+  if (grpc_cq_begin_op(cq_for_notification, tag) == false) {
+    gpr_free(rc);
+    error = GRPC_CALL_ERROR_COMPLETION_QUEUE_SHUTDOWN;
+    goto done;
+  }
   rc->cq_idx = cq_idx;
   rc->type = REGISTERED_CALL;
   rc->server = server;
diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c
index 8cef15da80..96c16105e7 100644
--- a/src/core/lib/surface/version.c
+++ b/src/core/lib/surface/version.c
@@ -23,4 +23,4 @@
 
 const char *grpc_version_string(void) { return "4.0.0-dev"; }
 
-const char *grpc_g_stands_for(void) { return "gregarious"; }
+const char *grpc_g_stands_for(void) { return "gambit"; }
diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c
index 311ae6390d..8b57693413 100644
--- a/src/core/lib/transport/bdp_estimator.c
+++ b/src/core/lib/transport/bdp_estimator.c
@@ -33,20 +33,24 @@ void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) {
   estimator->bw_est = 0;
 }
 
-bool grpc_bdp_estimator_get_estimate(grpc_bdp_estimator *estimator,
+bool grpc_bdp_estimator_get_estimate(const grpc_bdp_estimator *estimator,
                                      int64_t *estimate) {
   *estimate = estimator->estimate;
   return true;
 }
 
-bool grpc_bdp_estimator_get_bw(grpc_bdp_estimator *estimator, double *bw) {
+bool grpc_bdp_estimator_get_bw(const grpc_bdp_estimator *estimator,
+                               double *bw) {
   *bw = estimator->bw_est;
   return true;
 }
 
-bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator,
+void grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator,
                                            int64_t num_bytes) {
   estimator->accumulator += num_bytes;
+}
+
+bool grpc_bdp_estimator_need_ping(const grpc_bdp_estimator *estimator) {
   switch (estimator->ping_state) {
     case GRPC_BDP_PING_UNSCHEDULED:
       return true;
diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h
index a232d1f87f..1ef0dc99dd 100644
--- a/src/core/lib/transport/bdp_estimator.h
+++ b/src/core/lib/transport/bdp_estimator.h
@@ -47,13 +47,15 @@ typedef struct grpc_bdp_estimator {
 void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name);
 
 // Returns true if a reasonable estimate could be obtained
-bool grpc_bdp_estimator_get_estimate(grpc_bdp_estimator *estimator,
+bool grpc_bdp_estimator_get_estimate(const grpc_bdp_estimator *estimator,
                                      int64_t *estimate);
-// Returns true if a reasonable estimate could be obtained
-bool grpc_bdp_estimator_get_bw(grpc_bdp_estimator *estimator, double *bw);
+// Tracks new bytes read.
+bool grpc_bdp_estimator_get_bw(const grpc_bdp_estimator *estimator, double *bw);
 // Returns true if the user should schedule a ping
-bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator,
+void grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator,
                                            int64_t num_bytes);
+// Returns true if the user should schedule a ping
+bool grpc_bdp_estimator_need_ping(const grpc_bdp_estimator *estimator);
 // Schedule a ping: call in response to receiving a true from
 // grpc_bdp_estimator_add_incoming_bytes once a ping has been scheduled by a
 // transport (but not necessarily started)
diff --git a/src/core/lib/transport/byte_stream.c b/src/core/lib/transport/byte_stream.c
index 3355814017..fb03a10315 100644
--- a/src/core/lib/transport/byte_stream.c
+++ b/src/core/lib/transport/byte_stream.c
@@ -19,29 +19,37 @@
 #include "src/core/lib/transport/byte_stream.h"
 
 #include <stdlib.h>
+#include <string.h>
 
 #include <grpc/support/log.h>
 
 #include "src/core/lib/slice/slice_internal.h"
 
-int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx,
-                          grpc_byte_stream *byte_stream, size_t max_size_hint,
-                          grpc_closure *on_complete) {
-  return byte_stream->next(exec_ctx, byte_stream, max_size_hint, on_complete);
+bool grpc_byte_stream_next(grpc_exec_ctx *exec_ctx,
+                           grpc_byte_stream *byte_stream, size_t max_size_hint,
+                           grpc_closure *on_complete) {
+  return byte_stream->vtable->next(exec_ctx, byte_stream, max_size_hint,
+                                   on_complete);
 }
 
 grpc_error *grpc_byte_stream_pull(grpc_exec_ctx *exec_ctx,
                                   grpc_byte_stream *byte_stream,
                                   grpc_slice *slice) {
-  return byte_stream->pull(exec_ctx, byte_stream, slice);
+  return byte_stream->vtable->pull(exec_ctx, byte_stream, slice);
+}
+
+void grpc_byte_stream_shutdown(grpc_exec_ctx *exec_ctx,
+                               grpc_byte_stream *byte_stream,
+                               grpc_error *error) {
+  byte_stream->vtable->shutdown(exec_ctx, byte_stream, error);
 }
 
 void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
                               grpc_byte_stream *byte_stream) {
-  byte_stream->destroy(exec_ctx, byte_stream);
+  byte_stream->vtable->destroy(exec_ctx, byte_stream);
 }
 
-/* slice_buffer_stream */
+// grpc_slice_buffer_stream
 
 static bool slice_buffer_stream_next(grpc_exec_ctx *exec_ctx,
                                      grpc_byte_stream *byte_stream,
@@ -56,6 +64,9 @@ static grpc_error *slice_buffer_stream_pull(grpc_exec_ctx *exec_ctx,
                                             grpc_byte_stream *byte_stream,
                                             grpc_slice *slice) {
   grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream;
+  if (stream->shutdown_error != GRPC_ERROR_NONE) {
+    return GRPC_ERROR_REF(stream->shutdown_error);
+  }
   GPR_ASSERT(stream->cursor < stream->backing_buffer->count);
   *slice =
       grpc_slice_ref_internal(stream->backing_buffer->slices[stream->cursor]);
@@ -63,8 +74,23 @@ static grpc_error *slice_buffer_stream_pull(grpc_exec_ctx *exec_ctx,
   return GRPC_ERROR_NONE;
 }
 
+static void slice_buffer_stream_shutdown(grpc_exec_ctx *exec_ctx,
+                                         grpc_byte_stream *byte_stream,
+                                         grpc_error *error) {
+  grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream;
+  GRPC_ERROR_UNREF(stream->shutdown_error);
+  stream->shutdown_error = error;
+}
+
 static void slice_buffer_stream_destroy(grpc_exec_ctx *exec_ctx,
-                                        grpc_byte_stream *byte_stream) {}
+                                        grpc_byte_stream *byte_stream) {
+  grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream;
+  GRPC_ERROR_UNREF(stream->shutdown_error);
+}
+
+static const grpc_byte_stream_vtable slice_buffer_stream_vtable = {
+    slice_buffer_stream_next, slice_buffer_stream_pull,
+    slice_buffer_stream_shutdown, slice_buffer_stream_destroy};
 
 void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream,
                                    grpc_slice_buffer *slice_buffer,
@@ -72,9 +98,89 @@ void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream,
   GPR_ASSERT(slice_buffer->length <= UINT32_MAX);
   stream->base.length = (uint32_t)slice_buffer->length;
   stream->base.flags = flags;
-  stream->base.next = slice_buffer_stream_next;
-  stream->base.pull = slice_buffer_stream_pull;
-  stream->base.destroy = slice_buffer_stream_destroy;
+  stream->base.vtable = &slice_buffer_stream_vtable;
   stream->backing_buffer = slice_buffer;
   stream->cursor = 0;
+  stream->shutdown_error = GRPC_ERROR_NONE;
+}
+
+// grpc_caching_byte_stream
+
+void grpc_byte_stream_cache_init(grpc_byte_stream_cache *cache,
+                                 grpc_byte_stream *underlying_stream) {
+  cache->underlying_stream = underlying_stream;
+  grpc_slice_buffer_init(&cache->cache_buffer);
+}
+
+void grpc_byte_stream_cache_destroy(grpc_exec_ctx *exec_ctx,
+                                    grpc_byte_stream_cache *cache) {
+  grpc_byte_stream_destroy(exec_ctx, cache->underlying_stream);
+  grpc_slice_buffer_destroy_internal(exec_ctx, &cache->cache_buffer);
+}
+
+static bool caching_byte_stream_next(grpc_exec_ctx *exec_ctx,
+                                     grpc_byte_stream *byte_stream,
+                                     size_t max_size_hint,
+                                     grpc_closure *on_complete) {
+  grpc_caching_byte_stream *stream = (grpc_caching_byte_stream *)byte_stream;
+  if (stream->shutdown_error != GRPC_ERROR_NONE) return true;
+  if (stream->cursor < stream->cache->cache_buffer.count) return true;
+  return grpc_byte_stream_next(exec_ctx, stream->cache->underlying_stream,
+                               max_size_hint, on_complete);
+}
+
+static grpc_error *caching_byte_stream_pull(grpc_exec_ctx *exec_ctx,
+                                            grpc_byte_stream *byte_stream,
+                                            grpc_slice *slice) {
+  grpc_caching_byte_stream *stream = (grpc_caching_byte_stream *)byte_stream;
+  if (stream->shutdown_error != GRPC_ERROR_NONE) {
+    return GRPC_ERROR_REF(stream->shutdown_error);
+  }
+  if (stream->cursor < stream->cache->cache_buffer.count) {
+    *slice = grpc_slice_ref_internal(
+        stream->cache->cache_buffer.slices[stream->cursor]);
+    ++stream->cursor;
+    return GRPC_ERROR_NONE;
+  }
+  grpc_error *error =
+      grpc_byte_stream_pull(exec_ctx, stream->cache->underlying_stream, slice);
+  if (error == GRPC_ERROR_NONE) {
+    ++stream->cursor;
+    grpc_slice_buffer_add(&stream->cache->cache_buffer,
+                          grpc_slice_ref_internal(*slice));
+  }
+  return error;
+}
+
+static void caching_byte_stream_shutdown(grpc_exec_ctx *exec_ctx,
+                                         grpc_byte_stream *byte_stream,
+                                         grpc_error *error) {
+  grpc_caching_byte_stream *stream = (grpc_caching_byte_stream *)byte_stream;
+  GRPC_ERROR_UNREF(stream->shutdown_error);
+  stream->shutdown_error = GRPC_ERROR_REF(error);
+  grpc_byte_stream_shutdown(exec_ctx, stream->cache->underlying_stream, error);
+}
+
+static void caching_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
+                                        grpc_byte_stream *byte_stream) {
+  grpc_caching_byte_stream *stream = (grpc_caching_byte_stream *)byte_stream;
+  GRPC_ERROR_UNREF(stream->shutdown_error);
+}
+
+static const grpc_byte_stream_vtable caching_byte_stream_vtable = {
+    caching_byte_stream_next, caching_byte_stream_pull,
+    caching_byte_stream_shutdown, caching_byte_stream_destroy};
+
+void grpc_caching_byte_stream_init(grpc_caching_byte_stream *stream,
+                                   grpc_byte_stream_cache *cache) {
+  memset(stream, 0, sizeof(*stream));
+  stream->base.length = cache->underlying_stream->length;
+  stream->base.flags = cache->underlying_stream->flags;
+  stream->base.vtable = &caching_byte_stream_vtable;
+  stream->cache = cache;
+  stream->shutdown_error = GRPC_ERROR_NONE;
+}
+
+void grpc_caching_byte_stream_reset(grpc_caching_byte_stream *stream) {
+  stream->cursor = 0;
 }
diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h
index f172296e4b..1e1e8310b8 100644
--- a/src/core/lib/transport/byte_stream.h
+++ b/src/core/lib/transport/byte_stream.h
@@ -28,52 +28,109 @@
 /** Mask of all valid internal flags. */
 #define GRPC_WRITE_INTERNAL_USED_MASK (GRPC_WRITE_INTERNAL_COMPRESS)
 
-struct grpc_byte_stream;
 typedef struct grpc_byte_stream grpc_byte_stream;
 
-struct grpc_byte_stream {
-  uint32_t length;
-  uint32_t flags;
+typedef struct {
   bool (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream,
                size_t max_size_hint, grpc_closure *on_complete);
   grpc_error *(*pull)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream,
                       grpc_slice *slice);
+  void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream,
+                   grpc_error *error);
   void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream);
+} grpc_byte_stream_vtable;
+
+struct grpc_byte_stream {
+  uint32_t length;
+  uint32_t flags;
+  const grpc_byte_stream_vtable *vtable;
 };
 
-/* returns 1 if the bytes are available immediately (in which case
- * on_complete will not be called), 0 if the bytes will be available
- * asynchronously.
- *
- * max_size_hint can be set as a hint as to the maximum number
- * of bytes that would be acceptable to read.
- */
-int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx,
-                          grpc_byte_stream *byte_stream, size_t max_size_hint,
-                          grpc_closure *on_complete);
+// Returns true if the bytes are available immediately (in which case
+// on_complete will not be called), false if the bytes will be available
+// asynchronously.
+//
+// max_size_hint can be set as a hint as to the maximum number
+// of bytes that would be acceptable to read.
+bool grpc_byte_stream_next(grpc_exec_ctx *exec_ctx,
+                           grpc_byte_stream *byte_stream, size_t max_size_hint,
+                           grpc_closure *on_complete);
 
-/* returns the next slice in the byte stream when it is ready (indicated by
- * either grpc_byte_stream_next returning 1 or on_complete passed to
- * grpc_byte_stream_next is called).
- *
- * once a slice is returned into *slice, it is owned by the caller.
- */
+// Returns the next slice in the byte stream when it is ready (indicated by
+// either grpc_byte_stream_next returning true or on_complete passed to
+// grpc_byte_stream_next is called).
+//
+// Once a slice is returned into *slice, it is owned by the caller.
 grpc_error *grpc_byte_stream_pull(grpc_exec_ctx *exec_ctx,
                                   grpc_byte_stream *byte_stream,
                                   grpc_slice *slice);
 
+// Shuts down the byte stream.
+//
+// If there is a pending call to on_complete from grpc_byte_stream_next(),
+// it will be invoked with the error passed to grpc_byte_stream_shutdown().
+//
+// The next call to grpc_byte_stream_pull() (if any) will return the error
+// passed to grpc_byte_stream_shutdown().
+void grpc_byte_stream_shutdown(grpc_exec_ctx *exec_ctx,
+                               grpc_byte_stream *byte_stream,
+                               grpc_error *error);
+
 void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
                               grpc_byte_stream *byte_stream);
 
-/* grpc_byte_stream that wraps a slice buffer */
+// grpc_slice_buffer_stream
+//
+// A grpc_byte_stream that wraps a slice buffer.
+
 typedef struct grpc_slice_buffer_stream {
   grpc_byte_stream base;
   grpc_slice_buffer *backing_buffer;
   size_t cursor;
+  grpc_error *shutdown_error;
 } grpc_slice_buffer_stream;
 
 void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream,
                                    grpc_slice_buffer *slice_buffer,
                                    uint32_t flags);
 
+// grpc_caching_byte_stream
+//
+// A grpc_byte_stream that that wraps an underlying byte stream but caches
+// the resulting slices in a slice buffer.  If an initial attempt fails
+// without fully draining the underlying stream, a new caching stream
+// can be created from the same underlying cache, in which case it will
+// return whatever is in the backing buffer before continuing to read the
+// underlying stream.
+//
+// NOTE: No synchronization is done, so it is not safe to have multiple
+// grpc_caching_byte_streams simultaneously drawing from the same underlying
+// grpc_byte_stream_cache at the same time.
+
+typedef struct {
+  grpc_byte_stream *underlying_stream;
+  grpc_slice_buffer cache_buffer;
+} grpc_byte_stream_cache;
+
+// Takes ownership of underlying_stream.
+void grpc_byte_stream_cache_init(grpc_byte_stream_cache *cache,
+                                 grpc_byte_stream *underlying_stream);
+
+// Must not be called while still in use by a grpc_caching_byte_stream.
+void grpc_byte_stream_cache_destroy(grpc_exec_ctx *exec_ctx,
+                                    grpc_byte_stream_cache *cache);
+
+typedef struct {
+  grpc_byte_stream base;
+  grpc_byte_stream_cache *cache;
+  size_t cursor;
+  grpc_error *shutdown_error;
+} grpc_caching_byte_stream;
+
+void grpc_caching_byte_stream_init(grpc_caching_byte_stream *stream,
+                                   grpc_byte_stream_cache *cache);
+
+// Resets the byte stream to the start of the underlying stream.
+void grpc_caching_byte_stream_reset(grpc_caching_byte_stream *stream);
+
 #endif /* GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H */
diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c
index 2388f19f81..28f05d5c44 100644
--- a/src/core/lib/transport/static_metadata.c
+++ b/src/core/lib/transport/static_metadata.c
@@ -40,65 +40,68 @@ static uint8_t g_bytes[] = {
     114, 45,  115, 116, 97,  116, 115, 45,  98,  105, 110, 103, 114, 112, 99,
     45,  116, 97,  103, 115, 45,  98,  105, 110, 103, 114, 112, 99,  45,  116,
     114, 97,  99,  101, 45,  98,  105, 110, 99,  111, 110, 116, 101, 110, 116,
-    45,  116, 121, 112, 101, 103, 114, 112, 99,  45,  105, 110, 116, 101, 114,
-    110, 97,  108, 45,  101, 110, 99,  111, 100, 105, 110, 103, 45,  114, 101,
-    113, 117, 101, 115, 116, 117, 115, 101, 114, 45,  97,  103, 101, 110, 116,
-    104, 111, 115, 116, 108, 98,  45,  116, 111, 107, 101, 110, 103, 114, 112,
-    99,  45,  116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99,  46,  119,
-    97,  105, 116, 95,  102, 111, 114, 95,  114, 101, 97,  100, 121, 103, 114,
-    112, 99,  46,  116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99,  46,
-    109, 97,  120, 95,  114, 101, 113, 117, 101, 115, 116, 95,  109, 101, 115,
-    115, 97,  103, 101, 95,  98,  121, 116, 101, 115, 103, 114, 112, 99,  46,
-    109, 97,  120, 95,  114, 101, 115, 112, 111, 110, 115, 101, 95,  109, 101,
-    115, 115, 97,  103, 101, 95,  98,  121, 116, 101, 115, 47,  103, 114, 112,
-    99,  46,  108, 98,  46,  118, 49,  46,  76,  111, 97,  100, 66,  97,  108,
-    97,  110, 99,  101, 114, 47,  66,  97,  108, 97,  110, 99,  101, 76,  111,
-    97,  100, 48,  49,  50,  105, 100, 101, 110, 116, 105, 116, 121, 103, 122,
-    105, 112, 100, 101, 102, 108, 97,  116, 101, 116, 114, 97,  105, 108, 101,
-    114, 115, 97,  112, 112, 108, 105, 99,  97,  116, 105, 111, 110, 47,  103,
-    114, 112, 99,  80,  79,  83,  84,  50,  48,  48,  52,  48,  52,  104, 116,
-    116, 112, 104, 116, 116, 112, 115, 103, 114, 112, 99,  71,  69,  84,  80,
-    85,  84,  47,  47,  105, 110, 100, 101, 120, 46,  104, 116, 109, 108, 50,
-    48,  52,  50,  48,  54,  51,  48,  52,  52,  48,  48,  53,  48,  48,  97,
-    99,  99,  101, 112, 116, 45,  99,  104, 97,  114, 115, 101, 116, 97,  99,
-    99,  101, 112, 116, 45,  101, 110, 99,  111, 100, 105, 110, 103, 103, 122,
-    105, 112, 44,  32,  100, 101, 102, 108, 97,  116, 101, 97,  99,  99,  101,
-    112, 116, 45,  108, 97,  110, 103, 117, 97,  103, 101, 97,  99,  99,  101,
-    112, 116, 45,  114, 97,  110, 103, 101, 115, 97,  99,  99,  101, 112, 116,
-    97,  99,  99,  101, 115, 115, 45,  99,  111, 110, 116, 114, 111, 108, 45,
-    97,  108, 108, 111, 119, 45,  111, 114, 105, 103, 105, 110, 97,  103, 101,
-    97,  108, 108, 111, 119, 97,  117, 116, 104, 111, 114, 105, 122, 97,  116,
-    105, 111, 110, 99,  97,  99,  104, 101, 45,  99,  111, 110, 116, 114, 111,
-    108, 99,  111, 110, 116, 101, 110, 116, 45,  100, 105, 115, 112, 111, 115,
-    105, 116, 105, 111, 110, 99,  111, 110, 116, 101, 110, 116, 45,  101, 110,
-    99,  111, 100, 105, 110, 103, 99,  111, 110, 116, 101, 110, 116, 45,  108,
-    97,  110, 103, 117, 97,  103, 101, 99,  111, 110, 116, 101, 110, 116, 45,
-    108, 101, 110, 103, 116, 104, 99,  111, 110, 116, 101, 110, 116, 45,  108,
-    111, 99,  97,  116, 105, 111, 110, 99,  111, 110, 116, 101, 110, 116, 45,
-    114, 97,  110, 103, 101, 99,  111, 111, 107, 105, 101, 100, 97,  116, 101,
-    101, 116, 97,  103, 101, 120, 112, 101, 99,  116, 101, 120, 112, 105, 114,
-    101, 115, 102, 114, 111, 109, 105, 102, 45,  109, 97,  116, 99,  104, 105,
-    102, 45,  109, 111, 100, 105, 102, 105, 101, 100, 45,  115, 105, 110, 99,
-    101, 105, 102, 45,  110, 111, 110, 101, 45,  109, 97,  116, 99,  104, 105,
-    102, 45,  114, 97,  110, 103, 101, 105, 102, 45,  117, 110, 109, 111, 100,
-    105, 102, 105, 101, 100, 45,  115, 105, 110, 99,  101, 108, 97,  115, 116,
-    45,  109, 111, 100, 105, 102, 105, 101, 100, 108, 98,  45,  99,  111, 115,
-    116, 45,  98,  105, 110, 108, 105, 110, 107, 108, 111, 99,  97,  116, 105,
-    111, 110, 109, 97,  120, 45,  102, 111, 114, 119, 97,  114, 100, 115, 112,
-    114, 111, 120, 121, 45,  97,  117, 116, 104, 101, 110, 116, 105, 99,  97,
-    116, 101, 112, 114, 111, 120, 121, 45,  97,  117, 116, 104, 111, 114, 105,
-    122, 97,  116, 105, 111, 110, 114, 97,  110, 103, 101, 114, 101, 102, 101,
-    114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, 121,
-    45,  97,  102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, 116,
-    45,  99,  111, 111, 107, 105, 101, 115, 116, 114, 105, 99,  116, 45,  116,
-    114, 97,  110, 115, 112, 111, 114, 116, 45,  115, 101, 99,  117, 114, 105,
-    116, 121, 116, 114, 97,  110, 115, 102, 101, 114, 45,  101, 110, 99,  111,
-    100, 105, 110, 103, 118, 97,  114, 121, 118, 105, 97,  119, 119, 119, 45,
-    97,  117, 116, 104, 101, 110, 116, 105, 99,  97,  116, 101, 105, 100, 101,
-    110, 116, 105, 116, 121, 44,  100, 101, 102, 108, 97,  116, 101, 105, 100,
-    101, 110, 116, 105, 116, 121, 44,  103, 122, 105, 112, 100, 101, 102, 108,
-    97,  116, 101, 44,  103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116,
-    121, 44,  100, 101, 102, 108, 97,  116, 101, 44,  103, 122, 105, 112};
+    45,  116, 121, 112, 101, 99,  111, 110, 116, 101, 110, 116, 45,  101, 110,
+    99,  111, 100, 105, 110, 103, 97,  99,  99,  101, 112, 116, 45,  101, 110,
+    99,  111, 100, 105, 110, 103, 103, 114, 112, 99,  45,  105, 110, 116, 101,
+    114, 110, 97,  108, 45,  101, 110, 99,  111, 100, 105, 110, 103, 45,  114,
+    101, 113, 117, 101, 115, 116, 103, 114, 112, 99,  45,  105, 110, 116, 101,
+    114, 110, 97,  108, 45,  115, 116, 114, 101, 97,  109, 45,  101, 110, 99,
+    111, 100, 105, 110, 103, 45,  114, 101, 113, 117, 101, 115, 116, 117, 115,
+    101, 114, 45,  97,  103, 101, 110, 116, 104, 111, 115, 116, 108, 98,  45,
+    116, 111, 107, 101, 110, 103, 114, 112, 99,  45,  116, 105, 109, 101, 111,
+    117, 116, 103, 114, 112, 99,  46,  119, 97,  105, 116, 95,  102, 111, 114,
+    95,  114, 101, 97,  100, 121, 103, 114, 112, 99,  46,  116, 105, 109, 101,
+    111, 117, 116, 103, 114, 112, 99,  46,  109, 97,  120, 95,  114, 101, 113,
+    117, 101, 115, 116, 95,  109, 101, 115, 115, 97,  103, 101, 95,  98,  121,
+    116, 101, 115, 103, 114, 112, 99,  46,  109, 97,  120, 95,  114, 101, 115,
+    112, 111, 110, 115, 101, 95,  109, 101, 115, 115, 97,  103, 101, 95,  98,
+    121, 116, 101, 115, 47,  103, 114, 112, 99,  46,  108, 98,  46,  118, 49,
+    46,  76,  111, 97,  100, 66,  97,  108, 97,  110, 99,  101, 114, 47,  66,
+    97,  108, 97,  110, 99,  101, 76,  111, 97,  100, 48,  49,  50,  105, 100,
+    101, 110, 116, 105, 116, 121, 103, 122, 105, 112, 100, 101, 102, 108, 97,
+    116, 101, 116, 114, 97,  105, 108, 101, 114, 115, 97,  112, 112, 108, 105,
+    99,  97,  116, 105, 111, 110, 47,  103, 114, 112, 99,  80,  79,  83,  84,
+    50,  48,  48,  52,  48,  52,  104, 116, 116, 112, 104, 116, 116, 112, 115,
+    103, 114, 112, 99,  71,  69,  84,  80,  85,  84,  47,  47,  105, 110, 100,
+    101, 120, 46,  104, 116, 109, 108, 50,  48,  52,  50,  48,  54,  51,  48,
+    52,  52,  48,  48,  53,  48,  48,  97,  99,  99,  101, 112, 116, 45,  99,
+    104, 97,  114, 115, 101, 116, 103, 122, 105, 112, 44,  32,  100, 101, 102,
+    108, 97,  116, 101, 97,  99,  99,  101, 112, 116, 45,  108, 97,  110, 103,
+    117, 97,  103, 101, 97,  99,  99,  101, 112, 116, 45,  114, 97,  110, 103,
+    101, 115, 97,  99,  99,  101, 112, 116, 97,  99,  99,  101, 115, 115, 45,
+    99,  111, 110, 116, 114, 111, 108, 45,  97,  108, 108, 111, 119, 45,  111,
+    114, 105, 103, 105, 110, 97,  103, 101, 97,  108, 108, 111, 119, 97,  117,
+    116, 104, 111, 114, 105, 122, 97,  116, 105, 111, 110, 99,  97,  99,  104,
+    101, 45,  99,  111, 110, 116, 114, 111, 108, 99,  111, 110, 116, 101, 110,
+    116, 45,  100, 105, 115, 112, 111, 115, 105, 116, 105, 111, 110, 99,  111,
+    110, 116, 101, 110, 116, 45,  108, 97,  110, 103, 117, 97,  103, 101, 99,
+    111, 110, 116, 101, 110, 116, 45,  108, 101, 110, 103, 116, 104, 99,  111,
+    110, 116, 101, 110, 116, 45,  108, 111, 99,  97,  116, 105, 111, 110, 99,
+    111, 110, 116, 101, 110, 116, 45,  114, 97,  110, 103, 101, 99,  111, 111,
+    107, 105, 101, 100, 97,  116, 101, 101, 116, 97,  103, 101, 120, 112, 101,
+    99,  116, 101, 120, 112, 105, 114, 101, 115, 102, 114, 111, 109, 105, 102,
+    45,  109, 97,  116, 99,  104, 105, 102, 45,  109, 111, 100, 105, 102, 105,
+    101, 100, 45,  115, 105, 110, 99,  101, 105, 102, 45,  110, 111, 110, 101,
+    45,  109, 97,  116, 99,  104, 105, 102, 45,  114, 97,  110, 103, 101, 105,
+    102, 45,  117, 110, 109, 111, 100, 105, 102, 105, 101, 100, 45,  115, 105,
+    110, 99,  101, 108, 97,  115, 116, 45,  109, 111, 100, 105, 102, 105, 101,
+    100, 108, 98,  45,  99,  111, 115, 116, 45,  98,  105, 110, 108, 105, 110,
+    107, 108, 111, 99,  97,  116, 105, 111, 110, 109, 97,  120, 45,  102, 111,
+    114, 119, 97,  114, 100, 115, 112, 114, 111, 120, 121, 45,  97,  117, 116,
+    104, 101, 110, 116, 105, 99,  97,  116, 101, 112, 114, 111, 120, 121, 45,
+    97,  117, 116, 104, 111, 114, 105, 122, 97,  116, 105, 111, 110, 114, 97,
+    110, 103, 101, 114, 101, 102, 101, 114, 101, 114, 114, 101, 102, 114, 101,
+    115, 104, 114, 101, 116, 114, 121, 45,  97,  102, 116, 101, 114, 115, 101,
+    114, 118, 101, 114, 115, 101, 116, 45,  99,  111, 111, 107, 105, 101, 115,
+    116, 114, 105, 99,  116, 45,  116, 114, 97,  110, 115, 112, 111, 114, 116,
+    45,  115, 101, 99,  117, 114, 105, 116, 121, 116, 114, 97,  110, 115, 102,
+    101, 114, 45,  101, 110, 99,  111, 100, 105, 110, 103, 118, 97,  114, 121,
+    118, 105, 97,  119, 119, 119, 45,  97,  117, 116, 104, 101, 110, 116, 105,
+    99,  97,  116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44,  100, 101,
+    102, 108, 97,  116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44,  103,
+    122, 105, 112, 100, 101, 102, 108, 97,  116, 101, 44,  103, 122, 105, 112,
+    105, 100, 101, 110, 116, 105, 116, 121, 44,  100, 101, 102, 108, 97,  116,
+    101, 44,  103, 122, 105, 112};
 
 static void static_ref(void *unused) {}
 static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {}
@@ -209,6 +212,7 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = {
     {&grpc_static_metadata_vtable, &static_sub_refcnt},
     {&grpc_static_metadata_vtable, &static_sub_refcnt},
     {&grpc_static_metadata_vtable, &static_sub_refcnt},
+    {&grpc_static_metadata_vtable, &static_sub_refcnt},
 };
 
 const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = {
@@ -243,193 +247,194 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = {
     {.refcount = &grpc_static_metadata_refcounts[14],
      .data.refcounted = {g_bytes + 158, 12}},
     {.refcount = &grpc_static_metadata_refcounts[15],
-     .data.refcounted = {g_bytes + 170, 30}},
+     .data.refcounted = {g_bytes + 170, 16}},
     {.refcount = &grpc_static_metadata_refcounts[16],
-     .data.refcounted = {g_bytes + 200, 10}},
+     .data.refcounted = {g_bytes + 186, 15}},
     {.refcount = &grpc_static_metadata_refcounts[17],
-     .data.refcounted = {g_bytes + 210, 4}},
+     .data.refcounted = {g_bytes + 201, 30}},
     {.refcount = &grpc_static_metadata_refcounts[18],
-     .data.refcounted = {g_bytes + 214, 8}},
+     .data.refcounted = {g_bytes + 231, 37}},
     {.refcount = &grpc_static_metadata_refcounts[19],
-     .data.refcounted = {g_bytes + 222, 12}},
+     .data.refcounted = {g_bytes + 268, 10}},
     {.refcount = &grpc_static_metadata_refcounts[20],
-     .data.refcounted = {g_bytes + 234, 0}},
+     .data.refcounted = {g_bytes + 278, 4}},
     {.refcount = &grpc_static_metadata_refcounts[21],
-     .data.refcounted = {g_bytes + 234, 19}},
+     .data.refcounted = {g_bytes + 282, 8}},
     {.refcount = &grpc_static_metadata_refcounts[22],
-     .data.refcounted = {g_bytes + 253, 12}},
+     .data.refcounted = {g_bytes + 290, 12}},
     {.refcount = &grpc_static_metadata_refcounts[23],
-     .data.refcounted = {g_bytes + 265, 30}},
+     .data.refcounted = {g_bytes + 302, 0}},
     {.refcount = &grpc_static_metadata_refcounts[24],
-     .data.refcounted = {g_bytes + 295, 31}},
+     .data.refcounted = {g_bytes + 302, 19}},
     {.refcount = &grpc_static_metadata_refcounts[25],
-     .data.refcounted = {g_bytes + 326, 36}},
+     .data.refcounted = {g_bytes + 321, 12}},
     {.refcount = &grpc_static_metadata_refcounts[26],
-     .data.refcounted = {g_bytes + 362, 1}},
+     .data.refcounted = {g_bytes + 333, 30}},
     {.refcount = &grpc_static_metadata_refcounts[27],
-     .data.refcounted = {g_bytes + 363, 1}},
+     .data.refcounted = {g_bytes + 363, 31}},
     {.refcount = &grpc_static_metadata_refcounts[28],
-     .data.refcounted = {g_bytes + 364, 1}},
+     .data.refcounted = {g_bytes + 394, 36}},
     {.refcount = &grpc_static_metadata_refcounts[29],
-     .data.refcounted = {g_bytes + 365, 8}},
+     .data.refcounted = {g_bytes + 430, 1}},
     {.refcount = &grpc_static_metadata_refcounts[30],
-     .data.refcounted = {g_bytes + 373, 4}},
+     .data.refcounted = {g_bytes + 431, 1}},
     {.refcount = &grpc_static_metadata_refcounts[31],
-     .data.refcounted = {g_bytes + 377, 7}},
+     .data.refcounted = {g_bytes + 432, 1}},
     {.refcount = &grpc_static_metadata_refcounts[32],
-     .data.refcounted = {g_bytes + 384, 8}},
+     .data.refcounted = {g_bytes + 433, 8}},
     {.refcount = &grpc_static_metadata_refcounts[33],
-     .data.refcounted = {g_bytes + 392, 16}},
+     .data.refcounted = {g_bytes + 441, 4}},
     {.refcount = &grpc_static_metadata_refcounts[34],
-     .data.refcounted = {g_bytes + 408, 4}},
+     .data.refcounted = {g_bytes + 445, 7}},
     {.refcount = &grpc_static_metadata_refcounts[35],
-     .data.refcounted = {g_bytes + 412, 3}},
+     .data.refcounted = {g_bytes + 452, 8}},
     {.refcount = &grpc_static_metadata_refcounts[36],
-     .data.refcounted = {g_bytes + 415, 3}},
+     .data.refcounted = {g_bytes + 460, 16}},
     {.refcount = &grpc_static_metadata_refcounts[37],
-     .data.refcounted = {g_bytes + 418, 4}},
+     .data.refcounted = {g_bytes + 476, 4}},
     {.refcount = &grpc_static_metadata_refcounts[38],
-     .data.refcounted = {g_bytes + 422, 5}},
+     .data.refcounted = {g_bytes + 480, 3}},
     {.refcount = &grpc_static_metadata_refcounts[39],
-     .data.refcounted = {g_bytes + 427, 4}},
+     .data.refcounted = {g_bytes + 483, 3}},
     {.refcount = &grpc_static_metadata_refcounts[40],
-     .data.refcounted = {g_bytes + 431, 3}},
+     .data.refcounted = {g_bytes + 486, 4}},
     {.refcount = &grpc_static_metadata_refcounts[41],
-     .data.refcounted = {g_bytes + 434, 3}},
+     .data.refcounted = {g_bytes + 490, 5}},
     {.refcount = &grpc_static_metadata_refcounts[42],
-     .data.refcounted = {g_bytes + 437, 1}},
+     .data.refcounted = {g_bytes + 495, 4}},
     {.refcount = &grpc_static_metadata_refcounts[43],
-     .data.refcounted = {g_bytes + 438, 11}},
+     .data.refcounted = {g_bytes + 499, 3}},
     {.refcount = &grpc_static_metadata_refcounts[44],
-     .data.refcounted = {g_bytes + 449, 3}},
+     .data.refcounted = {g_bytes + 502, 3}},
     {.refcount = &grpc_static_metadata_refcounts[45],
-     .data.refcounted = {g_bytes + 452, 3}},
+     .data.refcounted = {g_bytes + 505, 1}},
     {.refcount = &grpc_static_metadata_refcounts[46],
-     .data.refcounted = {g_bytes + 455, 3}},
+     .data.refcounted = {g_bytes + 506, 11}},
     {.refcount = &grpc_static_metadata_refcounts[47],
-     .data.refcounted = {g_bytes + 458, 3}},
+     .data.refcounted = {g_bytes + 517, 3}},
     {.refcount = &grpc_static_metadata_refcounts[48],
-     .data.refcounted = {g_bytes + 461, 3}},
+     .data.refcounted = {g_bytes + 520, 3}},
     {.refcount = &grpc_static_metadata_refcounts[49],
-     .data.refcounted = {g_bytes + 464, 14}},
+     .data.refcounted = {g_bytes + 523, 3}},
     {.refcount = &grpc_static_metadata_refcounts[50],
-     .data.refcounted = {g_bytes + 478, 15}},
+     .data.refcounted = {g_bytes + 526, 3}},
     {.refcount = &grpc_static_metadata_refcounts[51],
-     .data.refcounted = {g_bytes + 493, 13}},
+     .data.refcounted = {g_bytes + 529, 3}},
     {.refcount = &grpc_static_metadata_refcounts[52],
-     .data.refcounted = {g_bytes + 506, 15}},
+     .data.refcounted = {g_bytes + 532, 14}},
     {.refcount = &grpc_static_metadata_refcounts[53],
-     .data.refcounted = {g_bytes + 521, 13}},
+     .data.refcounted = {g_bytes + 546, 13}},
     {.refcount = &grpc_static_metadata_refcounts[54],
-     .data.refcounted = {g_bytes + 534, 6}},
+     .data.refcounted = {g_bytes + 559, 15}},
     {.refcount = &grpc_static_metadata_refcounts[55],
-     .data.refcounted = {g_bytes + 540, 27}},
+     .data.refcounted = {g_bytes + 574, 13}},
     {.refcount = &grpc_static_metadata_refcounts[56],
-     .data.refcounted = {g_bytes + 567, 3}},
+     .data.refcounted = {g_bytes + 587, 6}},
     {.refcount = &grpc_static_metadata_refcounts[57],
-     .data.refcounted = {g_bytes + 570, 5}},
+     .data.refcounted = {g_bytes + 593, 27}},
     {.refcount = &grpc_static_metadata_refcounts[58],
-     .data.refcounted = {g_bytes + 575, 13}},
+     .data.refcounted = {g_bytes + 620, 3}},
     {.refcount = &grpc_static_metadata_refcounts[59],
-     .data.refcounted = {g_bytes + 588, 13}},
+     .data.refcounted = {g_bytes + 623, 5}},
     {.refcount = &grpc_static_metadata_refcounts[60],
-     .data.refcounted = {g_bytes + 601, 19}},
+     .data.refcounted = {g_bytes + 628, 13}},
     {.refcount = &grpc_static_metadata_refcounts[61],
-     .data.refcounted = {g_bytes + 620, 16}},
+     .data.refcounted = {g_bytes + 641, 13}},
     {.refcount = &grpc_static_metadata_refcounts[62],
-     .data.refcounted = {g_bytes + 636, 16}},
+     .data.refcounted = {g_bytes + 654, 19}},
     {.refcount = &grpc_static_metadata_refcounts[63],
-     .data.refcounted = {g_bytes + 652, 14}},
+     .data.refcounted = {g_bytes + 673, 16}},
     {.refcount = &grpc_static_metadata_refcounts[64],
-     .data.refcounted = {g_bytes + 666, 16}},
+     .data.refcounted = {g_bytes + 689, 14}},
     {.refcount = &grpc_static_metadata_refcounts[65],
-     .data.refcounted = {g_bytes + 682, 13}},
+     .data.refcounted = {g_bytes + 703, 16}},
     {.refcount = &grpc_static_metadata_refcounts[66],
-     .data.refcounted = {g_bytes + 695, 6}},
+     .data.refcounted = {g_bytes + 719, 13}},
     {.refcount = &grpc_static_metadata_refcounts[67],
-     .data.refcounted = {g_bytes + 701, 4}},
+     .data.refcounted = {g_bytes + 732, 6}},
     {.refcount = &grpc_static_metadata_refcounts[68],
-     .data.refcounted = {g_bytes + 705, 4}},
+     .data.refcounted = {g_bytes + 738, 4}},
     {.refcount = &grpc_static_metadata_refcounts[69],
-     .data.refcounted = {g_bytes + 709, 6}},
+     .data.refcounted = {g_bytes + 742, 4}},
     {.refcount = &grpc_static_metadata_refcounts[70],
-     .data.refcounted = {g_bytes + 715, 7}},
+     .data.refcounted = {g_bytes + 746, 6}},
     {.refcount = &grpc_static_metadata_refcounts[71],
-     .data.refcounted = {g_bytes + 722, 4}},
+     .data.refcounted = {g_bytes + 752, 7}},
     {.refcount = &grpc_static_metadata_refcounts[72],
-     .data.refcounted = {g_bytes + 726, 8}},
+     .data.refcounted = {g_bytes + 759, 4}},
     {.refcount = &grpc_static_metadata_refcounts[73],
-     .data.refcounted = {g_bytes + 734, 17}},
+     .data.refcounted = {g_bytes + 763, 8}},
     {.refcount = &grpc_static_metadata_refcounts[74],
-     .data.refcounted = {g_bytes + 751, 13}},
+     .data.refcounted = {g_bytes + 771, 17}},
     {.refcount = &grpc_static_metadata_refcounts[75],
-     .data.refcounted = {g_bytes + 764, 8}},
+     .data.refcounted = {g_bytes + 788, 13}},
     {.refcount = &grpc_static_metadata_refcounts[76],
-     .data.refcounted = {g_bytes + 772, 19}},
+     .data.refcounted = {g_bytes + 801, 8}},
     {.refcount = &grpc_static_metadata_refcounts[77],
-     .data.refcounted = {g_bytes + 791, 13}},
+     .data.refcounted = {g_bytes + 809, 19}},
     {.refcount = &grpc_static_metadata_refcounts[78],
-     .data.refcounted = {g_bytes + 804, 11}},
+     .data.refcounted = {g_bytes + 828, 13}},
     {.refcount = &grpc_static_metadata_refcounts[79],
-     .data.refcounted = {g_bytes + 815, 4}},
+     .data.refcounted = {g_bytes + 841, 11}},
     {.refcount = &grpc_static_metadata_refcounts[80],
-     .data.refcounted = {g_bytes + 819, 8}},
+     .data.refcounted = {g_bytes + 852, 4}},
     {.refcount = &grpc_static_metadata_refcounts[81],
-     .data.refcounted = {g_bytes + 827, 12}},
+     .data.refcounted = {g_bytes + 856, 8}},
     {.refcount = &grpc_static_metadata_refcounts[82],
-     .data.refcounted = {g_bytes + 839, 18}},
+     .data.refcounted = {g_bytes + 864, 12}},
     {.refcount = &grpc_static_metadata_refcounts[83],
-     .data.refcounted = {g_bytes + 857, 19}},
+     .data.refcounted = {g_bytes + 876, 18}},
     {.refcount = &grpc_static_metadata_refcounts[84],
-     .data.refcounted = {g_bytes + 876, 5}},
+     .data.refcounted = {g_bytes + 894, 19}},
     {.refcount = &grpc_static_metadata_refcounts[85],
-     .data.refcounted = {g_bytes + 881, 7}},
+     .data.refcounted = {g_bytes + 913, 5}},
     {.refcount = &grpc_static_metadata_refcounts[86],
-     .data.refcounted = {g_bytes + 888, 7}},
+     .data.refcounted = {g_bytes + 918, 7}},
     {.refcount = &grpc_static_metadata_refcounts[87],
-     .data.refcounted = {g_bytes + 895, 11}},
+     .data.refcounted = {g_bytes + 925, 7}},
     {.refcount = &grpc_static_metadata_refcounts[88],
-     .data.refcounted = {g_bytes + 906, 6}},
+     .data.refcounted = {g_bytes + 932, 11}},
     {.refcount = &grpc_static_metadata_refcounts[89],
-     .data.refcounted = {g_bytes + 912, 10}},
+     .data.refcounted = {g_bytes + 943, 6}},
     {.refcount = &grpc_static_metadata_refcounts[90],
-     .data.refcounted = {g_bytes + 922, 25}},
+     .data.refcounted = {g_bytes + 949, 10}},
     {.refcount = &grpc_static_metadata_refcounts[91],
-     .data.refcounted = {g_bytes + 947, 17}},
+     .data.refcounted = {g_bytes + 959, 25}},
     {.refcount = &grpc_static_metadata_refcounts[92],
-     .data.refcounted = {g_bytes + 964, 4}},
+     .data.refcounted = {g_bytes + 984, 17}},
     {.refcount = &grpc_static_metadata_refcounts[93],
-     .data.refcounted = {g_bytes + 968, 3}},
+     .data.refcounted = {g_bytes + 1001, 4}},
     {.refcount = &grpc_static_metadata_refcounts[94],
-     .data.refcounted = {g_bytes + 971, 16}},
+     .data.refcounted = {g_bytes + 1005, 3}},
     {.refcount = &grpc_static_metadata_refcounts[95],
-     .data.refcounted = {g_bytes + 987, 16}},
+     .data.refcounted = {g_bytes + 1008, 16}},
     {.refcount = &grpc_static_metadata_refcounts[96],
-     .data.refcounted = {g_bytes + 1003, 13}},
+     .data.refcounted = {g_bytes + 1024, 16}},
     {.refcount = &grpc_static_metadata_refcounts[97],
-     .data.refcounted = {g_bytes + 1016, 12}},
+     .data.refcounted = {g_bytes + 1040, 13}},
     {.refcount = &grpc_static_metadata_refcounts[98],
-     .data.refcounted = {g_bytes + 1028, 21}},
+     .data.refcounted = {g_bytes + 1053, 12}},
+    {.refcount = &grpc_static_metadata_refcounts[99],
+     .data.refcounted = {g_bytes + 1065, 21}},
 };
 
 uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = {
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8};
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8, 2, 4, 4};
 
 static const int8_t elems_r[] = {
-    10,  8,   -3,  0,   9,  21, -77, 22,  0,   10, -7,  0,   0,   0,
-    14,  0,   13,  12,  11, 0,  0,   0,   0,   0,  0,   0,   0,   0,
-    0,   0,   0,   0,   0,  0,  0,   0,   0,   0,  0,   0,   0,   0,
-    0,   0,   0,   0,   0,  0,  0,   -50, -51, 16, -53, -54, -55, -56,
-    -56, -57, -58, -59, 0,  37, 36,  35,  34,  33, 32,  31,  30,  29,
-    28,  27,  26,  25,  24, 23, 22,  21,  20,  19, 18,  17,  16,  15,
-    14,  13,  12,  11,  10, 13, 12,  11,  10,  9,  8,   7,   0};
+    11, 9,   -3, 0,   10,  27,  -74, 28,  0,   14, -7, 0,  0,  0,  18, 8,  -2,
+    0,  0,   13, 12,  11,  0,   0,   0,   0,   0,  0,  0,  0,  0,  0,  0,  0,
+    0,  0,   0,  0,   0,   0,   0,   0,   0,   0,  0,  0,  0,  0,  0,  0,  0,
+    0,  -50, 0,  -33, -55, -56, -57, -58, -57, 0,  40, 39, 38, 37, 36, 35, 34,
+    33, 32,  31, 30,  29,  28,  28,  27,  26,  25, 24, 23, 22, 21, 20, 19, 22,
+    21, 20,  19, 18,  17,  16,  15,  14,  13,  12, 12, 11, 0};
 static uint32_t elems_phash(uint32_t i) {
-  i -= 42;
-  uint32_t x = i % 97;
-  uint32_t y = i / 97;
+  i -= 45;
+  uint32_t x = i % 98;
+  uint32_t y = i / 98;
   uint32_t h = x;
   if (y < GPR_ARRAY_SIZE(elems_r)) {
     uint32_t delta = (uint32_t)elems_r[y];
@@ -439,30 +444,31 @@ static uint32_t elems_phash(uint32_t i) {
 }
 
 static const uint16_t elem_keys[] = {
-    1019, 1020, 1021, 242,  243,  244,  245,  246,  139,  140,  42,   43,
-    433,  434,  435,  920,  921,  922,  719,  720,  1406, 527,  721,  1604,
-    1703, 1802, 4871, 4970, 5001, 5168, 5267, 5366, 5465, 1419, 5564, 5663,
-    5762, 5861, 5960, 6059, 6158, 6257, 6356, 6455, 6554, 6653, 6752, 6851,
-    6950, 7049, 7148, 7247, 7346, 7445, 7544, 7643, 7742, 7841, 7940, 8039,
-    8138, 8237, 8336, 8435, 8534, 8633, 1085, 1086, 1087, 1088, 8732, 8831,
-    8930, 9029, 9128, 9227, 9326, 0,    317,  0,    0,    0,    0,    0,
+    1032, 1033, 1034, 247,  248,  249,  250,  251,  1623, 143,  144,  45,
+    46,   440,  441,  442,  1523, 1632, 1633, 932,  933,  934,  729,  730,
+    1423, 1532, 1533, 535,  731,  1923, 2023, 2123, 5223, 5523, 5623, 5723,
+    5823, 1436, 1653, 5923, 6023, 6123, 6223, 6323, 6423, 6523, 6623, 6723,
+    6823, 6923, 7023, 7123, 7223, 5423, 7323, 7423, 7523, 7623, 7723, 7823,
+    7923, 8023, 8123, 8223, 1096, 1097, 1098, 1099, 8323, 8423, 8523, 8623,
+    8723, 8823, 8923, 9023, 9123, 9223, 9323, 323,  9423, 9523, 1697, 0,
     0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-    0,    0,    0,    0,    0,    133,  233,  234,  0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    137,  238,  239,  0,    0,
     0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
     0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-    0};
+    0,    0,    0,    0,    0};
 static const uint8_t elem_idxs[] = {
-    74,  77,  75,  19,  20,  21,  22,  23,  15,  16,  17,  18,  11,  12,  13,
-    3,   4,   5,   0,   1,   41,  6,   2,   70,  48,  55,  24,  25,  26,  27,
-    28,  29,  30,  7,   31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  42,
-    43,  44,  45,  46,  47,  49,  50,  51,  52,  53,  54,  56,  57,  58,  59,
-    60,  61,  62,  63,  64,  65,  76,  78,  79,  80,  66,  67,  68,  69,  71,
-    72,  73,  255, 14,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 8,   9,   10};
+    76,  79,  77,  19,  20,  21,  22,  23,  25,  15,  16,  17,  18,  11,
+    12,  13,  38,  83,  84,  3,   4,   5,   0,   1,   43,  36,  37,  6,
+    2,   72,  50,  57,  24,  28,  29,  30,  31,  7,   26,  32,  33,  34,
+    35,  39,  40,  41,  42,  44,  45,  46,  47,  48,  49,  27,  51,  52,
+    53,  54,  55,  56,  58,  59,  60,  61,  78,  80,  81,  82,  62,  63,
+    64,  65,  66,  67,  68,  69,  70,  71,  73,  14,  74,  75,  85,  255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 8,   9,   10};
 
 grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) {
   if (a == -1 || b == -1) return GRPC_MDNULL;
-  uint32_t k = (uint32_t)(a * 99 + b);
+  uint32_t k = (uint32_t)(a * 100 + b);
   uint32_t h = elems_phash(k);
   return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k &&
                  elem_idxs[h] != 255
@@ -474,328 +480,350 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) {
 grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {
     {{.refcount = &grpc_static_metadata_refcounts[7],
       .data.refcounted = {g_bytes + 50, 11}},
-     {.refcount = &grpc_static_metadata_refcounts[26],
-      .data.refcounted = {g_bytes + 362, 1}}},
+     {.refcount = &grpc_static_metadata_refcounts[29],
+      .data.refcounted = {g_bytes + 430, 1}}},
     {{.refcount = &grpc_static_metadata_refcounts[7],
       .data.refcounted = {g_bytes + 50, 11}},
-     {.refcount = &grpc_static_metadata_refcounts[27],
-      .data.refcounted = {g_bytes + 363, 1}}},
+     {.refcount = &grpc_static_metadata_refcounts[30],
+      .data.refcounted = {g_bytes + 431, 1}}},
     {{.refcount = &grpc_static_metadata_refcounts[7],
       .data.refcounted = {g_bytes + 50, 11}},
-     {.refcount = &grpc_static_metadata_refcounts[28],
-      .data.refcounted = {g_bytes + 364, 1}}},
+     {.refcount = &grpc_static_metadata_refcounts[31],
+      .data.refcounted = {g_bytes + 432, 1}}},
     {{.refcount = &grpc_static_metadata_refcounts[9],
       .data.refcounted = {g_bytes + 77, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[29],
-      .data.refcounted = {g_bytes + 365, 8}}},
+     {.refcount = &grpc_static_metadata_refcounts[32],
+      .data.refcounted = {g_bytes + 433, 8}}},
     {{.refcount = &grpc_static_metadata_refcounts[9],
       .data.refcounted = {g_bytes + 77, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[30],
-      .data.refcounted = {g_bytes + 373, 4}}},
+     {.refcount = &grpc_static_metadata_refcounts[33],
+      .data.refcounted = {g_bytes + 441, 4}}},
     {{.refcount = &grpc_static_metadata_refcounts[9],
       .data.refcounted = {g_bytes + 77, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[31],
-      .data.refcounted = {g_bytes + 377, 7}}},
+     {.refcount = &grpc_static_metadata_refcounts[34],
+      .data.refcounted = {g_bytes + 445, 7}}},
     {{.refcount = &grpc_static_metadata_refcounts[5],
       .data.refcounted = {g_bytes + 36, 2}},
-     {.refcount = &grpc_static_metadata_refcounts[32],
-      .data.refcounted = {g_bytes + 384, 8}}},
+     {.refcount = &grpc_static_metadata_refcounts[35],
+      .data.refcounted = {g_bytes + 452, 8}}},
     {{.refcount = &grpc_static_metadata_refcounts[14],
       .data.refcounted = {g_bytes + 158, 12}},
-     {.refcount = &grpc_static_metadata_refcounts[33],
-      .data.refcounted = {g_bytes + 392, 16}}},
+     {.refcount = &grpc_static_metadata_refcounts[36],
+      .data.refcounted = {g_bytes + 460, 16}}},
     {{.refcount = &grpc_static_metadata_refcounts[1],
       .data.refcounted = {g_bytes + 5, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[34],
-      .data.refcounted = {g_bytes + 408, 4}}},
+     {.refcount = &grpc_static_metadata_refcounts[37],
+      .data.refcounted = {g_bytes + 476, 4}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[35],
-      .data.refcounted = {g_bytes + 412, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[38],
+      .data.refcounted = {g_bytes + 480, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[36],
-      .data.refcounted = {g_bytes + 415, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[39],
+      .data.refcounted = {g_bytes + 483, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[4],
       .data.refcounted = {g_bytes + 29, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[37],
-      .data.refcounted = {g_bytes + 418, 4}}},
+     {.refcount = &grpc_static_metadata_refcounts[40],
+      .data.refcounted = {g_bytes + 486, 4}}},
     {{.refcount = &grpc_static_metadata_refcounts[4],
       .data.refcounted = {g_bytes + 29, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[38],
-      .data.refcounted = {g_bytes + 422, 5}}},
+     {.refcount = &grpc_static_metadata_refcounts[41],
+      .data.refcounted = {g_bytes + 490, 5}}},
     {{.refcount = &grpc_static_metadata_refcounts[4],
       .data.refcounted = {g_bytes + 29, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[39],
-      .data.refcounted = {g_bytes + 427, 4}}},
+     {.refcount = &grpc_static_metadata_refcounts[42],
+      .data.refcounted = {g_bytes + 495, 4}}},
     {{.refcount = &grpc_static_metadata_refcounts[3],
       .data.refcounted = {g_bytes + 19, 10}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[1],
       .data.refcounted = {g_bytes + 5, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[40],
-      .data.refcounted = {g_bytes + 431, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[43],
+      .data.refcounted = {g_bytes + 499, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[1],
       .data.refcounted = {g_bytes + 5, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[41],
-      .data.refcounted = {g_bytes + 434, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[44],
+      .data.refcounted = {g_bytes + 502, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[0],
       .data.refcounted = {g_bytes + 0, 5}},
-     {.refcount = &grpc_static_metadata_refcounts[42],
-      .data.refcounted = {g_bytes + 437, 1}}},
+     {.refcount = &grpc_static_metadata_refcounts[45],
+      .data.refcounted = {g_bytes + 505, 1}}},
     {{.refcount = &grpc_static_metadata_refcounts[0],
       .data.refcounted = {g_bytes + 0, 5}},
-     {.refcount = &grpc_static_metadata_refcounts[43],
-      .data.refcounted = {g_bytes + 438, 11}}},
+     {.refcount = &grpc_static_metadata_refcounts[46],
+      .data.refcounted = {g_bytes + 506, 11}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[44],
-      .data.refcounted = {g_bytes + 449, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[47],
+      .data.refcounted = {g_bytes + 517, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[45],
-      .data.refcounted = {g_bytes + 452, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[48],
+      .data.refcounted = {g_bytes + 520, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[46],
-      .data.refcounted = {g_bytes + 455, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[49],
+      .data.refcounted = {g_bytes + 523, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[47],
-      .data.refcounted = {g_bytes + 458, 3}}},
+     {.refcount = &grpc_static_metadata_refcounts[50],
+      .data.refcounted = {g_bytes + 526, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[2],
       .data.refcounted = {g_bytes + 12, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[48],
-      .data.refcounted = {g_bytes + 461, 3}}},
-    {{.refcount = &grpc_static_metadata_refcounts[49],
-      .data.refcounted = {g_bytes + 464, 14}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[50],
-      .data.refcounted = {g_bytes + 478, 15}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[50],
-      .data.refcounted = {g_bytes + 478, 15}},
      {.refcount = &grpc_static_metadata_refcounts[51],
-      .data.refcounted = {g_bytes + 493, 13}}},
+      .data.refcounted = {g_bytes + 529, 3}}},
     {{.refcount = &grpc_static_metadata_refcounts[52],
-      .data.refcounted = {g_bytes + 506, 15}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[53],
-      .data.refcounted = {g_bytes + 521, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 532, 14}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[16],
+      .data.refcounted = {g_bytes + 186, 15}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[16],
+      .data.refcounted = {g_bytes + 186, 15}},
+     {.refcount = &grpc_static_metadata_refcounts[53],
+      .data.refcounted = {g_bytes + 546, 13}}},
     {{.refcount = &grpc_static_metadata_refcounts[54],
-      .data.refcounted = {g_bytes + 534, 6}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 559, 15}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[55],
-      .data.refcounted = {g_bytes + 540, 27}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 574, 13}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[56],
-      .data.refcounted = {g_bytes + 567, 3}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 587, 6}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[57],
-      .data.refcounted = {g_bytes + 570, 5}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 593, 27}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[58],
-      .data.refcounted = {g_bytes + 575, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 620, 3}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[59],
-      .data.refcounted = {g_bytes + 588, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 623, 5}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[60],
-      .data.refcounted = {g_bytes + 601, 19}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 628, 13}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[61],
-      .data.refcounted = {g_bytes + 620, 16}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 641, 13}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[62],
-      .data.refcounted = {g_bytes + 636, 16}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 654, 19}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[15],
+      .data.refcounted = {g_bytes + 170, 16}},
+     {.refcount = &grpc_static_metadata_refcounts[32],
+      .data.refcounted = {g_bytes + 433, 8}}},
+    {{.refcount = &grpc_static_metadata_refcounts[15],
+      .data.refcounted = {g_bytes + 170, 16}},
+     {.refcount = &grpc_static_metadata_refcounts[33],
+      .data.refcounted = {g_bytes + 441, 4}}},
+    {{.refcount = &grpc_static_metadata_refcounts[15],
+      .data.refcounted = {g_bytes + 170, 16}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[63],
-      .data.refcounted = {g_bytes + 652, 14}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 673, 16}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[64],
-      .data.refcounted = {g_bytes + 666, 16}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 689, 14}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[65],
-      .data.refcounted = {g_bytes + 682, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 703, 16}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[66],
+      .data.refcounted = {g_bytes + 719, 13}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[14],
       .data.refcounted = {g_bytes + 158, 12}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[66],
-      .data.refcounted = {g_bytes + 695, 6}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[67],
-      .data.refcounted = {g_bytes + 701, 4}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 732, 6}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[68],
-      .data.refcounted = {g_bytes + 705, 4}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 738, 4}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[69],
-      .data.refcounted = {g_bytes + 709, 6}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 742, 4}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[70],
-      .data.refcounted = {g_bytes + 715, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 746, 6}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[71],
-      .data.refcounted = {g_bytes + 722, 4}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[17],
-      .data.refcounted = {g_bytes + 210, 4}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 752, 7}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[72],
-      .data.refcounted = {g_bytes + 726, 8}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 759, 4}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[20],
+      .data.refcounted = {g_bytes + 278, 4}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[73],
-      .data.refcounted = {g_bytes + 734, 17}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 763, 8}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[74],
-      .data.refcounted = {g_bytes + 751, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 771, 17}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[75],
-      .data.refcounted = {g_bytes + 764, 8}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 788, 13}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[76],
-      .data.refcounted = {g_bytes + 772, 19}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 801, 8}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[77],
-      .data.refcounted = {g_bytes + 791, 13}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[18],
-      .data.refcounted = {g_bytes + 214, 8}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 809, 19}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[78],
-      .data.refcounted = {g_bytes + 804, 11}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 828, 13}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[21],
+      .data.refcounted = {g_bytes + 282, 8}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[79],
-      .data.refcounted = {g_bytes + 815, 4}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 841, 11}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[80],
-      .data.refcounted = {g_bytes + 819, 8}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 852, 4}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[81],
-      .data.refcounted = {g_bytes + 827, 12}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 856, 8}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[82],
-      .data.refcounted = {g_bytes + 839, 18}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 864, 12}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[83],
-      .data.refcounted = {g_bytes + 857, 19}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 876, 18}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[84],
-      .data.refcounted = {g_bytes + 876, 5}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 894, 19}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[85],
-      .data.refcounted = {g_bytes + 881, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 913, 5}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[86],
-      .data.refcounted = {g_bytes + 888, 7}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 918, 7}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[87],
-      .data.refcounted = {g_bytes + 895, 11}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 925, 7}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[88],
-      .data.refcounted = {g_bytes + 906, 6}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 932, 11}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[89],
-      .data.refcounted = {g_bytes + 912, 10}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 943, 6}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[90],
-      .data.refcounted = {g_bytes + 922, 25}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 949, 10}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[91],
-      .data.refcounted = {g_bytes + 947, 17}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[16],
-      .data.refcounted = {g_bytes + 200, 10}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 959, 25}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[92],
-      .data.refcounted = {g_bytes + 964, 4}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 984, 17}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[19],
+      .data.refcounted = {g_bytes + 268, 10}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[93],
-      .data.refcounted = {g_bytes + 968, 3}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
+      .data.refcounted = {g_bytes + 1001, 4}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[94],
-      .data.refcounted = {g_bytes + 971, 16}},
-     {.refcount = &grpc_static_metadata_refcounts[20],
-      .data.refcounted = {g_bytes + 234, 0}}},
-    {{.refcount = &grpc_static_metadata_refcounts[10],
-      .data.refcounted = {g_bytes + 90, 20}},
-     {.refcount = &grpc_static_metadata_refcounts[29],
-      .data.refcounted = {g_bytes + 365, 8}}},
+      .data.refcounted = {g_bytes + 1005, 3}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
+    {{.refcount = &grpc_static_metadata_refcounts[95],
+      .data.refcounted = {g_bytes + 1008, 16}},
+     {.refcount = &grpc_static_metadata_refcounts[23],
+      .data.refcounted = {g_bytes + 302, 0}}},
     {{.refcount = &grpc_static_metadata_refcounts[10],
       .data.refcounted = {g_bytes + 90, 20}},
-     {.refcount = &grpc_static_metadata_refcounts[31],
-      .data.refcounted = {g_bytes + 377, 7}}},
+     {.refcount = &grpc_static_metadata_refcounts[32],
+      .data.refcounted = {g_bytes + 433, 8}}},
     {{.refcount = &grpc_static_metadata_refcounts[10],
       .data.refcounted = {g_bytes + 90, 20}},
-     {.refcount = &grpc_static_metadata_refcounts[95],
-      .data.refcounted = {g_bytes + 987, 16}}},
+     {.refcount = &grpc_static_metadata_refcounts[34],
+      .data.refcounted = {g_bytes + 445, 7}}},
     {{.refcount = &grpc_static_metadata_refcounts[10],
       .data.refcounted = {g_bytes + 90, 20}},
-     {.refcount = &grpc_static_metadata_refcounts[30],
-      .data.refcounted = {g_bytes + 373, 4}}},
+     {.refcount = &grpc_static_metadata_refcounts[96],
+      .data.refcounted = {g_bytes + 1024, 16}}},
     {{.refcount = &grpc_static_metadata_refcounts[10],
       .data.refcounted = {g_bytes + 90, 20}},
-     {.refcount = &grpc_static_metadata_refcounts[96],
-      .data.refcounted = {g_bytes + 1003, 13}}},
+     {.refcount = &grpc_static_metadata_refcounts[33],
+      .data.refcounted = {g_bytes + 441, 4}}},
     {{.refcount = &grpc_static_metadata_refcounts[10],
       .data.refcounted = {g_bytes + 90, 20}},
      {.refcount = &grpc_static_metadata_refcounts[97],
-      .data.refcounted = {g_bytes + 1016, 12}}},
+      .data.refcounted = {g_bytes + 1040, 13}}},
     {{.refcount = &grpc_static_metadata_refcounts[10],
       .data.refcounted = {g_bytes + 90, 20}},
      {.refcount = &grpc_static_metadata_refcounts[98],
-      .data.refcounted = {g_bytes + 1028, 21}}},
+      .data.refcounted = {g_bytes + 1053, 12}}},
+    {{.refcount = &grpc_static_metadata_refcounts[10],
+      .data.refcounted = {g_bytes + 90, 20}},
+     {.refcount = &grpc_static_metadata_refcounts[99],
+      .data.refcounted = {g_bytes + 1065, 21}}},
+    {{.refcount = &grpc_static_metadata_refcounts[16],
+      .data.refcounted = {g_bytes + 186, 15}},
+     {.refcount = &grpc_static_metadata_refcounts[32],
+      .data.refcounted = {g_bytes + 433, 8}}},
+    {{.refcount = &grpc_static_metadata_refcounts[16],
+      .data.refcounted = {g_bytes + 186, 15}},
+     {.refcount = &grpc_static_metadata_refcounts[33],
+      .data.refcounted = {g_bytes + 441, 4}}},
+    {{.refcount = &grpc_static_metadata_refcounts[16],
+      .data.refcounted = {g_bytes + 186, 15}},
+     {.refcount = &grpc_static_metadata_refcounts[97],
+      .data.refcounted = {g_bytes + 1040, 13}}},
 };
-const uint8_t grpc_static_accept_encoding_metadata[8] = {0,  74, 75, 76,
-                                                         77, 78, 79, 80};
+const uint8_t grpc_static_accept_encoding_metadata[8] = {0,  76, 77, 78,
+                                                         79, 80, 81, 82};
+
+const uint8_t grpc_static_accept_stream_encoding_metadata[4] = {0, 83, 84, 85};
diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h
index baa86de142..93ab90dff8 100644
--- a/src/core/lib/transport/static_metadata.h
+++ b/src/core/lib/transport/static_metadata.h
@@ -29,7 +29,7 @@
 
 #include "src/core/lib/transport/metadata.h"
 
-#define GRPC_STATIC_MDSTR_COUNT 99
+#define GRPC_STATIC_MDSTR_COUNT 100
 extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];
 /* ":path" */
 #define GRPC_MDSTR_PATH (grpc_static_slice_table[0])
@@ -61,178 +61,181 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_GRPC_TRACE_BIN (grpc_static_slice_table[13])
 /* "content-type" */
 #define GRPC_MDSTR_CONTENT_TYPE (grpc_static_slice_table[14])
+/* "content-encoding" */
+#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[15])
+/* "accept-encoding" */
+#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[16])
 /* "grpc-internal-encoding-request" */
-#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (grpc_static_slice_table[15])
+#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (grpc_static_slice_table[17])
+/* "grpc-internal-stream-encoding-request" */
+#define GRPC_MDSTR_GRPC_INTERNAL_STREAM_ENCODING_REQUEST \
+  (grpc_static_slice_table[18])
 /* "user-agent" */
-#define GRPC_MDSTR_USER_AGENT (grpc_static_slice_table[16])
+#define GRPC_MDSTR_USER_AGENT (grpc_static_slice_table[19])
 /* "host" */
-#define GRPC_MDSTR_HOST (grpc_static_slice_table[17])
+#define GRPC_MDSTR_HOST (grpc_static_slice_table[20])
 /* "lb-token" */
-#define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[18])
+#define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[21])
 /* "grpc-timeout" */
-#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[19])
+#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[22])
 /* "" */
-#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[20])
+#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[23])
 /* "grpc.wait_for_ready" */
-#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[21])
+#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[24])
 /* "grpc.timeout" */
-#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[22])
+#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[25])
 /* "grpc.max_request_message_bytes" */
 #define GRPC_MDSTR_GRPC_DOT_MAX_REQUEST_MESSAGE_BYTES \
-  (grpc_static_slice_table[23])
+  (grpc_static_slice_table[26])
 /* "grpc.max_response_message_bytes" */
 #define GRPC_MDSTR_GRPC_DOT_MAX_RESPONSE_MESSAGE_BYTES \
-  (grpc_static_slice_table[24])
+  (grpc_static_slice_table[27])
 /* "/grpc.lb.v1.LoadBalancer/BalanceLoad" */
 #define GRPC_MDSTR_SLASH_GRPC_DOT_LB_DOT_V1_DOT_LOADBALANCER_SLASH_BALANCELOAD \
-  (grpc_static_slice_table[25])
+  (grpc_static_slice_table[28])
 /* "0" */
-#define GRPC_MDSTR_0 (grpc_static_slice_table[26])
+#define GRPC_MDSTR_0 (grpc_static_slice_table[29])
 /* "1" */
-#define GRPC_MDSTR_1 (grpc_static_slice_table[27])
+#define GRPC_MDSTR_1 (grpc_static_slice_table[30])
 /* "2" */
-#define GRPC_MDSTR_2 (grpc_static_slice_table[28])
+#define GRPC_MDSTR_2 (grpc_static_slice_table[31])
 /* "identity" */
-#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[29])
+#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[32])
 /* "gzip" */
-#define GRPC_MDSTR_GZIP (grpc_static_slice_table[30])
+#define GRPC_MDSTR_GZIP (grpc_static_slice_table[33])
 /* "deflate" */
-#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[31])
+#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[34])
 /* "trailers" */
-#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[32])
+#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[35])
 /* "application/grpc" */
-#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[33])
+#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[36])
 /* "POST" */
-#define GRPC_MDSTR_POST (grpc_static_slice_table[34])
+#define GRPC_MDSTR_POST (grpc_static_slice_table[37])
 /* "200" */
-#define GRPC_MDSTR_200 (grpc_static_slice_table[35])
+#define GRPC_MDSTR_200 (grpc_static_slice_table[38])
 /* "404" */
-#define GRPC_MDSTR_404 (grpc_static_slice_table[36])
+#define GRPC_MDSTR_404 (grpc_static_slice_table[39])
 /* "http" */
-#define GRPC_MDSTR_HTTP (grpc_static_slice_table[37])
+#define GRPC_MDSTR_HTTP (grpc_static_slice_table[40])
 /* "https" */
-#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[38])
+#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[41])
 /* "grpc" */
-#define GRPC_MDSTR_GRPC (grpc_static_slice_table[39])
+#define GRPC_MDSTR_GRPC (grpc_static_slice_table[42])
 /* "GET" */
-#define GRPC_MDSTR_GET (grpc_static_slice_table[40])
+#define GRPC_MDSTR_GET (grpc_static_slice_table[43])
 /* "PUT" */
-#define GRPC_MDSTR_PUT (grpc_static_slice_table[41])
+#define GRPC_MDSTR_PUT (grpc_static_slice_table[44])
 /* "/" */
-#define GRPC_MDSTR_SLASH (grpc_static_slice_table[42])
+#define GRPC_MDSTR_SLASH (grpc_static_slice_table[45])
 /* "/index.html" */
-#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[43])
+#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[46])
 /* "204" */
-#define GRPC_MDSTR_204 (grpc_static_slice_table[44])
+#define GRPC_MDSTR_204 (grpc_static_slice_table[47])
 /* "206" */
-#define GRPC_MDSTR_206 (grpc_static_slice_table[45])
+#define GRPC_MDSTR_206 (grpc_static_slice_table[48])
 /* "304" */
-#define GRPC_MDSTR_304 (grpc_static_slice_table[46])
+#define GRPC_MDSTR_304 (grpc_static_slice_table[49])
 /* "400" */
-#define GRPC_MDSTR_400 (grpc_static_slice_table[47])
+#define GRPC_MDSTR_400 (grpc_static_slice_table[50])
 /* "500" */
-#define GRPC_MDSTR_500 (grpc_static_slice_table[48])
+#define GRPC_MDSTR_500 (grpc_static_slice_table[51])
 /* "accept-charset" */
-#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[49])
-/* "accept-encoding" */
-#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[50])
+#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[52])
 /* "gzip, deflate" */
-#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[51])
+#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[53])
 /* "accept-language" */
-#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[52])
+#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[54])
 /* "accept-ranges" */
-#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[53])
+#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[55])
 /* "accept" */
-#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[54])
+#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[56])
 /* "access-control-allow-origin" */
-#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[55])
+#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[57])
 /* "age" */
-#define GRPC_MDSTR_AGE (grpc_static_slice_table[56])
+#define GRPC_MDSTR_AGE (grpc_static_slice_table[58])
 /* "allow" */
-#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[57])
+#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[59])
 /* "authorization" */
-#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[58])
+#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[60])
 /* "cache-control" */
-#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[59])
+#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[61])
 /* "content-disposition" */
-#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[60])
-/* "content-encoding" */
-#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[61])
+#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[62])
 /* "content-language" */
-#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[62])
+#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[63])
 /* "content-length" */
-#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[63])
+#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[64])
 /* "content-location" */
-#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[64])
+#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[65])
 /* "content-range" */
-#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[65])
+#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[66])
 /* "cookie" */
-#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[66])
+#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[67])
 /* "date" */
-#define GRPC_MDSTR_DATE (grpc_static_slice_table[67])
+#define GRPC_MDSTR_DATE (grpc_static_slice_table[68])
 /* "etag" */
-#define GRPC_MDSTR_ETAG (grpc_static_slice_table[68])
+#define GRPC_MDSTR_ETAG (grpc_static_slice_table[69])
 /* "expect" */
-#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[69])
+#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[70])
 /* "expires" */
-#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[70])
+#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[71])
 /* "from" */
-#define GRPC_MDSTR_FROM (grpc_static_slice_table[71])
+#define GRPC_MDSTR_FROM (grpc_static_slice_table[72])
 /* "if-match" */
-#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[72])
+#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[73])
 /* "if-modified-since" */
-#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[73])
+#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[74])
 /* "if-none-match" */
-#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[74])
+#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[75])
 /* "if-range" */
-#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[75])
+#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[76])
 /* "if-unmodified-since" */
-#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[76])
+#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[77])
 /* "last-modified" */
-#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[77])
+#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[78])
 /* "lb-cost-bin" */
-#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[78])
+#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[79])
 /* "link" */
-#define GRPC_MDSTR_LINK (grpc_static_slice_table[79])
+#define GRPC_MDSTR_LINK (grpc_static_slice_table[80])
 /* "location" */
-#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[80])
+#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[81])
 /* "max-forwards" */
-#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[81])
+#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[82])
 /* "proxy-authenticate" */
-#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[82])
+#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[83])
 /* "proxy-authorization" */
-#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[83])
+#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[84])
 /* "range" */
-#define GRPC_MDSTR_RANGE (grpc_static_slice_table[84])
+#define GRPC_MDSTR_RANGE (grpc_static_slice_table[85])
 /* "referer" */
-#define GRPC_MDSTR_REFERER (grpc_static_slice_table[85])
+#define GRPC_MDSTR_REFERER (grpc_static_slice_table[86])
 /* "refresh" */
-#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[86])
+#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[87])
 /* "retry-after" */
-#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[87])
+#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[88])
 /* "server" */
-#define GRPC_MDSTR_SERVER (grpc_static_slice_table[88])
+#define GRPC_MDSTR_SERVER (grpc_static_slice_table[89])
 /* "set-cookie" */
-#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[89])
+#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[90])
 /* "strict-transport-security" */
-#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[90])
+#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[91])
 /* "transfer-encoding" */
-#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[91])
+#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[92])
 /* "vary" */
-#define GRPC_MDSTR_VARY (grpc_static_slice_table[92])
+#define GRPC_MDSTR_VARY (grpc_static_slice_table[93])
 /* "via" */
-#define GRPC_MDSTR_VIA (grpc_static_slice_table[93])
+#define GRPC_MDSTR_VIA (grpc_static_slice_table[94])
 /* "www-authenticate" */
-#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[94])
+#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[95])
 /* "identity,deflate" */
-#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[95])
+#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[96])
 /* "identity,gzip" */
-#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[96])
+#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[97])
 /* "deflate,gzip" */
-#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[97])
+#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[98])
 /* "identity,deflate,gzip" */
 #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \
-  (grpc_static_slice_table[98])
+  (grpc_static_slice_table[99])
 
 extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable;
 extern grpc_slice_refcount
@@ -244,7 +247,7 @@ extern grpc_slice_refcount
 #define GRPC_STATIC_METADATA_INDEX(static_slice) \
   ((int)((static_slice).refcount - grpc_static_metadata_refcounts))
 
-#define GRPC_STATIC_MDELEM_COUNT 81
+#define GRPC_STATIC_MDELEM_COUNT 86
 extern grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 /* "grpc-status": "0" */
@@ -355,141 +358,156 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 /* "content-disposition": "" */
 #define GRPC_MDELEM_CONTENT_DISPOSITION_EMPTY \
   (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[35], GRPC_MDELEM_STORAGE_STATIC))
+/* "content-encoding": "identity" */
+#define GRPC_MDELEM_CONTENT_ENCODING_IDENTITY \
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[36], GRPC_MDELEM_STORAGE_STATIC))
+/* "content-encoding": "gzip" */
+#define GRPC_MDELEM_CONTENT_ENCODING_GZIP \
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[37], GRPC_MDELEM_STORAGE_STATIC))
 /* "content-encoding": "" */
 #define GRPC_MDELEM_CONTENT_ENCODING_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[36], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[38], GRPC_MDELEM_STORAGE_STATIC))
 /* "content-language": "" */
 #define GRPC_MDELEM_CONTENT_LANGUAGE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[37], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[39], GRPC_MDELEM_STORAGE_STATIC))
 /* "content-length": "" */
 #define GRPC_MDELEM_CONTENT_LENGTH_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[38], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[40], GRPC_MDELEM_STORAGE_STATIC))
 /* "content-location": "" */
 #define GRPC_MDELEM_CONTENT_LOCATION_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[39], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[41], GRPC_MDELEM_STORAGE_STATIC))
 /* "content-range": "" */
 #define GRPC_MDELEM_CONTENT_RANGE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[40], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[42], GRPC_MDELEM_STORAGE_STATIC))
 /* "content-type": "" */
 #define GRPC_MDELEM_CONTENT_TYPE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[41], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[43], GRPC_MDELEM_STORAGE_STATIC))
 /* "cookie": "" */
 #define GRPC_MDELEM_COOKIE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[42], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[44], GRPC_MDELEM_STORAGE_STATIC))
 /* "date": "" */
 #define GRPC_MDELEM_DATE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[43], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[45], GRPC_MDELEM_STORAGE_STATIC))
 /* "etag": "" */
 #define GRPC_MDELEM_ETAG_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[44], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[46], GRPC_MDELEM_STORAGE_STATIC))
 /* "expect": "" */
 #define GRPC_MDELEM_EXPECT_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[45], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[47], GRPC_MDELEM_STORAGE_STATIC))
 /* "expires": "" */
 #define GRPC_MDELEM_EXPIRES_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[46], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[48], GRPC_MDELEM_STORAGE_STATIC))
 /* "from": "" */
 #define GRPC_MDELEM_FROM_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[47], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[49], GRPC_MDELEM_STORAGE_STATIC))
 /* "host": "" */
 #define GRPC_MDELEM_HOST_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[48], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[50], GRPC_MDELEM_STORAGE_STATIC))
 /* "if-match": "" */
 #define GRPC_MDELEM_IF_MATCH_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[49], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[51], GRPC_MDELEM_STORAGE_STATIC))
 /* "if-modified-since": "" */
 #define GRPC_MDELEM_IF_MODIFIED_SINCE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[50], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[52], GRPC_MDELEM_STORAGE_STATIC))
 /* "if-none-match": "" */
 #define GRPC_MDELEM_IF_NONE_MATCH_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[51], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[53], GRPC_MDELEM_STORAGE_STATIC))
 /* "if-range": "" */
 #define GRPC_MDELEM_IF_RANGE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[52], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[54], GRPC_MDELEM_STORAGE_STATIC))
 /* "if-unmodified-since": "" */
 #define GRPC_MDELEM_IF_UNMODIFIED_SINCE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[53], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC))
 /* "last-modified": "" */
 #define GRPC_MDELEM_LAST_MODIFIED_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[54], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC))
 /* "lb-token": "" */
 #define GRPC_MDELEM_LB_TOKEN_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC))
 /* "lb-cost-bin": "" */
 #define GRPC_MDELEM_LB_COST_BIN_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC))
 /* "link": "" */
 #define GRPC_MDELEM_LINK_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC))
 /* "location": "" */
 #define GRPC_MDELEM_LOCATION_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC))
 /* "max-forwards": "" */
 #define GRPC_MDELEM_MAX_FORWARDS_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC))
 /* "proxy-authenticate": "" */
 #define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC))
 /* "proxy-authorization": "" */
 #define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC))
 /* "range": "" */
 #define GRPC_MDELEM_RANGE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC))
 /* "referer": "" */
 #define GRPC_MDELEM_REFERER_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC))
 /* "refresh": "" */
 #define GRPC_MDELEM_REFRESH_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC))
 /* "retry-after": "" */
 #define GRPC_MDELEM_RETRY_AFTER_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC))
 /* "server": "" */
 #define GRPC_MDELEM_SERVER_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC))
 /* "set-cookie": "" */
 #define GRPC_MDELEM_SET_COOKIE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC))
 /* "strict-transport-security": "" */
 #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC))
 /* "transfer-encoding": "" */
 #define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC))
 /* "user-agent": "" */
 #define GRPC_MDELEM_USER_AGENT_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC))
 /* "vary": "" */
 #define GRPC_MDELEM_VARY_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC))
 /* "via": "" */
 #define GRPC_MDELEM_VIA_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC))
 /* "www-authenticate": "" */
 #define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "identity" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "deflate" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "identity,deflate" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "gzip" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "identity,gzip" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "deflate,gzip" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[81], GRPC_MDELEM_STORAGE_STATIC))
 /* "grpc-accept-encoding": "identity,deflate,gzip" */
 #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \
-  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC))
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[82], GRPC_MDELEM_STORAGE_STATIC))
+/* "accept-encoding": "identity" */
+#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY \
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[83], GRPC_MDELEM_STORAGE_STATIC))
+/* "accept-encoding": "gzip" */
+#define GRPC_MDELEM_ACCEPT_ENCODING_GZIP \
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[84], GRPC_MDELEM_STORAGE_STATIC))
+/* "accept-encoding": "identity,gzip" */
+#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[85], GRPC_MDELEM_STORAGE_STATIC))
 
 grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b);
 typedef enum {
@@ -508,7 +526,10 @@ typedef enum {
   GRPC_BATCH_GRPC_TAGS_BIN,
   GRPC_BATCH_GRPC_TRACE_BIN,
   GRPC_BATCH_CONTENT_TYPE,
+  GRPC_BATCH_CONTENT_ENCODING,
+  GRPC_BATCH_ACCEPT_ENCODING,
   GRPC_BATCH_GRPC_INTERNAL_ENCODING_REQUEST,
+  GRPC_BATCH_GRPC_INTERNAL_STREAM_ENCODING_REQUEST,
   GRPC_BATCH_USER_AGENT,
   GRPC_BATCH_HOST,
   GRPC_BATCH_LB_TOKEN,
@@ -533,7 +554,10 @@ typedef union {
     struct grpc_linked_mdelem *grpc_tags_bin;
     struct grpc_linked_mdelem *grpc_trace_bin;
     struct grpc_linked_mdelem *content_type;
+    struct grpc_linked_mdelem *content_encoding;
+    struct grpc_linked_mdelem *accept_encoding;
     struct grpc_linked_mdelem *grpc_internal_encoding_request;
+    struct grpc_linked_mdelem *grpc_internal_stream_encoding_request;
     struct grpc_linked_mdelem *user_agent;
     struct grpc_linked_mdelem *host;
     struct grpc_linked_mdelem *lb_token;
@@ -552,4 +576,10 @@ extern const uint8_t grpc_static_accept_encoding_metadata[8];
   (GRPC_MAKE_MDELEM(                                                           \
       &grpc_static_mdelem_table[grpc_static_accept_encoding_metadata[(algs)]], \
       GRPC_MDELEM_STORAGE_STATIC))
+
+extern const uint8_t grpc_static_accept_stream_encoding_metadata[4];
+#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs)                \
+  (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table                                  \
+                        [grpc_static_accept_stream_encoding_metadata[(algs)]], \
+                    GRPC_MDELEM_STORAGE_STATIC))
 #endif /* GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H */
diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c
index 7281602d66..650b0559aa 100644
--- a/src/core/lib/transport/transport.c
+++ b/src/core/lib/transport/transport.c
@@ -197,37 +197,40 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
                                     then_schedule_closure);
 }
 
-char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
-                              grpc_transport *transport) {
-  return transport->vtable->get_peer(exec_ctx, transport);
-}
-
 grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx,
                                            grpc_transport *transport) {
   return transport->vtable->get_endpoint(exec_ctx, transport);
 }
 
+// This comment should be sung to the tune of
+// "Supercalifragilisticexpialidocious":
+//
 // grpc_transport_stream_op_batch_finish_with_failure
 // is a function that must always unref cancel_error
 // though it lives in lib, it handles transport stream ops sure
 // it's grpc_transport_stream_op_batch_finish_with_failure
-
 void grpc_transport_stream_op_batch_finish_with_failure(
-    grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *op,
-    grpc_error *error) {
-  if (op->recv_message) {
-    GRPC_CLOSURE_SCHED(exec_ctx, op->payload->recv_message.recv_message_ready,
-                       GRPC_ERROR_REF(error));
+    grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *batch,
+    grpc_error *error, grpc_call_combiner *call_combiner) {
+  if (batch->send_message) {
+    grpc_byte_stream_destroy(exec_ctx,
+                             batch->payload->send_message.send_message);
+  }
+  if (batch->recv_message) {
+    GRPC_CALL_COMBINER_START(exec_ctx, call_combiner,
+                             batch->payload->recv_message.recv_message_ready,
+                             GRPC_ERROR_REF(error),
+                             "failing recv_message_ready");
   }
-  if (op->recv_initial_metadata) {
-    GRPC_CLOSURE_SCHED(
-        exec_ctx,
-        op->payload->recv_initial_metadata.recv_initial_metadata_ready,
-        GRPC_ERROR_REF(error));
+  if (batch->recv_initial_metadata) {
+    GRPC_CALL_COMBINER_START(
+        exec_ctx, call_combiner,
+        batch->payload->recv_initial_metadata.recv_initial_metadata_ready,
+        GRPC_ERROR_REF(error), "failing recv_initial_metadata_ready");
   }
-  GRPC_CLOSURE_SCHED(exec_ctx, op->on_complete, error);
-  if (op->cancel_stream) {
-    GRPC_ERROR_UNREF(op->payload->cancel_stream.cancel_error);
+  GRPC_CLOSURE_SCHED(exec_ctx, batch->on_complete, error);
+  if (batch->cancel_stream) {
+    GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error);
   }
 }
 
diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h
index 84e53e683a..fbf5dcb8b5 100644
--- a/src/core/lib/transport/transport.h
+++ b/src/core/lib/transport/transport.h
@@ -22,6 +22,7 @@
 #include <stddef.h>
 
 #include "src/core/lib/channel/context.h"
+#include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
@@ -152,6 +153,9 @@ struct grpc_transport_stream_op_batch_payload {
     /** Iff send_initial_metadata != NULL, flags associated with
         send_initial_metadata: a bitfield of GRPC_INITIAL_METADATA_xxx */
     uint32_t send_initial_metadata_flags;
+    // If non-NULL, will be set by the transport to the peer string
+    // (a char*, which the caller takes ownership of).
+    gpr_atm *peer_string;
   } send_initial_metadata;
 
   struct {
@@ -159,6 +163,11 @@ struct grpc_transport_stream_op_batch_payload {
   } send_trailing_metadata;
 
   struct {
+    // The transport (or a filter that decides to return a failure before
+    // the op gets down to the transport) is responsible for calling
+    // grpc_byte_stream_destroy() on this.
+    // The batch's on_complete will not be called until after the byte
+    // stream is destroyed.
     grpc_byte_stream *send_message;
   } send_message;
 
@@ -171,9 +180,16 @@ struct grpc_transport_stream_op_batch_payload {
     // immediately available.  This may be a signal that we received a
     // Trailers-Only response.
     bool *trailing_metadata_available;
+    // If non-NULL, will be set by the transport to the peer string
+    // (a char*, which the caller takes ownership of).
+    gpr_atm *peer_string;
   } recv_initial_metadata;
 
   struct {
+    // Will be set by the transport to point to the byte stream
+    // containing a received message.
+    // The caller is responsible for calling grpc_byte_stream_destroy()
+    // on this byte stream.
     grpc_byte_stream **recv_message;
     /** Should be enqueued when one message is ready to be processed. */
     grpc_closure *recv_message_ready;
@@ -284,7 +300,7 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
 
 void grpc_transport_stream_op_batch_finish_with_failure(
     grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *op,
-    grpc_error *error);
+    grpc_error *error, grpc_call_combiner *call_combiner);
 
 char *grpc_transport_stream_op_batch_string(grpc_transport_stream_op_batch *op);
 char *grpc_transport_op_string(grpc_transport_op *op);
@@ -323,10 +339,6 @@ void grpc_transport_close(grpc_transport *transport);
 /* Destroy the transport */
 void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, grpc_transport *transport);
 
-/* Get the transports peer */
-char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
-                              grpc_transport *transport);
-
 /* Get the endpoint used by \a transport */
 grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx,
                                            grpc_transport *transport);
diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h
index fc772c6dd1..bbae69c223 100644
--- a/src/core/lib/transport/transport_impl.h
+++ b/src/core/lib/transport/transport_impl.h
@@ -59,9 +59,6 @@ typedef struct grpc_transport_vtable {
   /* implementation of grpc_transport_destroy */
   void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_transport *self);
 
-  /* implementation of grpc_transport_get_peer */
-  char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_transport *self);
-
   /* implementation of grpc_transport_get_endpoint */
   grpc_endpoint *(*get_endpoint)(grpc_exec_ctx *exec_ctx, grpc_transport *self);
 } grpc_transport_vtable;
diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c
index 7b18229ba6..409a6c4103 100644
--- a/src/core/lib/transport/transport_op_string.c
+++ b/src/core/lib/transport/transport_op_string.c
@@ -112,6 +112,13 @@ char *grpc_transport_stream_op_batch_string(
     gpr_strvec_add(&b, tmp);
   }
 
+  if (op->collect_stats) {
+    gpr_strvec_add(&b, gpr_strdup(" "));
+    gpr_asprintf(&tmp, "COLLECT_STATS:%p",
+                 op->payload->collect_stats.collect_stats);
+    gpr_strvec_add(&b, tmp);
+  }
+
   out = gpr_strvec_flatten(&b, NULL);
   gpr_strvec_destroy(&b);
 
diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c
index 1280680663..967126ecee 100644
--- a/src/core/tsi/fake_transport_security.c
+++ b/src/core/tsi/fake_transport_security.c
@@ -391,7 +391,7 @@ static tsi_result fake_handshaker_result_create_frame_protector(
 }
 
 static tsi_result fake_handshaker_result_get_unused_bytes(
-    const tsi_handshaker_result *self, unsigned char **bytes,
+    const tsi_handshaker_result *self, const unsigned char **bytes,
     size_t *bytes_size) {
   fake_handshaker_result *result = (fake_handshaker_result *)self;
   *bytes_size = result->unused_bytes_size;
@@ -407,8 +407,10 @@ static void fake_handshaker_result_destroy(tsi_handshaker_result *self) {
 
 static const tsi_handshaker_result_vtable handshaker_result_vtable = {
     fake_handshaker_result_extract_peer,
+    NULL, /* create_zero_copy_grpc_protector */
     fake_handshaker_result_create_frame_protector,
-    fake_handshaker_result_get_unused_bytes, fake_handshaker_result_destroy,
+    fake_handshaker_result_get_unused_bytes,
+    fake_handshaker_result_destroy,
 };
 
 static tsi_result fake_handshaker_result_create(
@@ -530,7 +532,7 @@ static void fake_handshaker_destroy(tsi_handshaker *self) {
 
 static tsi_result fake_handshaker_next(
     tsi_handshaker *self, const unsigned char *received_bytes,
-    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t received_bytes_size, const unsigned char **bytes_to_send,
     size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
     tsi_handshaker_on_next_done_cb cb, void *user_data) {
   /* Sanity check the arguments. */
diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c
index be11d64472..76213072a3 100644
--- a/src/core/tsi/transport_security.c
+++ b/src/core/tsi/transport_security.c
@@ -74,14 +74,12 @@ tsi_result tsi_frame_protector_protect(tsi_frame_protector *self,
                                        size_t *unprotected_bytes_size,
                                        unsigned char *protected_output_frames,
                                        size_t *protected_output_frames_size) {
-  if (self == NULL || unprotected_bytes == NULL ||
+  if (self == NULL || self->vtable == NULL || unprotected_bytes == NULL ||
       unprotected_bytes_size == NULL || protected_output_frames == NULL ||
       protected_output_frames_size == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
-  if (self->vtable == NULL || self->vtable->protect == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->protect == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->protect(self, unprotected_bytes, unprotected_bytes_size,
                                protected_output_frames,
                                protected_output_frames_size);
@@ -90,13 +88,11 @@ tsi_result tsi_frame_protector_protect(tsi_frame_protector *self,
 tsi_result tsi_frame_protector_protect_flush(
     tsi_frame_protector *self, unsigned char *protected_output_frames,
     size_t *protected_output_frames_size, size_t *still_pending_size) {
-  if (self == NULL || protected_output_frames == NULL ||
+  if (self == NULL || self->vtable == NULL || protected_output_frames == NULL ||
       protected_output_frames_size == NULL || still_pending_size == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
-  if (self->vtable == NULL || self->vtable->protect_flush == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->protect_flush == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->protect_flush(self, protected_output_frames,
                                      protected_output_frames_size,
                                      still_pending_size);
@@ -106,14 +102,12 @@ tsi_result tsi_frame_protector_unprotect(
     tsi_frame_protector *self, const unsigned char *protected_frames_bytes,
     size_t *protected_frames_bytes_size, unsigned char *unprotected_bytes,
     size_t *unprotected_bytes_size) {
-  if (self == NULL || protected_frames_bytes == NULL ||
+  if (self == NULL || self->vtable == NULL || protected_frames_bytes == NULL ||
       protected_frames_bytes_size == NULL || unprotected_bytes == NULL ||
       unprotected_bytes_size == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
-  if (self->vtable == NULL || self->vtable->unprotect == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->unprotect == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->unprotect(self, protected_frames_bytes,
                                  protected_frames_bytes_size, unprotected_bytes,
                                  unprotected_bytes_size);
@@ -131,48 +125,44 @@ void tsi_frame_protector_destroy(tsi_frame_protector *self) {
 tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self,
                                                     unsigned char *bytes,
                                                     size_t *bytes_size) {
-  if (self == NULL || bytes == NULL || bytes_size == NULL) {
+  if (self == NULL || self->vtable == NULL || bytes == NULL ||
+      bytes_size == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
   if (self->frame_protector_created) return TSI_FAILED_PRECONDITION;
-  if (self->vtable == NULL || self->vtable->get_bytes_to_send_to_peer == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->get_bytes_to_send_to_peer == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->get_bytes_to_send_to_peer(self, bytes, bytes_size);
 }
 
 tsi_result tsi_handshaker_process_bytes_from_peer(tsi_handshaker *self,
                                                   const unsigned char *bytes,
                                                   size_t *bytes_size) {
-  if (self == NULL || bytes == NULL || bytes_size == NULL) {
+  if (self == NULL || self->vtable == NULL || bytes == NULL ||
+      bytes_size == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
   if (self->frame_protector_created) return TSI_FAILED_PRECONDITION;
-  if (self->vtable == NULL || self->vtable->process_bytes_from_peer == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->process_bytes_from_peer == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->process_bytes_from_peer(self, bytes, bytes_size);
 }
 
 tsi_result tsi_handshaker_get_result(tsi_handshaker *self) {
-  if (self == NULL) return TSI_INVALID_ARGUMENT;
+  if (self == NULL || self->vtable == NULL) return TSI_INVALID_ARGUMENT;
   if (self->frame_protector_created) return TSI_FAILED_PRECONDITION;
-  if (self->vtable == NULL || self->vtable->get_result == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->get_result == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->get_result(self);
 }
 
 tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer) {
-  if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT;
+  if (self == NULL || self->vtable == NULL || peer == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
   memset(peer, 0, sizeof(tsi_peer));
   if (self->frame_protector_created) return TSI_FAILED_PRECONDITION;
   if (tsi_handshaker_get_result(self) != TSI_OK) {
     return TSI_FAILED_PRECONDITION;
   }
-  if (self->vtable == NULL || self->vtable->extract_peer == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->extract_peer == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->extract_peer(self, peer);
 }
 
@@ -180,14 +170,12 @@ tsi_result tsi_handshaker_create_frame_protector(
     tsi_handshaker *self, size_t *max_protected_frame_size,
     tsi_frame_protector **protector) {
   tsi_result result;
-  if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT;
-  if (self->frame_protector_created) return TSI_FAILED_PRECONDITION;
-  if (tsi_handshaker_get_result(self) != TSI_OK) {
-    return TSI_FAILED_PRECONDITION;
-  }
-  if (self->vtable == NULL || self->vtable->create_frame_protector == NULL) {
-    return TSI_UNIMPLEMENTED;
+  if (self == NULL || self->vtable == NULL || protector == NULL) {
+    return TSI_INVALID_ARGUMENT;
   }
+  if (self->frame_protector_created) return TSI_FAILED_PRECONDITION;
+  if (tsi_handshaker_get_result(self) != TSI_OK) return TSI_FAILED_PRECONDITION;
+  if (self->vtable->create_frame_protector == NULL) return TSI_UNIMPLEMENTED;
   result = self->vtable->create_frame_protector(self, max_protected_frame_size,
                                                 protector);
   if (result == TSI_OK) {
@@ -198,14 +186,12 @@ tsi_result tsi_handshaker_create_frame_protector(
 
 tsi_result tsi_handshaker_next(
     tsi_handshaker *self, const unsigned char *received_bytes,
-    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t received_bytes_size, const unsigned char **bytes_to_send,
     size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
     tsi_handshaker_on_next_done_cb cb, void *user_data) {
-  if (self == NULL) return TSI_INVALID_ARGUMENT;
+  if (self == NULL || self->vtable == NULL) return TSI_INVALID_ARGUMENT;
   if (self->handshaker_result_created) return TSI_FAILED_PRECONDITION;
-  if (self->vtable == NULL || self->vtable->next == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->next == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->next(self, received_bytes, received_bytes_size,
                             bytes_to_send, bytes_to_send_size,
                             handshaker_result, cb, user_data);
@@ -220,34 +206,33 @@ void tsi_handshaker_destroy(tsi_handshaker *self) {
 
 tsi_result tsi_handshaker_result_extract_peer(const tsi_handshaker_result *self,
                                               tsi_peer *peer) {
-  if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT;
-  memset(peer, 0, sizeof(tsi_peer));
-  if (self->vtable == NULL || self->vtable->extract_peer == NULL) {
-    return TSI_UNIMPLEMENTED;
+  if (self == NULL || self->vtable == NULL || peer == NULL) {
+    return TSI_INVALID_ARGUMENT;
   }
+  memset(peer, 0, sizeof(tsi_peer));
+  if (self->vtable->extract_peer == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->extract_peer(self, peer);
 }
 
 tsi_result tsi_handshaker_result_create_frame_protector(
     const tsi_handshaker_result *self, size_t *max_protected_frame_size,
     tsi_frame_protector **protector) {
-  if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT;
-  if (self->vtable == NULL || self->vtable->create_frame_protector == NULL) {
-    return TSI_UNIMPLEMENTED;
+  if (self == NULL || self->vtable == NULL || protector == NULL) {
+    return TSI_INVALID_ARGUMENT;
   }
+  if (self->vtable->create_frame_protector == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->create_frame_protector(self, max_protected_frame_size,
                                               protector);
 }
 
 tsi_result tsi_handshaker_result_get_unused_bytes(
-    const tsi_handshaker_result *self, unsigned char **bytes,
+    const tsi_handshaker_result *self, const unsigned char **bytes,
     size_t *bytes_size) {
-  if (self == NULL || bytes == NULL || bytes_size == NULL) {
+  if (self == NULL || self->vtable == NULL || bytes == NULL ||
+      bytes_size == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
-  if (self->vtable == NULL || self->vtable->get_unused_bytes == NULL) {
-    return TSI_UNIMPLEMENTED;
-  }
+  if (self->vtable->get_unused_bytes == NULL) return TSI_UNIMPLEMENTED;
   return self->vtable->get_unused_bytes(self, bytes, bytes_size);
 }
 
diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h
index 4a56c25602..b0d7039850 100644
--- a/src/core/tsi/transport_security.h
+++ b/src/core/tsi/transport_security.h
@@ -70,7 +70,8 @@ typedef struct {
                                        tsi_frame_protector **protector);
   void (*destroy)(tsi_handshaker *self);
   tsi_result (*next)(tsi_handshaker *self, const unsigned char *received_bytes,
-                     size_t received_bytes_size, unsigned char **bytes_to_send,
+                     size_t received_bytes_size,
+                     const unsigned char **bytes_to_send,
                      size_t *bytes_to_send_size,
                      tsi_handshaker_result **handshaker_result,
                      tsi_handshaker_on_next_done_cb cb, void *user_data);
@@ -86,11 +87,16 @@ struct tsi_handshaker {
    See transport_security_interface.h for documentation. */
 typedef struct {
   tsi_result (*extract_peer)(const tsi_handshaker_result *self, tsi_peer *peer);
+  tsi_result (*create_zero_copy_grpc_protector)(
+      const tsi_handshaker_result *self,
+      size_t *max_output_protected_frame_size,
+      tsi_zero_copy_grpc_protector **protector);
   tsi_result (*create_frame_protector)(const tsi_handshaker_result *self,
                                        size_t *max_output_protected_frame_size,
                                        tsi_frame_protector **protector);
   tsi_result (*get_unused_bytes)(const tsi_handshaker_result *self,
-                                 unsigned char **bytes, size_t *bytes_size);
+                                 const unsigned char **bytes,
+                                 size_t *bytes_size);
   void (*destroy)(tsi_handshaker_result *self);
 } tsi_handshaker_result_vtable;
 
diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c
index a0564945e4..1c2a57b3bd 100644
--- a/src/core/tsi/transport_security_adapter.c
+++ b/src/core/tsi/transport_security_adapter.c
@@ -50,7 +50,7 @@ static tsi_result adapter_result_create_frame_protector(
 }
 
 static tsi_result adapter_result_get_unused_bytes(
-    const tsi_handshaker_result *self, unsigned char **bytes,
+    const tsi_handshaker_result *self, const unsigned char **bytes,
     size_t *byte_size) {
   tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self;
   *bytes = impl->unused_bytes;
@@ -66,8 +66,11 @@ static void adapter_result_destroy(tsi_handshaker_result *self) {
 }
 
 static const tsi_handshaker_result_vtable result_vtable = {
-    adapter_result_extract_peer, adapter_result_create_frame_protector,
-    adapter_result_get_unused_bytes, adapter_result_destroy,
+    adapter_result_extract_peer,
+    NULL, /* create_zero_copy_grpc_protector */
+    adapter_result_create_frame_protector,
+    adapter_result_get_unused_bytes,
+    adapter_result_destroy,
 };
 
 /* Ownership of wrapped tsi_handshaker is transferred to the result object.  */
@@ -140,7 +143,7 @@ static void adapter_destroy(tsi_handshaker *self) {
 
 static tsi_result adapter_next(
     tsi_handshaker *self, const unsigned char *received_bytes,
-    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t received_bytes_size, const unsigned char **bytes_to_send,
     size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
     tsi_handshaker_on_next_done_cb cb, void *user_data) {
   /* Input sanity check.  */
diff --git a/src/core/tsi/transport_security_grpc.c b/src/core/tsi/transport_security_grpc.c
new file mode 100644
index 0000000000..5bcfdfa61f
--- /dev/null
+++ b/src/core/tsi/transport_security_grpc.c
@@ -0,0 +1,64 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/tsi/transport_security_grpc.h"
+
+/* This method creates a tsi_zero_copy_grpc_protector object.  */
+tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector(
+    const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_zero_copy_grpc_protector **protector) {
+  if (self == NULL || self->vtable == NULL || protector == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  if (self->vtable->create_zero_copy_grpc_protector == NULL) {
+    return TSI_UNIMPLEMENTED;
+  }
+  return self->vtable->create_zero_copy_grpc_protector(
+      self, max_output_protected_frame_size, protector);
+}
+
+/* --- tsi_zero_copy_grpc_protector common implementation. ---
+
+   Calls specific implementation after state/input validation. */
+
+tsi_result tsi_zero_copy_grpc_protector_protect(
+    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices,
+    grpc_slice_buffer *protected_slices) {
+  if (self == NULL || self->vtable == NULL || unprotected_slices == NULL ||
+      protected_slices == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  if (self->vtable->protect == NULL) return TSI_UNIMPLEMENTED;
+  return self->vtable->protect(self, unprotected_slices, protected_slices);
+}
+
+tsi_result tsi_zero_copy_grpc_protector_unprotect(
+    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices,
+    grpc_slice_buffer *unprotected_slices) {
+  if (self == NULL || self->vtable == NULL || protected_slices == NULL ||
+      unprotected_slices == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  if (self->vtable->unprotect == NULL) return TSI_UNIMPLEMENTED;
+  return self->vtable->unprotect(self, protected_slices, unprotected_slices);
+}
+
+void tsi_zero_copy_grpc_protector_destroy(tsi_zero_copy_grpc_protector *self) {
+  if (self == NULL) return;
+  self->vtable->destroy(self);
+}
diff --git a/src/core/tsi/transport_security_grpc.h b/src/core/tsi/transport_security_grpc.h
new file mode 100644
index 0000000000..5ab5297cc4
--- /dev/null
+++ b/src/core/tsi/transport_security_grpc.h
@@ -0,0 +1,80 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_TSI_TRANSPORT_SECURITY_GRPC_H
+#define GRPC_CORE_TSI_TRANSPORT_SECURITY_GRPC_H
+
+#include <grpc/slice_buffer.h>
+#include "src/core/tsi/transport_security.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* This method creates a tsi_zero_copy_grpc_protector object. It return TSI_OK
+   assuming there is no fatal error.
+   The caller is responsible for destroying the protector.  */
+tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector(
+    const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_zero_copy_grpc_protector **protector);
+
+/* -- tsi_zero_copy_grpc_protector object --  */
+
+/* Outputs protected frames.
+   - unprotected_slices is the unprotected data to be protected.
+   - protected_slices is the protected output frames. One or more frames
+     may be produced in this protect function.
+   - This method returns TSI_OK in case of success or a specific error code in
+     case of failure.  */
+tsi_result tsi_zero_copy_grpc_protector_protect(
+    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices,
+    grpc_slice_buffer *protected_slices);
+
+/* Outputs unprotected bytes.
+   - protected_slices is the bytes of protected frames.
+   - unprotected_slices is the unprotected output data.
+   - This method returns TSI_OK in case of success. Success includes cases where
+     there is not enough data to output in which case unprotected_slices has 0
+     bytes.  */
+tsi_result tsi_zero_copy_grpc_protector_unprotect(
+    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices,
+    grpc_slice_buffer *unprotected_slices);
+
+/* Destroys the tsi_zero_copy_grpc_protector object.  */
+void tsi_zero_copy_grpc_protector_destroy(tsi_zero_copy_grpc_protector *self);
+
+/* Base for tsi_zero_copy_grpc_protector implementations.  */
+typedef struct {
+  tsi_result (*protect)(tsi_zero_copy_grpc_protector *self,
+                        grpc_slice_buffer *unprotected_slices,
+                        grpc_slice_buffer *protected_slices);
+  tsi_result (*unprotect)(tsi_zero_copy_grpc_protector *self,
+                          grpc_slice_buffer *protected_slices,
+                          grpc_slice_buffer *unprotected_slices);
+  void (*destroy)(tsi_zero_copy_grpc_protector *self);
+} tsi_zero_copy_grpc_protector_vtable;
+
+struct tsi_zero_copy_grpc_protector {
+  const tsi_zero_copy_grpc_protector_vtable *vtable;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_CORE_TSI_TRANSPORT_SECURITY_GRPC_H */
diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h
index 137f8ee5c3..80c426bbdb 100644
--- a/src/core/tsi/transport_security_interface.h
+++ b/src/core/tsi/transport_security_interface.h
@@ -62,6 +62,15 @@ const char *tsi_result_to_string(tsi_result result);
 
 extern grpc_tracer_flag tsi_tracing_enabled;
 
+/* -- tsi_zero_copy_grpc_protector object --
+
+  This object protects and unprotects grpc slice buffers with zero or minimized
+  memory copy once the handshake is done. Implementations of this object must be
+  thread compatible. This object depends on grpc and the details of this object
+  is defined in transport_security_grpc.h.  */
+
+typedef struct tsi_zero_copy_grpc_protector tsi_zero_copy_grpc_protector;
+
 /* --- tsi_frame_protector object ---
 
   This object protects and unprotects buffers once the handshake is done.
@@ -221,7 +230,7 @@ tsi_result tsi_handshaker_result_create_frame_protector(
    Ownership of the bytes is retained by the handshaker result. As a
    consequence, the caller must not free the bytes.  */
 tsi_result tsi_handshaker_result_get_unused_bytes(
-    const tsi_handshaker_result *self, unsigned char **bytes,
+    const tsi_handshaker_result *self, const unsigned char **bytes,
     size_t *byte_size);
 
 /* This method releases the tsi_handshaker_handshaker object. After this method
@@ -429,7 +438,7 @@ typedef void (*tsi_handshaker_on_next_done_cb)(
    tsi_handshaker object.  */
 tsi_result tsi_handshaker_next(
     tsi_handshaker *self, const unsigned char *received_bytes,
-    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t received_bytes_size, const unsigned char **bytes_to_send,
     size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
     tsi_handshaker_on_next_done_cb cb, void *user_data);