Merge github.com:grpc/grpc into bigger_frames

46 files changed, 1209 insertions, 891 deletions

diff --git a/src/core/ext/census/gen/README.md b/src/core/ext/census/gen/README.md
index 72bef6542d..fdbac1084c 100644
--- a/src/core/ext/census/gen/README.md
+++ b/src/core/ext/census/gen/README.md
@@ -4,3 +4,7 @@ Files generated for use by Census stats and trace recording subsystem.
 * census.pb.{h,c} - Generated from src/core/ext/census/census.proto, using the
   script `tools/codegen/core/gen_nano_proto.sh src/proto/census/census.proto
   $PWD/src/core/ext/census/gen src/core/ext/census/gen`
+* trace_context.pb.{h,c} - Generated from
+  src/core/ext/census/trace_context.proto, using the script
+  `tools/codegen/core/gen_nano_proto.sh src/proto/census/trace_context.proto
+  $PWD/src/core/ext/census/gen src/core/ext/census/gen`
diff --git a/src/core/ext/census/gen/trace_context.pb.c b/src/core/ext/census/gen/trace_context.pb.c
new file mode 100644
index 0000000000..c8aea324ce
--- /dev/null
+++ b/src/core/ext/census/gen/trace_context.pb.c
@@ -0,0 +1,81 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.3.5-dev */
+
+#include "src/core/ext/census/gen/trace_context.pb.h"
+
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+
+
+const pb_field_t google_trace_TraceId_fields[3] = {
+    PB_FIELD(  1, FIXED64 , OPTIONAL, STATIC  , FIRST, google_trace_TraceId, hi, hi, 0),
+    PB_FIELD(  2, FIXED64 , OPTIONAL, STATIC  , OTHER, google_trace_TraceId, lo, hi, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_trace_TraceContext_fields[4] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, google_trace_TraceContext, trace_id, trace_id, &google_trace_TraceId_fields),
+    PB_FIELD(  2, FIXED64 , OPTIONAL, STATIC  , OTHER, google_trace_TraceContext, span_id, trace_id, 0),
+    PB_FIELD(  3, BOOL    , OPTIONAL, STATIC  , OTHER, google_trace_TraceContext, is_sampled, span_id, 0),
+    PB_LAST_FIELD
+};
+
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_32BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in 8 or 16 bit
+ * field descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_trace_TraceContext, trace_id) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_google_trace_TraceId_google_trace_TraceContext)
+#endif
+
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_16BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in the default
+ * 8 bit descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_trace_TraceContext, trace_id) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_google_trace_TraceId_google_trace_TraceContext)
+#endif
+
+
diff --git a/src/core/ext/census/gen/trace_context.pb.h b/src/core/ext/census/gen/trace_context.pb.h
new file mode 100644
index 0000000000..263c4c58cb
--- /dev/null
+++ b/src/core/ext/census/gen/trace_context.pb.h
@@ -0,0 +1,99 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.3.5-dev */
+
+#ifndef GRPC_CORE_EXT_CENSUS_GEN_TRACE_CONTEXT_PB_H
+#define GRPC_CORE_EXT_CENSUS_GEN_TRACE_CONTEXT_PB_H
+#include "third_party/nanopb/pb.h"
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Struct definitions */
+typedef struct _google_trace_TraceId {
+    bool has_hi;
+    uint64_t hi;
+    bool has_lo;
+    uint64_t lo;
+} google_trace_TraceId;
+
+typedef struct _google_trace_TraceContext {
+    bool has_trace_id;
+    google_trace_TraceId trace_id;
+    bool has_span_id;
+    uint64_t span_id;
+    bool has_is_sampled;
+    bool is_sampled;
+} google_trace_TraceContext;
+
+/* Default values for struct fields */
+
+/* Initializer values for message structs */
+#define google_trace_TraceId_init_default        {false, 0, false, 0}
+#define google_trace_TraceContext_init_default   {false, google_trace_TraceId_init_default, false, 0, false, 0}
+#define google_trace_TraceId_init_zero           {false, 0, false, 0}
+#define google_trace_TraceContext_init_zero      {false, google_trace_TraceId_init_zero, false, 0, false, 0}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define google_trace_TraceId_hi_tag              1
+#define google_trace_TraceId_lo_tag              2
+#define google_trace_TraceContext_trace_id_tag   1
+#define google_trace_TraceContext_span_id_tag    2
+#define google_trace_TraceContext_is_sampled_tag 3
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t google_trace_TraceId_fields[3];
+extern const pb_field_t google_trace_TraceContext_fields[4];
+
+/* Maximum encoded size of messages (where known) */
+#define google_trace_TraceId_size                18
+#define google_trace_TraceContext_size           31
+
+/* Message IDs (where set with "msgid" option) */
+#ifdef PB_MSGID
+
+#define TRACE_CONTEXT_MESSAGES \
+
+
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/src/core/ext/client_config/README.md b/src/core/ext/client_config/README.md
index 7024fd540d..eda01e3e71 100644
--- a/src/core/ext/client_config/README.md
+++ b/src/core/ext/client_config/README.md
@@ -12,7 +12,7 @@ data might include:
 - a load balancing policy to decide which server to send a request to
 - a set of filters to mutate outgoing requests (say, by adding metadata)
 
-The resolver provides this data as a stream of grpc_client_config objects to
+The resolver provides this data as a stream of grpc_resolver_result objects to
 the channel. We represent configuration as a stream so that it can be changed
 by the resolver during execution, by reacting to external events (such as a
 new configuration file being pushed to some store).
@@ -22,7 +22,7 @@ Load Balancing
 --------------
 
 Load balancing configuration is provided by a grpc_lb_policy object, stored as
-part of grpc_client_config.
+part of grpc_resolver_result.
 
 The primary job of the load balancing policies is to pick a target server given only the
 initial metadata for a request. It does this by providing a grpc_subchannel
diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c
index 2c0c4abffc..61e012578e 100644
--- a/src/core/ext/client_config/client_channel.c
+++ b/src/core/ext/client_config/client_channel.c
@@ -33,6 +33,7 @@
 
 #include "src/core/ext/client_config/client_channel.h"
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -41,10 +42,11 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/ext/client_config/subchannel_call_holder.h"
+#include "src/core/ext/client_config/subchannel.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/channel.h"
@@ -52,30 +54,31 @@
 
 /* Client channel implementation */
 
-typedef grpc_subchannel_call_holder call_data;
+/*************************************************************************
+ * CHANNEL-WIDE FUNCTIONS
+ */
 
 typedef struct client_channel_channel_data {
   /** resolver for this channel */
   grpc_resolver *resolver;
   /** have we started resolving this channel */
-  int started_resolving;
+  bool started_resolving;
 
   /** mutex protecting client configuration, including all
       variables below in this data structure */
-  gpr_mu mu_config;
-  /** currently active load balancer - guarded by mu_config */
+  gpr_mu mu;
+  /** currently active load balancer - guarded by mu */
   grpc_lb_policy *lb_policy;
-  /** incoming configuration - set by resolver.next
-      guarded by mu_config */
-  grpc_client_config *incoming_configuration;
+  /** incoming resolver result - set by resolver.next(), guarded by mu */
+  grpc_resolver_result *resolver_result;
   /** a list of closures that are all waiting for config to come in */
   grpc_closure_list waiting_for_config_closures;
   /** resolver callback */
-  grpc_closure on_config_changed;
+  grpc_closure on_resolver_result_changed;
   /** connectivity state being tracked */
   grpc_connectivity_state_tracker state_tracker;
   /** when an lb_policy arrives, should we try to exit idle */
-  int exit_idle_when_lb_policy_arrives;
+  bool exit_idle_when_lb_policy_arrives;
   /** owning stack */
   grpc_channel_stack *owning_stack;
   /** interested parties (owned) */
@@ -83,10 +86,8 @@ typedef struct client_channel_channel_data {
 } channel_data;
 
 /** We create one watcher for each new lb_policy that is returned from a
-   resolver,
-    to watch for state changes from the lb_policy. When a state change is seen,
-   we
-    update the channel, and create a new watcher */
+    resolver, to watch for state changes from the lb_policy. When a state
+    change is seen, we update the channel, and create a new watcher. */
 typedef struct {
   channel_data *chand;
   grpc_closure on_changed;
@@ -94,22 +95,6 @@ typedef struct {
   grpc_lb_policy *lb_policy;
 } lb_policy_connectivity_watcher;
 
-typedef struct {
-  grpc_closure closure;
-  grpc_call_element *elem;
-} waiting_call;
-
-static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data);
-}
-
-static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
-                                         grpc_call_element *elem,
-                                         grpc_transport_stream_op *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-  grpc_subchannel_call_holder_perform_op(exec_ctx, elem->call_data, op);
-}
-
 static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
                             grpc_lb_policy *lb_policy,
                             grpc_connectivity_state current_state);
@@ -156,9 +141,9 @@ static void on_lb_policy_state_changed(grpc_exec_ctx *exec_ctx, void *arg,
                                        grpc_error *error) {
   lb_policy_connectivity_watcher *w = arg;
 
-  gpr_mu_lock(&w->chand->mu_config);
+  gpr_mu_lock(&w->chand->mu);
   on_lb_policy_state_changed_locked(exec_ctx, w, error);
-  gpr_mu_unlock(&w->chand->mu_config);
+  gpr_mu_unlock(&w->chand->mu);
 
   GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "watch_lb_policy");
   gpr_free(w);
@@ -178,17 +163,17 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
                                         &w->on_changed);
 }
 
-static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                 grpc_error *error) {
+static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg,
+                                       grpc_error *error) {
   channel_data *chand = arg;
   grpc_lb_policy *lb_policy = NULL;
   grpc_lb_policy *old_lb_policy;
   grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE;
-  int exit_idle = 0;
+  bool exit_idle = false;
   grpc_error *state_error = GRPC_ERROR_CREATE("No load balancing policy");
 
-  if (chand->incoming_configuration != NULL) {
-    lb_policy = grpc_client_config_get_lb_policy(chand->incoming_configuration);
+  if (chand->resolver_result != NULL) {
+    lb_policy = grpc_resolver_result_get_lb_policy(chand->resolver_result);
     if (lb_policy != NULL) {
       GRPC_LB_POLICY_REF(lb_policy, "channel");
       GRPC_LB_POLICY_REF(lb_policy, "config_change");
@@ -197,17 +182,17 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
           grpc_lb_policy_check_connectivity(exec_ctx, lb_policy, &state_error);
     }
 
-    grpc_client_config_unref(exec_ctx, chand->incoming_configuration);
+    grpc_resolver_result_unref(exec_ctx, chand->resolver_result);
   }
 
-  chand->incoming_configuration = NULL;
+  chand->resolver_result = NULL;
 
   if (lb_policy != NULL) {
     grpc_pollset_set_add_pollset_set(exec_ctx, lb_policy->interested_parties,
                                      chand->interested_parties);
   }
 
-  gpr_mu_lock(&chand->mu_config);
+  gpr_mu_lock(&chand->mu);
   old_lb_policy = chand->lb_policy;
   chand->lb_policy = lb_policy;
   if (lb_policy != NULL) {
@@ -222,8 +207,8 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
   }
   if (lb_policy != NULL && chand->exit_idle_when_lb_policy_arrives) {
     GRPC_LB_POLICY_REF(lb_policy, "exit_idle");
-    exit_idle = 1;
-    chand->exit_idle_when_lb_policy_arrives = 0;
+    exit_idle = true;
+    chand->exit_idle_when_lb_policy_arrives = false;
   }
 
   if (error == GRPC_ERROR_NONE && chand->resolver) {
@@ -233,10 +218,9 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
       watch_lb_policy(exec_ctx, chand, lb_policy, state);
     }
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-    grpc_resolver_next(exec_ctx, chand->resolver,
-                       &chand->incoming_configuration,
-                       &chand->on_config_changed);
-    gpr_mu_unlock(&chand->mu_config);
+    grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result,
+                       &chand->on_resolver_result_changed);
+    gpr_mu_unlock(&chand->mu);
   } else {
     if (chand->resolver != NULL) {
       grpc_resolver_shutdown(exec_ctx, chand->resolver);
@@ -249,7 +233,7 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
         GRPC_ERROR_CREATE_REFERENCING("Got config after disconnection", refs,
                                       GPR_ARRAY_SIZE(refs)),
         "resolver_gone");
-    gpr_mu_unlock(&chand->mu_config);
+    gpr_mu_unlock(&chand->mu);
   }
 
   if (exit_idle) {
@@ -284,7 +268,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                  op->bind_pollset);
   }
 
-  gpr_mu_lock(&chand->mu_config);
+  gpr_mu_lock(&chand->mu);
   if (op->on_connectivity_state_change != NULL) {
     grpc_connectivity_state_notify_on_state_change(
         exec_ctx, &chand->state_tracker, op->connectivity_state,
@@ -329,7 +313,189 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
     }
     GRPC_ERROR_UNREF(op->disconnect_with_error);
   }
-  gpr_mu_unlock(&chand->mu_config);
+  gpr_mu_unlock(&chand->mu);
+}
+
+/* Constructor for channel_data */
+static void cc_init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_element *elem,
+                                 grpc_channel_element_args *args) {
+  channel_data *chand = elem->channel_data;
+
+  memset(chand, 0, sizeof(*chand));
+
+  GPR_ASSERT(args->is_last);
+  GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
+
+  gpr_mu_init(&chand->mu);
+  grpc_closure_init(&chand->on_resolver_result_changed,
+                    on_resolver_result_changed, chand);
+  chand->owning_stack = args->channel_stack;
+
+  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
+                               "client_channel");
+  chand->interested_parties = grpc_pollset_set_create();
+}
+
+/* Destructor for channel_data */
+static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx,
+                                    grpc_channel_element *elem) {
+  channel_data *chand = elem->channel_data;
+
+  if (chand->resolver != NULL) {
+    grpc_resolver_shutdown(exec_ctx, chand->resolver);
+    GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
+  }
+  if (chand->lb_policy != NULL) {
+    grpc_pollset_set_del_pollset_set(exec_ctx,
+                                     chand->lb_policy->interested_parties,
+                                     chand->interested_parties);
+    GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
+  }
+  grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
+  grpc_pollset_set_destroy(chand->interested_parties);
+  gpr_mu_destroy(&chand->mu);
+}
+
+/*************************************************************************
+ * PER-CALL FUNCTIONS
+ */
+
+#define GET_CALL(call_data) \
+  ((grpc_subchannel_call *)(gpr_atm_acq_load(&(call_data)->subchannel_call)))
+
+#define CANCELLED_CALL ((grpc_subchannel_call *)1)
+
+typedef enum {
+  GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING,
+  GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL
+} subchannel_creation_phase;
+
+/** Call data.  Holds a pointer to grpc_subchannel_call and the
+    associated machinery to create such a pointer.
+    Handles queueing of stream ops until a call object is ready, waiting
+    for initial metadata before trying to create a call object,
+    and handling cancellation gracefully. */
+typedef struct client_channel_call_data {
+  /** either 0 for no call, 1 for cancelled, or a pointer to a
+      grpc_subchannel_call */
+  gpr_atm subchannel_call;
+
+  gpr_mu mu;
+
+  subchannel_creation_phase creation_phase;
+  grpc_connected_subchannel *connected_subchannel;
+  grpc_polling_entity *pollent;
+
+  grpc_transport_stream_op *waiting_ops;
+  size_t waiting_ops_count;
+  size_t waiting_ops_capacity;
+
+  grpc_closure next_step;
+
+  grpc_call_stack *owning_call;
+} call_data;
+
+static void add_waiting_locked(call_data *calld, grpc_transport_stream_op *op) {
+  GPR_TIMER_BEGIN("add_waiting_locked", 0);
+  if (calld->waiting_ops_count == calld->waiting_ops_capacity) {
+    calld->waiting_ops_capacity = GPR_MAX(3, 2 * calld->waiting_ops_capacity);
+    calld->waiting_ops =
+        gpr_realloc(calld->waiting_ops,
+                    calld->waiting_ops_capacity * sizeof(*calld->waiting_ops));
+  }
+  calld->waiting_ops[calld->waiting_ops_count++] = *op;
+  GPR_TIMER_END("add_waiting_locked", 0);
+}
+
+static void fail_locked(grpc_exec_ctx *exec_ctx, call_data *calld,
+                        grpc_error *error) {
+  size_t i;
+  for (i = 0; i < calld->waiting_ops_count; i++) {
+    grpc_transport_stream_op_finish_with_failure(
+        exec_ctx, &calld->waiting_ops[i], GRPC_ERROR_REF(error));
+  }
+  calld->waiting_ops_count = 0;
+  GRPC_ERROR_UNREF(error);
+}
+
+typedef struct {
+  grpc_transport_stream_op *ops;
+  size_t nops;
+  grpc_subchannel_call *call;
+} retry_ops_args;
+
+static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
+  retry_ops_args *a = args;
+  size_t i;
+  for (i = 0; i < a->nops; i++) {
+    grpc_subchannel_call_process_op(exec_ctx, a->call, &a->ops[i]);
+  }
+  GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops");
+  gpr_free(a->ops);
+  gpr_free(a);
+}
+
+static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) {
+  retry_ops_args *a = gpr_malloc(sizeof(*a));
+  a->ops = calld->waiting_ops;
+  a->nops = calld->waiting_ops_count;
+  a->call = GET_CALL(calld);
+  if (a->call == CANCELLED_CALL) {
+    gpr_free(a);
+    fail_locked(exec_ctx, calld, GRPC_ERROR_CANCELLED);
+    return;
+  }
+  calld->waiting_ops = NULL;
+  calld->waiting_ops_count = 0;
+  calld->waiting_ops_capacity = 0;
+  GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
+  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
+                      GRPC_ERROR_NONE, NULL);
+}
+
+static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
+                             grpc_error *error) {
+  call_data *calld = arg;
+  gpr_mu_lock(&calld->mu);
+  GPR_ASSERT(calld->creation_phase ==
+             GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL);
+  calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  if (calld->connected_subchannel == NULL) {
+    gpr_atm_no_barrier_store(&calld->subchannel_call, 1);
+    fail_locked(exec_ctx, calld, GRPC_ERROR_CREATE_REFERENCING(
+                                     "Failed to create subchannel", &error, 1));
+  } else if (1 == gpr_atm_acq_load(&calld->subchannel_call)) {
+    /* already cancelled before subchannel became ready */
+    fail_locked(exec_ctx, calld,
+                GRPC_ERROR_CREATE_REFERENCING(
+                    "Cancelled before creating subchannel", &error, 1));
+  } else {
+    grpc_subchannel_call *subchannel_call = NULL;
+    grpc_error *new_error = grpc_connected_subchannel_create_call(
+        exec_ctx, calld->connected_subchannel, calld->pollent,
+        &subchannel_call);
+    if (new_error != GRPC_ERROR_NONE) {
+      new_error = grpc_error_add_child(new_error, error);
+      subchannel_call = CANCELLED_CALL;
+      fail_locked(exec_ctx, calld, new_error);
+    }
+    gpr_atm_rel_store(&calld->subchannel_call,
+                      (gpr_atm)(uintptr_t)subchannel_call);
+    retry_waiting_locked(exec_ctx, calld);
+  }
+  gpr_mu_unlock(&calld->mu);
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
+}
+
+static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
+  grpc_subchannel_call *subchannel_call = GET_CALL(calld);
+  if (subchannel_call == NULL || subchannel_call == CANCELLED_CALL) {
+    return NULL;
+  } else {
+    return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
+  }
 }
 
 typedef struct {
@@ -341,11 +507,11 @@ typedef struct {
   grpc_closure closure;
 } continue_picking_args;
 
-static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
-                              grpc_metadata_batch *initial_metadata,
-                              uint32_t initial_metadata_flags,
-                              grpc_connected_subchannel **connected_subchannel,
-                              grpc_closure *on_ready);
+static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                            grpc_metadata_batch *initial_metadata,
+                            uint32_t initial_metadata_flags,
+                            grpc_connected_subchannel **connected_subchannel,
+                            grpc_closure *on_ready);
 
 static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
                              grpc_error *error) {
@@ -354,22 +520,21 @@ static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
     /* cancelled, do nothing */
   } else if (error != GRPC_ERROR_NONE) {
     grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error), NULL);
-  } else if (cc_pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
-                                cpa->initial_metadata_flags,
-                                cpa->connected_subchannel, cpa->on_ready)) {
+  } else if (pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
+                             cpa->initial_metadata_flags,
+                             cpa->connected_subchannel, cpa->on_ready)) {
     grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE, NULL);
   }
   gpr_free(cpa);
 }
 
-static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
-                              grpc_metadata_batch *initial_metadata,
-                              uint32_t initial_metadata_flags,
-                              grpc_connected_subchannel **connected_subchannel,
-                              grpc_closure *on_ready) {
-  GPR_TIMER_BEGIN("cc_pick_subchannel", 0);
+static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                            grpc_metadata_batch *initial_metadata,
+                            uint32_t initial_metadata_flags,
+                            grpc_connected_subchannel **connected_subchannel,
+                            grpc_closure *on_ready) {
+  GPR_TIMER_BEGIN("pick_subchannel", 0);
 
-  grpc_call_element *elem = elemp;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   continue_picking_args *cpa;
@@ -377,7 +542,7 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
 
   GPR_ASSERT(connected_subchannel);
 
-  gpr_mu_lock(&chand->mu_config);
+  gpr_mu_lock(&chand->mu);
   if (initial_metadata == NULL) {
     if (chand->lb_policy != NULL) {
       grpc_lb_policy_cancel_pick(exec_ctx, chand->lb_policy,
@@ -392,28 +557,27 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
                             GRPC_ERROR_CREATE("Pick cancelled"), NULL);
       }
     }
-    gpr_mu_unlock(&chand->mu_config);
-    GPR_TIMER_END("cc_pick_subchannel", 0);
-    return 1;
+    gpr_mu_unlock(&chand->mu);
+    GPR_TIMER_END("pick_subchannel", 0);
+    return true;
   }
   if (chand->lb_policy != NULL) {
     grpc_lb_policy *lb_policy = chand->lb_policy;
     int r;
-    GRPC_LB_POLICY_REF(lb_policy, "cc_pick_subchannel");
-    gpr_mu_unlock(&chand->mu_config);
+    GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel");
+    gpr_mu_unlock(&chand->mu);
     r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollent,
                             initial_metadata, initial_metadata_flags,
                             connected_subchannel, on_ready);
-    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "cc_pick_subchannel");
-    GPR_TIMER_END("cc_pick_subchannel", 0);
+    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick_subchannel");
+    GPR_TIMER_END("pick_subchannel", 0);
     return r;
   }
   if (chand->resolver != NULL && !chand->started_resolving) {
-    chand->started_resolving = 1;
+    chand->started_resolving = true;
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-    grpc_resolver_next(exec_ctx, chand->resolver,
-                       &chand->incoming_configuration,
-                       &chand->on_config_changed);
+    grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result,
+                       &chand->on_resolver_result_changed);
   }
   if (chand->resolver != NULL) {
     cpa = gpr_malloc(sizeof(*cpa));
@@ -429,67 +593,145 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
     grpc_exec_ctx_sched(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected"),
                         NULL);
   }
-  gpr_mu_unlock(&chand->mu_config);
+  gpr_mu_unlock(&chand->mu);
 
-  GPR_TIMER_END("cc_pick_subchannel", 0);
-  return 0;
+  GPR_TIMER_END("pick_subchannel", 0);
+  return false;
+}
+
+// The logic here is fairly complicated, due to (a) the fact that we
+// need to handle the case where we receive the send op before the
+// initial metadata op, and (b) the need for efficiency, especially in
+// the streaming case.
+// TODO(ctiller): Explain this more thoroughly.
+static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
+                                         grpc_call_element *elem,
+                                         grpc_transport_stream_op *op) {
+  call_data *calld = elem->call_data;
+  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
+  /* try to (atomically) get the call */
+  grpc_subchannel_call *call = GET_CALL(calld);
+  GPR_TIMER_BEGIN("cc_start_transport_stream_op", 0);
+  if (call == CANCELLED_CALL) {
+    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                 GRPC_ERROR_CANCELLED);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  if (call != NULL) {
+    grpc_subchannel_call_process_op(exec_ctx, call, op);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  /* we failed; lock and figure out what to do */
+  gpr_mu_lock(&calld->mu);
+retry:
+  /* need to recheck that another thread hasn't set the call */
+  call = GET_CALL(calld);
+  if (call == CANCELLED_CALL) {
+    gpr_mu_unlock(&calld->mu);
+    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                 GRPC_ERROR_CANCELLED);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  if (call != NULL) {
+    gpr_mu_unlock(&calld->mu);
+    grpc_subchannel_call_process_op(exec_ctx, call, op);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  /* if this is a cancellation, then we can raise our cancelled flag */
+  if (op->cancel_error != GRPC_ERROR_NONE) {
+    if (!gpr_atm_rel_cas(&calld->subchannel_call, 0,
+                         (gpr_atm)(uintptr_t)CANCELLED_CALL)) {
+      goto retry;
+    } else {
+      switch (calld->creation_phase) {
+        case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING:
+          fail_locked(exec_ctx, calld, GRPC_ERROR_REF(op->cancel_error));
+          break;
+        case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL:
+          pick_subchannel(exec_ctx, elem, NULL, 0, &calld->connected_subchannel,
+                          NULL);
+          break;
+      }
+      gpr_mu_unlock(&calld->mu);
+      grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                   GRPC_ERROR_CANCELLED);
+      GPR_TIMER_END("cc_start_transport_stream_op", 0);
+      return;
+    }
+  }
+  /* if we don't have a subchannel, try to get one */
+  if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
+      calld->connected_subchannel == NULL &&
+      op->send_initial_metadata != NULL) {
+    calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL;
+    grpc_closure_init(&calld->next_step, subchannel_ready, calld);
+    GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel");
+    if (pick_subchannel(exec_ctx, elem, op->send_initial_metadata,
+                        op->send_initial_metadata_flags,
+                        &calld->connected_subchannel, &calld->next_step)) {
+      calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+      GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
+    }
+  }
+  /* if we've got a subchannel, then let's ask it to create a call */
+  if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
+      calld->connected_subchannel != NULL) {
+    grpc_subchannel_call *subchannel_call = NULL;
+    grpc_error *error = grpc_connected_subchannel_create_call(
+        exec_ctx, calld->connected_subchannel, calld->pollent,
+        &subchannel_call);
+    if (error != GRPC_ERROR_NONE) {
+      subchannel_call = CANCELLED_CALL;
+      fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error));
+      grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error);
+    }
+    gpr_atm_rel_store(&calld->subchannel_call,
+                      (gpr_atm)(uintptr_t)subchannel_call);
+    retry_waiting_locked(exec_ctx, calld);
+    goto retry;
+  }
+  /* nothing to be done but wait */
+  add_waiting_locked(calld, op);
+  gpr_mu_unlock(&calld->mu);
+  GPR_TIMER_END("cc_start_transport_stream_op", 0);
 }
 
 /* Constructor for call_data */
-static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem,
-                                  grpc_call_element_args *args) {
-  grpc_subchannel_call_holder_init(elem->call_data, cc_pick_subchannel, elem,
-                                   args->call_stack);
+static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_element *elem,
+                                     grpc_call_element_args *args) {
+  call_data *calld = elem->call_data;
+  gpr_atm_rel_store(&calld->subchannel_call, 0);
+  gpr_mu_init(&calld->mu);
+  calld->connected_subchannel = NULL;
+  calld->waiting_ops = NULL;
+  calld->waiting_ops_count = 0;
+  calld->waiting_ops_capacity = 0;
+  calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  calld->owning_call = args->call_stack;
+  calld->pollent = NULL;
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for call_data */
-static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              const grpc_call_final_info *final_info,
-                              void *and_free_memory) {
-  grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data);
-  gpr_free(and_free_memory);
-}
-
-/* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
-  channel_data *chand = elem->channel_data;
-
-  memset(chand, 0, sizeof(*chand));
-
-  GPR_ASSERT(args->is_last);
-  GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
-
-  gpr_mu_init(&chand->mu_config);
-  grpc_closure_init(&chand->on_config_changed, cc_on_config_changed, chand);
-  chand->owning_stack = args->channel_stack;
-
-  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
-                               "client_channel");
-  chand->interested_parties = grpc_pollset_set_create();
-}
-
-/* Destructor for channel_data */
-static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
-                                 grpc_channel_element *elem) {
-  channel_data *chand = elem->channel_data;
-
-  if (chand->resolver != NULL) {
-    grpc_resolver_shutdown(exec_ctx, chand->resolver);
-    GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
-  }
-  if (chand->lb_policy != NULL) {
-    grpc_pollset_set_del_pollset_set(exec_ctx,
-                                     chand->lb_policy->interested_parties,
-                                     chand->interested_parties);
-    GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
+static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_call_element *elem,
+                                 const grpc_call_final_info *final_info,
+                                 void *and_free_memory) {
+  call_data *calld = elem->call_data;
+  grpc_subchannel_call *call = GET_CALL(calld);
+  if (call != NULL && call != CANCELLED_CALL) {
+    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "client_channel_destroy_call");
   }
-  grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
-  grpc_pollset_set_destroy(chand->interested_parties);
-  gpr_mu_destroy(&chand->mu_config);
+  GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING);
+  gpr_mu_destroy(&calld->mu);
+  GPR_ASSERT(calld->waiting_ops_count == 0);
+  gpr_free(calld->waiting_ops);
+  gpr_free(and_free_memory);
 }
 
 static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
@@ -499,16 +741,20 @@ static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
   calld->pollent = pollent;
 }
 
+/*************************************************************************
+ * EXPORTED SYMBOLS
+ */
+
 const grpc_channel_filter grpc_client_channel_filter = {
     cc_start_transport_stream_op,
     cc_start_transport_op,
     sizeof(call_data),
-    init_call_elem,
+    cc_init_call_elem,
     cc_set_pollset_or_pollset_set,
-    destroy_call_elem,
+    cc_destroy_call_elem,
     sizeof(channel_data),
-    init_channel_elem,
-    destroy_channel_elem,
+    cc_init_channel_elem,
+    cc_destroy_channel_elem,
     cc_get_peer,
     "client-channel",
 };
@@ -519,41 +765,40 @@ void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx,
   /* post construction initialization: set the transport setup pointer */
   grpc_channel_element *elem = grpc_channel_stack_last_element(channel_stack);
   channel_data *chand = elem->channel_data;
-  gpr_mu_lock(&chand->mu_config);
+  gpr_mu_lock(&chand->mu);
   GPR_ASSERT(!chand->resolver);
   chand->resolver = resolver;
   GRPC_RESOLVER_REF(resolver, "channel");
   if (!grpc_closure_list_empty(chand->waiting_for_config_closures) ||
       chand->exit_idle_when_lb_policy_arrives) {
-    chand->started_resolving = 1;
+    chand->started_resolving = true;
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-    grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration,
-                       &chand->on_config_changed);
+    grpc_resolver_next(exec_ctx, resolver, &chand->resolver_result,
+                       &chand->on_resolver_result_changed);
   }
-  gpr_mu_unlock(&chand->mu_config);
+  gpr_mu_unlock(&chand->mu);
 }
 
 grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect) {
   channel_data *chand = elem->channel_data;
   grpc_connectivity_state out;
-  gpr_mu_lock(&chand->mu_config);
+  gpr_mu_lock(&chand->mu);
   out = grpc_connectivity_state_check(&chand->state_tracker, NULL);
   if (out == GRPC_CHANNEL_IDLE && try_to_connect) {
     if (chand->lb_policy != NULL) {
       grpc_lb_policy_exit_idle(exec_ctx, chand->lb_policy);
     } else {
-      chand->exit_idle_when_lb_policy_arrives = 1;
+      chand->exit_idle_when_lb_policy_arrives = true;
       if (!chand->started_resolving && chand->resolver != NULL) {
         GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-        chand->started_resolving = 1;
-        grpc_resolver_next(exec_ctx, chand->resolver,
-                           &chand->incoming_configuration,
-                           &chand->on_config_changed);
+        chand->started_resolving = true;
+        grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result,
+                           &chand->on_resolver_result_changed);
       }
     }
   }
-  gpr_mu_unlock(&chand->mu_config);
+  gpr_mu_unlock(&chand->mu);
   return out;
 }
 
@@ -588,8 +833,8 @@ void grpc_client_channel_watch_connectivity_state(
   grpc_closure_init(&w->my_closure, on_external_watch_complete, w);
   GRPC_CHANNEL_STACK_REF(w->chand->owning_stack,
                          "external_connectivity_watcher");
-  gpr_mu_lock(&chand->mu_config);
+  gpr_mu_lock(&chand->mu);
   grpc_connectivity_state_notify_on_state_change(
       exec_ctx, &chand->state_tracker, state, &w->my_closure);
-  gpr_mu_unlock(&chand->mu_config);
+  gpr_mu_unlock(&chand->mu);
 }
diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h
index 1c89b28b59..da1de3579a 100644
--- a/src/core/ext/client_config/lb_policy_factory.h
+++ b/src/core/ext/client_config/lb_policy_factory.h
@@ -43,8 +43,6 @@
 typedef struct grpc_lb_policy_factory grpc_lb_policy_factory;
 typedef struct grpc_lb_policy_factory_vtable grpc_lb_policy_factory_vtable;
 
-/** grpc_lb_policy provides grpc_client_config objects to grpc_channel
-    objects */
 struct grpc_lb_policy_factory {
   const grpc_lb_policy_factory_vtable *vtable;
 };
diff --git a/src/core/ext/client_config/resolver.c b/src/core/ext/client_config/resolver.c
index eb004455bd..7534ea62af 100644
--- a/src/core/ext/client_config/resolver.c
+++ b/src/core/ext/client_config/resolver.c
@@ -76,7 +76,7 @@ void grpc_resolver_channel_saw_error(grpc_exec_ctx *exec_ctx,
 }
 
 void grpc_resolver_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-                        grpc_client_config **target_config,
+                        grpc_resolver_result **result,
                         grpc_closure *on_complete) {
-  resolver->vtable->next(exec_ctx, resolver, target_config, on_complete);
+  resolver->vtable->next(exec_ctx, resolver, result, on_complete);
 }
diff --git a/src/core/ext/client_config/resolver.h b/src/core/ext/client_config/resolver.h
index 6ecb5d2774..88ac262d51 100644
--- a/src/core/ext/client_config/resolver.h
+++ b/src/core/ext/client_config/resolver.h
@@ -34,14 +34,14 @@
 #ifndef GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_H
 #define GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_H
 
-#include "src/core/ext/client_config/client_config.h"
+#include "src/core/ext/client_config/resolver_result.h"
 #include "src/core/ext/client_config/subchannel.h"
 #include "src/core/lib/iomgr/iomgr.h"
 
 typedef struct grpc_resolver grpc_resolver;
 typedef struct grpc_resolver_vtable grpc_resolver_vtable;
 
-/** grpc_resolver provides grpc_client_config objects to grpc_channel
+/** grpc_resolver provides grpc_resolver_result objects to grpc_channel
     objects */
 struct grpc_resolver {
   const grpc_resolver_vtable *vtable;
@@ -53,7 +53,7 @@ struct grpc_resolver_vtable {
   void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver);
   void (*channel_saw_error)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver);
   void (*next)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-               grpc_client_config **target_config, grpc_closure *on_complete);
+               grpc_resolver_result **result, grpc_closure *on_complete);
 };
 
 #ifdef GRPC_RESOLVER_REFCOUNT_DEBUG
@@ -82,13 +82,13 @@ void grpc_resolver_channel_saw_error(grpc_exec_ctx *exec_ctx,
                                      grpc_resolver *resolver);
 
 /** Get the next client config. Called by the channel to fetch a new
-    configuration. Expected to set *target_config with a new configuration,
+    configuration. Expected to set *result with a new configuration,
     and then schedule on_complete for execution.
 
-    If resolution is fatally broken, set *target_config to NULL and
+    If resolution is fatally broken, set *result to NULL and
     schedule on_complete. */
 void grpc_resolver_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-                        grpc_client_config **target_config,
+                        grpc_resolver_result **result,
                         grpc_closure *on_complete);
 
 #endif /* GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_H */
diff --git a/src/core/ext/client_config/resolver_factory.h b/src/core/ext/client_config/resolver_factory.h
index 4eb6979aad..f69bf79564 100644
--- a/src/core/ext/client_config/resolver_factory.h
+++ b/src/core/ext/client_config/resolver_factory.h
@@ -41,7 +41,7 @@
 typedef struct grpc_resolver_factory grpc_resolver_factory;
 typedef struct grpc_resolver_factory_vtable grpc_resolver_factory_vtable;
 
-/** grpc_resolver provides grpc_client_config objects to grpc_channel
+/** grpc_resolver provides grpc_resolver_result objects to grpc_channel
     objects */
 struct grpc_resolver_factory {
   const grpc_resolver_factory_vtable *vtable;
diff --git a/src/core/ext/client_config/client_config.c b/src/core/ext/client_config/resolver_result.c
index f9b8e68698..c6c4166e83 100644
--- a/src/core/ext/client_config/client_config.c
+++ b/src/core/ext/client_config/resolver_result.c
@@ -31,44 +31,45 @@
  *
  */
 
-#include "src/core/ext/client_config/client_config.h"
+#include "src/core/ext/client_config/resolver_result.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
 
-struct grpc_client_config {
+struct grpc_resolver_result {
   gpr_refcount refs;
   grpc_lb_policy *lb_policy;
 };
 
-grpc_client_config *grpc_client_config_create() {
-  grpc_client_config *c = gpr_malloc(sizeof(*c));
+grpc_resolver_result *grpc_resolver_result_create() {
+  grpc_resolver_result *c = gpr_malloc(sizeof(*c));
   memset(c, 0, sizeof(*c));
   gpr_ref_init(&c->refs, 1);
   return c;
 }
 
-void grpc_client_config_ref(grpc_client_config *c) { gpr_ref(&c->refs); }
+void grpc_resolver_result_ref(grpc_resolver_result *c) { gpr_ref(&c->refs); }
 
-void grpc_client_config_unref(grpc_exec_ctx *exec_ctx, grpc_client_config *c) {
+void grpc_resolver_result_unref(grpc_exec_ctx *exec_ctx,
+                                grpc_resolver_result *c) {
   if (gpr_unref(&c->refs)) {
     if (c->lb_policy != NULL) {
-      GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config");
+      GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "resolver_result");
     }
     gpr_free(c);
   }
 }
 
-void grpc_client_config_set_lb_policy(grpc_client_config *c,
-                                      grpc_lb_policy *lb_policy) {
+void grpc_resolver_result_set_lb_policy(grpc_resolver_result *c,
+                                        grpc_lb_policy *lb_policy) {
   GPR_ASSERT(c->lb_policy == NULL);
   if (lb_policy) {
-    GRPC_LB_POLICY_REF(lb_policy, "client_config");
+    GRPC_LB_POLICY_REF(lb_policy, "resolver_result");
   }
   c->lb_policy = lb_policy;
 }
 
-grpc_lb_policy *grpc_client_config_get_lb_policy(grpc_client_config *c) {
+grpc_lb_policy *grpc_resolver_result_get_lb_policy(grpc_resolver_result *c) {
   return c->lb_policy;
 }
diff --git a/src/core/ext/client_config/client_config.h b/src/core/ext/client_config/resolver_result.h
index a6290cbcf0..402f7dbd7e 100644
--- a/src/core/ext/client_config/client_config.h
+++ b/src/core/ext/client_config/resolver_result.h
@@ -31,23 +31,22 @@
  *
  */
 
-#ifndef GRPC_CORE_EXT_CLIENT_CONFIG_CLIENT_CONFIG_H
-#define GRPC_CORE_EXT_CLIENT_CONFIG_CLIENT_CONFIG_H
+#ifndef GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_RESULT_H
+#define GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_RESULT_H
 
 #include "src/core/ext/client_config/lb_policy.h"
 
-/** Total configuration for a client. Provided, and updated, by
-    grpc_resolver */
-typedef struct grpc_client_config grpc_client_config;
+/** Results reported from a grpc_resolver. */
+typedef struct grpc_resolver_result grpc_resolver_result;
 
-grpc_client_config *grpc_client_config_create();
-void grpc_client_config_ref(grpc_client_config *client_config);
-void grpc_client_config_unref(grpc_exec_ctx *exec_ctx,
-                              grpc_client_config *client_config);
+grpc_resolver_result *grpc_resolver_result_create();
+void grpc_resolver_result_ref(grpc_resolver_result *client_config);
+void grpc_resolver_result_unref(grpc_exec_ctx *exec_ctx,
+                                grpc_resolver_result *client_config);
 
-void grpc_client_config_set_lb_policy(grpc_client_config *client_config,
-                                      grpc_lb_policy *lb_policy);
-grpc_lb_policy *grpc_client_config_get_lb_policy(
-    grpc_client_config *client_config);
+void grpc_resolver_result_set_lb_policy(grpc_resolver_result *client_config,
+                                        grpc_lb_policy *lb_policy);
+grpc_lb_policy *grpc_resolver_result_get_lb_policy(
+    grpc_resolver_result *client_config);
 
-#endif /* GRPC_CORE_EXT_CLIENT_CONFIG_CLIENT_CONFIG_H */
+#endif /* GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_RESULT_H */
diff --git a/src/core/ext/client_config/subchannel_call_holder.c b/src/core/ext/client_config/subchannel_call_holder.c
deleted file mode 100644
index be6d054af4..0000000000
--- a/src/core/ext/client_config/subchannel_call_holder.c
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/ext/client_config/subchannel_call_holder.h"
-
-#include <grpc/support/alloc.h>
-
-#include "src/core/lib/profiling/timers.h"
-
-#define GET_CALL(holder) \
-  ((grpc_subchannel_call *)(gpr_atm_acq_load(&(holder)->subchannel_call)))
-
-#define CANCELLED_CALL ((grpc_subchannel_call *)1)
-
-static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *holder,
-                             grpc_error *error);
-static void retry_ops(grpc_exec_ctx *exec_ctx, void *retry_ops_args,
-                      grpc_error *error);
-
-static void add_waiting_locked(grpc_subchannel_call_holder *holder,
-                               grpc_transport_stream_op *op);
-static void fail_locked(grpc_exec_ctx *exec_ctx,
-                        grpc_subchannel_call_holder *holder, grpc_error *error);
-static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
-                                 grpc_subchannel_call_holder *holder);
-
-void grpc_subchannel_call_holder_init(
-    grpc_subchannel_call_holder *holder,
-    grpc_subchannel_call_holder_pick_subchannel pick_subchannel,
-    void *pick_subchannel_arg, grpc_call_stack *owning_call) {
-  gpr_atm_rel_store(&holder->subchannel_call, 0);
-  holder->pick_subchannel = pick_subchannel;
-  holder->pick_subchannel_arg = pick_subchannel_arg;
-  gpr_mu_init(&holder->mu);
-  holder->connected_subchannel = NULL;
-  holder->waiting_ops = NULL;
-  holder->waiting_ops_count = 0;
-  holder->waiting_ops_capacity = 0;
-  holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-  holder->owning_call = owning_call;
-  holder->pollent = NULL;
-}
-
-void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
-                                         grpc_subchannel_call_holder *holder) {
-  grpc_subchannel_call *call = GET_CALL(holder);
-  if (call != NULL && call != CANCELLED_CALL) {
-    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "holder");
-  }
-  GPR_ASSERT(holder->creation_phase ==
-             GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING);
-  gpr_mu_destroy(&holder->mu);
-  GPR_ASSERT(holder->waiting_ops_count == 0);
-  gpr_free(holder->waiting_ops);
-}
-
-// The logic here is fairly complicated, due to (a) the fact that we
-// need to handle the case where we receive the send op before the
-// initial metadata op, and (b) the need for efficiency, especially in
-// the streaming case.
-// TODO(ctiller): Explain this more thoroughly.
-void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
-                                            grpc_subchannel_call_holder *holder,
-                                            grpc_transport_stream_op *op) {
-  /* try to (atomically) get the call */
-  grpc_subchannel_call *call = GET_CALL(holder);
-  GPR_TIMER_BEGIN("grpc_subchannel_call_holder_perform_op", 0);
-  if (call == CANCELLED_CALL) {
-    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
-                                                 GRPC_ERROR_CANCELLED);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  if (call != NULL) {
-    grpc_subchannel_call_process_op(exec_ctx, call, op);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  /* we failed; lock and figure out what to do */
-  gpr_mu_lock(&holder->mu);
-retry:
-  /* need to recheck that another thread hasn't set the call */
-  call = GET_CALL(holder);
-  if (call == CANCELLED_CALL) {
-    gpr_mu_unlock(&holder->mu);
-    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
-                                                 GRPC_ERROR_CANCELLED);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  if (call != NULL) {
-    gpr_mu_unlock(&holder->mu);
-    grpc_subchannel_call_process_op(exec_ctx, call, op);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  /* if this is a cancellation, then we can raise our cancelled flag */
-  if (op->cancel_error != GRPC_ERROR_NONE) {
-    if (!gpr_atm_rel_cas(&holder->subchannel_call, 0,
-                         (gpr_atm)(uintptr_t)CANCELLED_CALL)) {
-      goto retry;
-    } else {
-      switch (holder->creation_phase) {
-        case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING:
-          fail_locked(exec_ctx, holder, GRPC_ERROR_REF(op->cancel_error));
-          break;
-        case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL:
-          holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, NULL,
-                                  0, &holder->connected_subchannel, NULL);
-          break;
-      }
-      gpr_mu_unlock(&holder->mu);
-      grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
-                                                   GRPC_ERROR_CANCELLED);
-      GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-      return;
-    }
-  }
-  /* if we don't have a subchannel, try to get one */
-  if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
-      holder->connected_subchannel == NULL &&
-      op->send_initial_metadata != NULL) {
-    holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL;
-    grpc_closure_init(&holder->next_step, subchannel_ready, holder);
-    GRPC_CALL_STACK_REF(holder->owning_call, "pick_subchannel");
-    if (holder->pick_subchannel(
-            exec_ctx, holder->pick_subchannel_arg, op->send_initial_metadata,
-            op->send_initial_metadata_flags, &holder->connected_subchannel,
-            &holder->next_step)) {
-      holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-      GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
-    }
-  }
-  /* if we've got a subchannel, then let's ask it to create a call */
-  if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
-      holder->connected_subchannel != NULL) {
-    grpc_subchannel_call *subchannel_call = NULL;
-    grpc_error *error = grpc_connected_subchannel_create_call(
-        exec_ctx, holder->connected_subchannel, holder->pollent,
-        &subchannel_call);
-    if (error != GRPC_ERROR_NONE) {
-      subchannel_call = CANCELLED_CALL;
-      fail_locked(exec_ctx, holder, GRPC_ERROR_REF(error));
-      grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error);
-    }
-    gpr_atm_rel_store(&holder->subchannel_call,
-                      (gpr_atm)(uintptr_t)subchannel_call);
-    retry_waiting_locked(exec_ctx, holder);
-    goto retry;
-  }
-  /* nothing to be done but wait */
-  add_waiting_locked(holder, op);
-  gpr_mu_unlock(&holder->mu);
-  GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-}
-
-static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
-                             grpc_error *error) {
-  grpc_subchannel_call_holder *holder = arg;
-  gpr_mu_lock(&holder->mu);
-  GPR_ASSERT(holder->creation_phase ==
-             GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL);
-  holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-  if (holder->connected_subchannel == NULL) {
-    gpr_atm_no_barrier_store(&holder->subchannel_call, 1);
-    fail_locked(exec_ctx, holder,
-                GRPC_ERROR_CREATE_REFERENCING("Failed to create subchannel",
-                                              &error, 1));
-  } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) {
-    /* already cancelled before subchannel became ready */
-    fail_locked(exec_ctx, holder,
-                GRPC_ERROR_CREATE_REFERENCING(
-                    "Cancelled before creating subchannel", &error, 1));
-  } else {
-    grpc_subchannel_call *subchannel_call = NULL;
-    grpc_error *new_error = grpc_connected_subchannel_create_call(
-        exec_ctx, holder->connected_subchannel, holder->pollent,
-        &subchannel_call);
-    if (new_error != GRPC_ERROR_NONE) {
-      new_error = grpc_error_add_child(new_error, error);
-      subchannel_call = CANCELLED_CALL;
-      fail_locked(exec_ctx, holder, new_error);
-    }
-    gpr_atm_rel_store(&holder->subchannel_call,
-                      (gpr_atm)(uintptr_t)subchannel_call);
-    retry_waiting_locked(exec_ctx, holder);
-  }
-  gpr_mu_unlock(&holder->mu);
-  GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
-}
-
-typedef struct {
-  grpc_transport_stream_op *ops;
-  size_t nops;
-  grpc_subchannel_call *call;
-} retry_ops_args;
-
-static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
-                                 grpc_subchannel_call_holder *holder) {
-  retry_ops_args *a = gpr_malloc(sizeof(*a));
-  a->ops = holder->waiting_ops;
-  a->nops = holder->waiting_ops_count;
-  a->call = GET_CALL(holder);
-  if (a->call == CANCELLED_CALL) {
-    gpr_free(a);
-    fail_locked(exec_ctx, holder, GRPC_ERROR_CANCELLED);
-    return;
-  }
-  holder->waiting_ops = NULL;
-  holder->waiting_ops_count = 0;
-  holder->waiting_ops_capacity = 0;
-  GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
-  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
-                      GRPC_ERROR_NONE, NULL);
-}
-
-static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
-  retry_ops_args *a = args;
-  size_t i;
-  for (i = 0; i < a->nops; i++) {
-    grpc_subchannel_call_process_op(exec_ctx, a->call, &a->ops[i]);
-  }
-  GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops");
-  gpr_free(a->ops);
-  gpr_free(a);
-}
-
-static void add_waiting_locked(grpc_subchannel_call_holder *holder,
-                               grpc_transport_stream_op *op) {
-  GPR_TIMER_BEGIN("add_waiting_locked", 0);
-  if (holder->waiting_ops_count == holder->waiting_ops_capacity) {
-    holder->waiting_ops_capacity = GPR_MAX(3, 2 * holder->waiting_ops_capacity);
-    holder->waiting_ops =
-        gpr_realloc(holder->waiting_ops, holder->waiting_ops_capacity *
-                                             sizeof(*holder->waiting_ops));
-  }
-  holder->waiting_ops[holder->waiting_ops_count++] = *op;
-  GPR_TIMER_END("add_waiting_locked", 0);
-}
-
-static void fail_locked(grpc_exec_ctx *exec_ctx,
-                        grpc_subchannel_call_holder *holder,
-                        grpc_error *error) {
-  size_t i;
-  for (i = 0; i < holder->waiting_ops_count; i++) {
-    grpc_transport_stream_op_finish_with_failure(
-        exec_ctx, &holder->waiting_ops[i], GRPC_ERROR_REF(error));
-  }
-  holder->waiting_ops_count = 0;
-  GRPC_ERROR_UNREF(error);
-}
-
-char *grpc_subchannel_call_holder_get_peer(
-    grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder) {
-  grpc_subchannel_call *subchannel_call = GET_CALL(holder);
-
-  if (subchannel_call == NULL || subchannel_call == CANCELLED_CALL) {
-    return NULL;
-  } else {
-    return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
-  }
-}
diff --git a/src/core/ext/client_config/subchannel_call_holder.h b/src/core/ext/client_config/subchannel_call_holder.h
deleted file mode 100644
index 8d2deb02f3..0000000000
--- a/src/core/ext/client_config/subchannel_call_holder.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H
-#define GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H
-
-#include "src/core/ext/client_config/subchannel.h"
-#include "src/core/lib/iomgr/polling_entity.h"
-
-/** Pick a subchannel for grpc_subchannel_call_holder;
-    Return 1 if subchannel is available immediately (in which case on_ready
-    should not be called), or 0 otherwise (in which case on_ready should be
-    called when the subchannel is available) */
-typedef int (*grpc_subchannel_call_holder_pick_subchannel)(
-    grpc_exec_ctx *exec_ctx, void *arg, grpc_metadata_batch *initial_metadata,
-    uint32_t initial_metadata_flags,
-    grpc_connected_subchannel **connected_subchannel, grpc_closure *on_ready);
-
-typedef enum {
-  GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING,
-  GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL
-} grpc_subchannel_call_holder_creation_phase;
-
-/** Wrapper for holding a pointer to grpc_subchannel_call, and the
-    associated machinery to create such a pointer.
-    Handles queueing of stream ops until a call object is ready, waiting
-    for initial metadata before trying to create a call object,
-    and handling cancellation gracefully.
-
-    The channel filter uses this as their call_data. */
-typedef struct grpc_subchannel_call_holder {
-  /** either 0 for no call, 1 for cancelled, or a pointer to a
-      grpc_subchannel_call */
-  gpr_atm subchannel_call;
-  /** Helper function to choose the subchannel on which to create
-      the call object. Channel filter delegates to the load
-      balancing policy (once it's ready). */
-  grpc_subchannel_call_holder_pick_subchannel pick_subchannel;
-  void *pick_subchannel_arg;
-
-  gpr_mu mu;
-
-  grpc_subchannel_call_holder_creation_phase creation_phase;
-  grpc_connected_subchannel *connected_subchannel;
-  grpc_polling_entity *pollent;
-
-  grpc_transport_stream_op *waiting_ops;
-  size_t waiting_ops_count;
-  size_t waiting_ops_capacity;
-
-  grpc_closure next_step;
-
-  grpc_call_stack *owning_call;
-} grpc_subchannel_call_holder;
-
-void grpc_subchannel_call_holder_init(
-    grpc_subchannel_call_holder *holder,
-    grpc_subchannel_call_holder_pick_subchannel pick_subchannel,
-    void *pick_subchannel_arg, grpc_call_stack *owning_call);
-void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
-                                         grpc_subchannel_call_holder *holder);
-
-void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
-                                            grpc_subchannel_call_holder *holder,
-                                            grpc_transport_stream_op *op);
-char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx,
-                                           grpc_subchannel_call_holder *holder);
-
-#endif /* GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H */
diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c
index 31ac968670..79682e78b5 100644
--- a/src/core/ext/resolver/dns/native/dns_resolver.c
+++ b/src/core/ext/resolver/dns/native/dns_resolver.c
@@ -67,16 +67,16 @@ typedef struct {
   gpr_mu mu;
   /** are we currently resolving? */
   int resolving;
-  /** which version of resolved_config have we published? */
+  /** which version of the result have we published? */
   int published_version;
-  /** which version of resolved_config is current? */
+  /** which version of the result is current? */
   int resolved_version;
   /** pending next completion, or NULL */
   grpc_closure *next_completion;
-  /** target config address for next completion */
-  grpc_client_config **target_config;
-  /** current (fully resolved) config */
-  grpc_client_config *resolved_config;
+  /** target result address for next completion */
+  grpc_resolver_result **target_result;
+  /** current (fully resolved) result */
+  grpc_resolver_result *resolved_result;
   /** retry timer */
   bool have_retry_timer;
   grpc_timer retry_timer;
@@ -97,7 +97,7 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
 static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
 static void dns_channel_saw_error(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
 static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
-                     grpc_client_config **target_config,
+                     grpc_resolver_result **target_result,
                      grpc_closure *on_complete);
 
 static const grpc_resolver_vtable dns_resolver_vtable = {
@@ -110,7 +110,7 @@ static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
     grpc_timer_cancel(exec_ctx, &r->retry_timer);
   }
   if (r->next_completion != NULL) {
-    *r->target_config = NULL;
+    *r->target_result = NULL;
     grpc_exec_ctx_sched(exec_ctx, r->next_completion,
                         GRPC_ERROR_CREATE("Resolver Shutdown"), NULL);
     r->next_completion = NULL;
@@ -130,13 +130,13 @@ static void dns_channel_saw_error(grpc_exec_ctx *exec_ctx,
 }
 
 static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-                     grpc_client_config **target_config,
+                     grpc_resolver_result **target_result,
                      grpc_closure *on_complete) {
   dns_resolver *r = (dns_resolver *)resolver;
   gpr_mu_lock(&r->mu);
   GPR_ASSERT(!r->next_completion);
   r->next_completion = on_complete;
-  r->target_config = target_config;
+  r->target_result = target_result;
   if (r->resolved_version == 0 && !r->resolving) {
     gpr_backoff_reset(&r->backoff_state);
     dns_start_resolving_locked(exec_ctx, r);
@@ -165,7 +165,7 @@ static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg,
 static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
                             grpc_error *error) {
   dns_resolver *r = arg;
-  grpc_client_config *config = NULL;
+  grpc_resolver_result *result = NULL;
   grpc_lb_policy *lb_policy;
   gpr_mu_lock(&r->mu);
   GPR_ASSERT(r->resolving);
@@ -173,14 +173,14 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_resolved_addresses *addresses = r->addresses;
   if (addresses != NULL) {
     grpc_lb_policy_args lb_policy_args;
-    config = grpc_client_config_create();
+    result = grpc_resolver_result_create();
     memset(&lb_policy_args, 0, sizeof(lb_policy_args));
     lb_policy_args.addresses = addresses;
     lb_policy_args.client_channel_factory = r->client_channel_factory;
     lb_policy =
         grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args);
     if (lb_policy != NULL) {
-      grpc_client_config_set_lb_policy(config, lb_policy);
+      grpc_resolver_result_set_lb_policy(result, lb_policy);
       GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction");
     }
     grpc_resolved_addresses_destroy(addresses);
@@ -203,10 +203,10 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_timer_init(exec_ctx, &r->retry_timer, next_try, dns_on_retry_timer, r,
                     now);
   }
-  if (r->resolved_config) {
-    grpc_client_config_unref(exec_ctx, r->resolved_config);
+  if (r->resolved_result) {
+    grpc_resolver_result_unref(exec_ctx, r->resolved_result);
   }
-  r->resolved_config = config;
+  r->resolved_result = result;
   r->resolved_version++;
   dns_maybe_finish_next_locked(exec_ctx, r);
   gpr_mu_unlock(&r->mu);
@@ -228,9 +228,9 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
                                          dns_resolver *r) {
   if (r->next_completion != NULL &&
       r->resolved_version != r->published_version) {
-    *r->target_config = r->resolved_config;
-    if (r->resolved_config) {
-      grpc_client_config_ref(r->resolved_config);
+    *r->target_result = r->resolved_result;
+    if (r->resolved_result) {
+      grpc_resolver_result_ref(r->resolved_result);
     }
     grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
@@ -241,8 +241,8 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
 static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) {
   dns_resolver *r = (dns_resolver *)gr;
   gpr_mu_destroy(&r->mu);
-  if (r->resolved_config) {
-    grpc_client_config_unref(exec_ctx, r->resolved_config);
+  if (r->resolved_result) {
+    grpc_resolver_result_unref(exec_ctx, r->resolved_result);
   }
   grpc_client_channel_factory_unref(exec_ctx, r->client_channel_factory);
   gpr_free(r->name);
diff --git a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
index 1f7cce2f43..3807522d2b 100644
--- a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
+++ b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
@@ -66,8 +66,8 @@ typedef struct {
   int published;
   /** pending next completion, or NULL */
   grpc_closure *next_completion;
-  /** target config address for next completion */
-  grpc_client_config **target_config;
+  /** target result address for next completion */
+  grpc_resolver_result **target_result;
 } sockaddr_resolver;
 
 static void sockaddr_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
@@ -79,7 +79,7 @@ static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
 static void sockaddr_channel_saw_error(grpc_exec_ctx *exec_ctx,
                                        grpc_resolver *r);
 static void sockaddr_next(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
-                          grpc_client_config **target_config,
+                          grpc_resolver_result **target_result,
                           grpc_closure *on_complete);
 
 static const grpc_resolver_vtable sockaddr_resolver_vtable = {
@@ -91,7 +91,7 @@ static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx,
   sockaddr_resolver *r = (sockaddr_resolver *)resolver;
   gpr_mu_lock(&r->mu);
   if (r->next_completion != NULL) {
-    *r->target_config = NULL;
+    *r->target_result = NULL;
     grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
@@ -108,13 +108,13 @@ static void sockaddr_channel_saw_error(grpc_exec_ctx *exec_ctx,
 }
 
 static void sockaddr_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-                          grpc_client_config **target_config,
+                          grpc_resolver_result **target_result,
                           grpc_closure *on_complete) {
   sockaddr_resolver *r = (sockaddr_resolver *)resolver;
   gpr_mu_lock(&r->mu);
   GPR_ASSERT(!r->next_completion);
   r->next_completion = on_complete;
-  r->target_config = target_config;
+  r->target_result = target_result;
   sockaddr_maybe_finish_next_locked(exec_ctx, r);
   gpr_mu_unlock(&r->mu);
 }
@@ -122,17 +122,17 @@ static void sockaddr_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
 static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
                                               sockaddr_resolver *r) {
   if (r->next_completion != NULL && !r->published) {
-    grpc_client_config *cfg = grpc_client_config_create();
+    grpc_resolver_result *result = grpc_resolver_result_create();
     grpc_lb_policy_args lb_policy_args;
     memset(&lb_policy_args, 0, sizeof(lb_policy_args));
     lb_policy_args.addresses = r->addresses;
     lb_policy_args.client_channel_factory = r->client_channel_factory;
     grpc_lb_policy *lb_policy =
         grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args);
-    grpc_client_config_set_lb_policy(cfg, lb_policy);
+    grpc_resolver_result_set_lb_policy(result, lb_policy);
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "sockaddr");
     r->published = 1;
-    *r->target_config = cfg;
+    *r->target_result = result;
     grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c
index a7a775cc53..ef68cc86ea 100644
--- a/src/core/lib/channel/http_client_filter.c
+++ b/src/core/lib/channel/http_client_filter.c
@@ -43,6 +43,9 @@
 #define EXPECTED_CONTENT_TYPE "application/grpc"
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
+/* default maximum size of payload eligable for GET request */
+static const size_t kMaxPayloadSizeForGet = 2048;
+
 typedef struct call_data {
   grpc_linked_mdelem method;
   grpc_linked_mdelem scheme;
@@ -50,20 +53,39 @@ typedef struct call_data {
   grpc_linked_mdelem te_trailers;
   grpc_linked_mdelem content_type;
   grpc_linked_mdelem user_agent;
+  grpc_linked_mdelem payload_bin;
 
   grpc_metadata_batch *recv_initial_metadata;
+  uint8_t *payload_bytes;
+
+  /* Vars to read data off of send_message */
+  grpc_transport_stream_op send_op;
+  uint32_t send_length;
+  uint32_t send_flags;
+  gpr_slice incoming_slice;
+  grpc_slice_buffer_stream replacement_stream;
+  gpr_slice_buffer slices;
+  /* flag that indicates that all slices of send_messages aren't availble */
+  bool send_message_blocked;
 
   /** Closure to call when finished with the hc_on_recv hook */
   grpc_closure *on_done_recv;
+  grpc_closure *on_complete;
+  grpc_closure *post_send;
+
   /** Receive closures are chained: we inject this closure as the on_done_recv
       up-call on transport_op, and remember to call our on_done_recv member
       after handling it. */
   grpc_closure hc_on_recv;
+  grpc_closure hc_on_complete;
+  grpc_closure got_slice;
+  grpc_closure send_done;
 } call_data;
 
 typedef struct channel_data {
   grpc_mdelem *static_scheme;
   grpc_mdelem *user_agent;
+  size_t max_payload_size_for_get;
 } channel_data;
 
 typedef struct {
@@ -119,6 +141,24 @@ static void hc_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
   calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, error);
 }
 
+static void hc_on_complete(grpc_exec_ctx *exec_ctx, void *user_data,
+                           grpc_error *error) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  if (calld->payload_bytes) {
+    gpr_free(calld->payload_bytes);
+    calld->payload_bytes = NULL;
+  }
+  calld->on_complete->cb(exec_ctx, calld->on_complete->cb_arg, error);
+}
+
+static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
+  grpc_call_element *elem = elemp;
+  call_data *calld = elem->call_data;
+  gpr_slice_buffer_reset_and_unref(&calld->slices);
+  calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error);
+}
+
 static grpc_mdelem *client_strip_filter(void *user_data, grpc_mdelem *md) {
   /* eat the things we'd like to set ourselves */
   if (md->key == GRPC_MDSTR_METHOD) return NULL;
@@ -129,22 +169,105 @@ static grpc_mdelem *client_strip_filter(void *user_data, grpc_mdelem *md) {
   return md;
 }
 
-static void hc_mutate_op(grpc_call_element *elem,
+static void continue_send_message(grpc_exec_ctx *exec_ctx,
+                                  grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
+  uint8_t *wrptr = calld->payload_bytes;
+  while (grpc_byte_stream_next(exec_ctx, calld->send_op.send_message,
+                               &calld->incoming_slice, ~(size_t)0,
+                               &calld->got_slice)) {
+    memcpy(wrptr, GPR_SLICE_START_PTR(calld->incoming_slice),
+           GPR_SLICE_LENGTH(calld->incoming_slice));
+    wrptr += GPR_SLICE_LENGTH(calld->incoming_slice);
+    gpr_slice_buffer_add(&calld->slices, calld->incoming_slice);
+    if (calld->send_length == calld->slices.length) {
+      calld->send_message_blocked = false;
+      break;
+    }
+  }
+}
+
+static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
+  grpc_call_element *elem = elemp;
+  call_data *calld = elem->call_data;
+  calld->send_message_blocked = false;
+  gpr_slice_buffer_add(&calld->slices, calld->incoming_slice);
+  if (calld->send_length == calld->slices.length) {
+    /* Pass down the original send_message op that was blocked.*/
+    grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices,
+                                  calld->send_flags);
+    calld->send_op.send_message = &calld->replacement_stream.base;
+    calld->post_send = calld->send_op.on_complete;
+    calld->send_op.on_complete = &calld->send_done;
+    grpc_call_next_op(exec_ctx, elem, &calld->send_op);
+  } else {
+    continue_send_message(exec_ctx, elem);
+  }
+}
+
+static void hc_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                          grpc_transport_stream_op *op) {
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   channel_data *channeld = elem->channel_data;
+
   if (op->send_initial_metadata != NULL) {
+    /* Decide which HTTP VERB to use. We use GET if the request is marked
+    cacheable, and the operation contains both initial metadata and send
+    message, and the payload is below the size threshold, and all the data
+    for this request is immediately available. */
+    grpc_mdelem *method = GRPC_MDELEM_METHOD_POST;
+    calld->send_message_blocked = false;
+    if ((op->send_initial_metadata_flags &
+         GRPC_INITIAL_METADATA_CACHEABLE_REQUEST) &&
+        op->send_message != NULL &&
+        op->send_message->length < channeld->max_payload_size_for_get) {
+      method = GRPC_MDELEM_METHOD_GET;
+      calld->send_message_blocked = true;
+    } else if (op->send_initial_metadata_flags &
+               GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) {
+      method = GRPC_MDELEM_METHOD_PUT;
+    }
+
+    /* Attempt to read the data from send_message and create a header field. */
+    if (method == GRPC_MDELEM_METHOD_GET) {
+      /* allocate memory to hold the entire payload */
+      calld->payload_bytes = gpr_malloc(op->send_message->length);
+      GPR_ASSERT(calld->payload_bytes);
+
+      /* read slices of send_message and copy into payload_bytes */
+      calld->send_op = *op;
+      calld->send_length = op->send_message->length;
+      calld->send_flags = op->send_message->flags;
+      continue_send_message(exec_ctx, elem);
+
+      if (calld->send_message_blocked == false) {
+        /* when all the send_message data is available, then create a MDELEM and
+        append to headers */
+        grpc_mdelem *payload_bin = grpc_mdelem_from_metadata_strings(
+            GRPC_MDSTR_GRPC_PAYLOAD_BIN,
+            grpc_mdstr_from_buffer(calld->payload_bytes,
+                                   op->send_message->length));
+        grpc_metadata_batch_add_tail(op->send_initial_metadata,
+                                     &calld->payload_bin, payload_bin);
+        calld->on_complete = op->on_complete;
+        op->on_complete = &calld->hc_on_complete;
+        op->send_message = NULL;
+      } else {
+        /* Not all data is available. Fall back to POST. */
+        gpr_log(GPR_DEBUG,
+                "Request is marked Cacheable but not all data is available.\
+                            Falling back to POST");
+        method = GRPC_MDELEM_METHOD_POST;
+      }
+    }
+
     grpc_metadata_batch_filter(op->send_initial_metadata, client_strip_filter,
                                elem);
     /* Send : prefixed headers, which have to be before any application
        layer headers. */
-    grpc_metadata_batch_add_head(
-        op->send_initial_metadata, &calld->method,
-        op->send_initial_metadata_flags &
-                GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST
-            ? GRPC_MDELEM_METHOD_PUT
-            : GRPC_MDELEM_METHOD_POST);
+    grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->method,
+                                 method);
     grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->scheme,
                                  channeld->static_scheme);
     grpc_metadata_batch_add_tail(op->send_initial_metadata, &calld->te_trailers,
@@ -169,9 +292,16 @@ static void hc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                   grpc_transport_stream_op *op) {
   GPR_TIMER_BEGIN("hc_start_transport_op", 0);
   GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-  hc_mutate_op(elem, op);
+  hc_mutate_op(exec_ctx, elem, op);
   GPR_TIMER_END("hc_start_transport_op", 0);
-  grpc_call_next_op(exec_ctx, elem, op);
+  call_data *calld = elem->call_data;
+  if (op->send_message != NULL && calld->send_message_blocked) {
+    /* Don't forward the op. send_message contains slices that aren't ready
+    yet. The call will be forwarded by the op_complete of slice read call.
+    */
+  } else {
+    grpc_call_next_op(exec_ctx, elem, op);
+  }
 }
 
 /* Constructor for call_data */
@@ -180,14 +310,23 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
   calld->on_done_recv = NULL;
+  calld->on_complete = NULL;
+  calld->payload_bytes = NULL;
+  gpr_slice_buffer_init(&calld->slices);
   grpc_closure_init(&calld->hc_on_recv, hc_on_recv, elem);
+  grpc_closure_init(&calld->hc_on_complete, hc_on_complete, elem);
+  grpc_closure_init(&calld->got_slice, got_slice, elem);
+  grpc_closure_init(&calld->send_done, send_done, elem);
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
-                              void *ignored) {}
+                              void *ignored) {
+  call_data *calld = elem->call_data;
+  gpr_slice_buffer_destroy(&calld->slices);
+}
 
 static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) {
   unsigned i;
@@ -210,6 +349,22 @@ static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) {
   return GRPC_MDELEM_SCHEME_HTTP;
 }
 
+static size_t max_payload_size_from_args(const grpc_channel_args *args) {
+  if (args != NULL) {
+    for (size_t i = 0; i < args->num_args; ++i) {
+      if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET)) {
+        if (args->args[i].type != GRPC_ARG_INTEGER) {
+          gpr_log(GPR_ERROR, "%s: must be an integer",
+                  GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET);
+        } else {
+          return (size_t)args->args[i].value.integer;
+        }
+      }
+    }
+  }
+  return kMaxPayloadSizeForGet;
+}
+
 static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args,
                                         const char *transport_name) {
   gpr_strvec v;
@@ -233,8 +388,9 @@ static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args,
     }
   }
 
-  gpr_asprintf(&tmp, "%sgrpc-c/%s (%s; %s)", is_first ? "" : " ",
-               grpc_version_string(), GPR_PLATFORM_STRING, transport_name);
+  gpr_asprintf(&tmp, "%sgrpc-c/%s (%s; %s; %s)", is_first ? "" : " ",
+               grpc_version_string(), GPR_PLATFORM_STRING, transport_name,
+               grpc_g_stands_for());
   is_first = 0;
   gpr_strvec_add(&v, tmp);
 
@@ -267,6 +423,8 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(!args->is_last);
   GPR_ASSERT(args->optional_transport != NULL);
   chand->static_scheme = scheme_from_args(args->channel_args);
+  chand->max_payload_size_for_get =
+      max_payload_size_from_args(args->channel_args);
   chand->user_agent = grpc_mdelem_from_metadata_strings(
       GRPC_MDSTR_USER_AGENT,
       user_agent_from_args(args->channel_args,
diff --git a/src/core/lib/channel/http_client_filter.h b/src/core/lib/channel/http_client_filter.h
index 47081175ea..9e6e106e9c 100644
--- a/src/core/lib/channel/http_client_filter.h
+++ b/src/core/lib/channel/http_client_filter.h
@@ -41,4 +41,7 @@ extern const grpc_channel_filter grpc_http_client_filter;
 /* Channel arg to override the http2 :scheme header */
 #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme"
 
+/* Channel arg to determine maximum size of payload eligable for GET request */
+#define GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET "grpc.max_payload_size_for_get"
+
 #endif /* GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H */
diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c
index 5ce51f9016..0f2bf97824 100644
--- a/src/core/lib/channel/http_server_filter.c
+++ b/src/core/lib/channel/http_server_filter.c
@@ -49,17 +49,32 @@ typedef struct call_data {
   uint8_t seen_scheme;
   uint8_t seen_te_trailers;
   uint8_t seen_authority;
+  uint8_t seen_payload_bin;
   grpc_linked_mdelem status;
   grpc_linked_mdelem content_type;
 
+  /* flag to ensure payload_bin is delivered only once */
+  uint8_t payload_bin_delivered;
+
   grpc_metadata_batch *recv_initial_metadata;
   bool *recv_idempotent_request;
+  bool *recv_cacheable_request;
   /** Closure to call when finished with the hs_on_recv hook */
   grpc_closure *on_done_recv;
+  /** Closure to call when we retrieve read message from the payload-bin header
+   */
+  grpc_closure *recv_message_ready;
+  grpc_closure *on_complete;
+  grpc_byte_stream **pp_recv_message;
+  gpr_slice_buffer read_slice_buffer;
+  grpc_slice_buffer_stream read_stream;
+
   /** Receive closures are chained: we inject this closure as the on_done_recv
       up-call on transport_op, and remember to call our on_done_recv member
       after handling it. */
   grpc_closure hs_on_recv;
+  grpc_closure hs_on_complete;
+  grpc_closure hs_recv_message_ready;
 } call_data;
 
 typedef struct channel_data { uint8_t unused; } channel_data;
@@ -76,16 +91,20 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
 
   /* Check if it is one of the headers we care about. */
   if (md == GRPC_MDELEM_TE_TRAILERS || md == GRPC_MDELEM_METHOD_POST ||
-      md == GRPC_MDELEM_METHOD_PUT || md == GRPC_MDELEM_SCHEME_HTTP ||
-      md == GRPC_MDELEM_SCHEME_HTTPS ||
+      md == GRPC_MDELEM_METHOD_PUT || md == GRPC_MDELEM_METHOD_GET ||
+      md == GRPC_MDELEM_SCHEME_HTTP || md == GRPC_MDELEM_SCHEME_HTTPS ||
       md == GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC) {
     /* swallow it */
     if (md == GRPC_MDELEM_METHOD_POST) {
       calld->seen_method = 1;
       *calld->recv_idempotent_request = false;
+      *calld->recv_cacheable_request = false;
     } else if (md == GRPC_MDELEM_METHOD_PUT) {
       calld->seen_method = 1;
       *calld->recv_idempotent_request = true;
+    } else if (md == GRPC_MDELEM_METHOD_GET) {
+      calld->seen_method = 1;
+      *calld->recv_cacheable_request = true;
     } else if (md->key == GRPC_MDSTR_SCHEME) {
       calld->seen_scheme = 1;
     } else if (md == GRPC_MDELEM_TE_TRAILERS) {
@@ -137,6 +156,16 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
         GRPC_MDSTR_AUTHORITY, GRPC_MDSTR_REF(md->value));
     calld->seen_authority = 1;
     return authority;
+  } else if (md->key == GRPC_MDSTR_GRPC_PAYLOAD_BIN) {
+    /* Retrieve the payload from the value of the 'grpc-internal-payload-bin'
+       header field */
+    calld->seen_payload_bin = 1;
+    gpr_slice_buffer_init(&calld->read_slice_buffer);
+    gpr_slice_buffer_add(&calld->read_slice_buffer,
+                         gpr_slice_ref(md->value->slice));
+    grpc_slice_buffer_stream_init(&calld->read_stream,
+                                  &calld->read_slice_buffer, 0);
+    return NULL;
   } else {
     return md;
   }
@@ -189,6 +218,36 @@ static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
   GRPC_ERROR_UNREF(err);
 }
 
+static void hs_on_complete(grpc_exec_ctx *exec_ctx, void *user_data,
+                           grpc_error *err) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  /* Call recv_message_ready if we got the payload via the header field */
+  if (calld->seen_payload_bin && calld->recv_message_ready != NULL) {
+    *calld->pp_recv_message = calld->payload_bin_delivered
+                                  ? NULL
+                                  : (grpc_byte_stream *)&calld->read_stream;
+    calld->recv_message_ready->cb(exec_ctx, calld->recv_message_ready->cb_arg,
+                                  err);
+    calld->recv_message_ready = NULL;
+    calld->payload_bin_delivered = true;
+  }
+  calld->on_complete->cb(exec_ctx, calld->on_complete->cb_arg, err);
+}
+
+static void hs_recv_message_ready(grpc_exec_ctx *exec_ctx, void *user_data,
+                                  grpc_error *err) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  if (calld->seen_payload_bin) {
+    /* do nothing. This is probably a GET request, and payload will be returned
+    in hs_on_complete callback. */
+  } else {
+    calld->recv_message_ready->cb(exec_ctx, calld->recv_message_ready->cb_arg,
+                                  err);
+  }
+}
+
 static void hs_mutate_op(grpc_call_element *elem,
                          grpc_transport_stream_op *op) {
   /* grab pointers to our data from the call element */
@@ -206,11 +265,25 @@ static void hs_mutate_op(grpc_call_element *elem,
   if (op->recv_initial_metadata) {
     /* substitute our callback for the higher callback */
     GPR_ASSERT(op->recv_idempotent_request != NULL);
+    GPR_ASSERT(op->recv_cacheable_request != NULL);
     calld->recv_initial_metadata = op->recv_initial_metadata;
     calld->recv_idempotent_request = op->recv_idempotent_request;
+    calld->recv_cacheable_request = op->recv_cacheable_request;
     calld->on_done_recv = op->recv_initial_metadata_ready;
     op->recv_initial_metadata_ready = &calld->hs_on_recv;
   }
+
+  if (op->recv_message) {
+    calld->recv_message_ready = op->recv_message_ready;
+    calld->pp_recv_message = op->recv_message;
+    if (op->recv_message_ready) {
+      op->recv_message_ready = &calld->hs_recv_message_ready;
+    }
+    if (op->on_complete) {
+      calld->on_complete = op->on_complete;
+      op->on_complete = &calld->hs_on_complete;
+    }
+  }
 }
 
 static void hs_start_transport_op(grpc_exec_ctx *exec_ctx,
@@ -232,6 +305,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   /* initialize members */
   memset(calld, 0, sizeof(*calld));
   grpc_closure_init(&calld->hs_on_recv, hs_on_recv, elem);
+  grpc_closure_init(&calld->hs_on_complete, hs_on_complete, elem);
+  grpc_closure_init(&calld->hs_recv_message_ready, hs_recv_message_ready, elem);
   return GRPC_ERROR_NONE;
 }
 
diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h
index bc7781250e..6c769accdb 100644
--- a/src/core/lib/iomgr/error.h
+++ b/src/core/lib/iomgr/error.h
@@ -47,7 +47,8 @@
 ///  if a grpc_error is passed to a grpc_closure callback function (functions
 ///    with the signature:
 ///      void (*f)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error))
-///    then those functions do not automatically own a ref to error
+///    then those functions do not own a ref to error (but are free to manually
+///    take a reference).
 ///  if a grpc_error is passed to *ANY OTHER FUNCTION* then that function takes
 ///    ownership of the error
 /// Errors have:
diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c
index 02bcbaa10f..eba347125e 100644
--- a/src/core/lib/iomgr/ev_epoll_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_linux.c
@@ -928,7 +928,8 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
     fd->polling_island = NULL;
   }
 
-  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error),
+                      NULL);
 
   gpr_mu_unlock(&fd->mu);
   UNREF_BY(fd, 2, reason); /* Drop the reference */
@@ -940,6 +941,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
     PI_UNREF(exec_ctx, unref_pi, "fd_orphan");
   }
   GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error));
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_error *fd_shutdown_error(bool shutdown) {
@@ -1353,8 +1355,10 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
   gpr_mu_unlock(&pollset->mu);
 
   do {
+    GRPC_SCHEDULING_START_BLOCKING_REGION;
     ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms,
                         sig_mask);
+    GRPC_SCHEDULING_END_BLOCKING_REGION;
     if (ep_rv < 0) {
       if (errno != EINTR) {
         gpr_asprintf(&err_msg,
diff --git a/src/core/lib/security/context/security_context.c b/src/core/lib/security/context/security_context.c
index 127b13ee50..2204fadf54 100644
--- a/src/core/lib/security/context/security_context.c
+++ b/src/core/lib/security/context/security_context.c
@@ -99,6 +99,9 @@ void grpc_client_security_context_destroy(void *ctx) {
   grpc_client_security_context *c = (grpc_client_security_context *)ctx;
   grpc_call_credentials_unref(c->creds);
   GRPC_AUTH_CONTEXT_UNREF(c->auth_context, "client_security_context");
+  if (c->extension.instance != NULL && c->extension.destroy != NULL) {
+    c->extension.destroy(c->extension.instance);
+  }
   gpr_free(ctx);
 }
 
@@ -114,6 +117,9 @@ grpc_server_security_context *grpc_server_security_context_create(void) {
 void grpc_server_security_context_destroy(void *ctx) {
   grpc_server_security_context *c = (grpc_server_security_context *)ctx;
   GRPC_AUTH_CONTEXT_UNREF(c->auth_context, "server_security_context");
+  if (c->extension.instance != NULL && c->extension.destroy != NULL) {
+    c->extension.destroy(c->extension.instance);
+  }
   gpr_free(ctx);
 }
 
diff --git a/src/core/lib/security/context/security_context.h b/src/core/lib/security/context/security_context.h
index 4e7666dfe3..1e131a0c23 100644
--- a/src/core/lib/security/context/security_context.h
+++ b/src/core/lib/security/context/security_context.h
@@ -84,6 +84,16 @@ void grpc_auth_context_unref(grpc_auth_context *policy);
 
 void grpc_auth_property_reset(grpc_auth_property *property);
 
+/* --- grpc_security_context_extension ---
+
+   Extension to the security context that may be set in a filter and accessed
+   later by a higher level method on a grpc_call object. */
+
+typedef struct {
+  void *instance;
+  void (*destroy)(void *);
+} grpc_security_context_extension;
+
 /* --- grpc_client_security_context ---
 
    Internal client-side security context. */
@@ -91,6 +101,7 @@ void grpc_auth_property_reset(grpc_auth_property *property);
 typedef struct {
   grpc_call_credentials *creds;
   grpc_auth_context *auth_context;
+  grpc_security_context_extension extension;
 } grpc_client_security_context;
 
 grpc_client_security_context *grpc_client_security_context_create(void);
@@ -102,6 +113,7 @@ void grpc_client_security_context_destroy(void *ctx);
 
 typedef struct {
   grpc_auth_context *auth_context;
+  grpc_security_context_extension extension;
 } grpc_server_security_context;
 
 grpc_server_security_context *grpc_server_security_context_create(void);
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index 64afcecc07..55e6d99057 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -149,6 +149,7 @@ struct call_data {
 
   grpc_metadata_batch *recv_initial_metadata;
   bool recv_idempotent_request;
+  bool recv_cacheable_request;
   grpc_metadata_array initial_metadata;
 
   request_matcher *request_matcher;
@@ -497,9 +498,12 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
             &rc->data.batch.details->method_capacity, calld->path);
       rc->data.batch.details->deadline = calld->deadline;
       rc->data.batch.details->flags =
-          0 | (calld->recv_idempotent_request
-                   ? GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST
-                   : 0);
+          (calld->recv_idempotent_request
+               ? GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST
+               : 0) |
+          (calld->recv_cacheable_request
+               ? GRPC_INITIAL_METADATA_CACHEABLE_REQUEST
+               : 0);
       break;
     case REGISTERED_CALL:
       *rc->data.registered.deadline = calld->deadline;
@@ -779,6 +783,7 @@ static void server_mutate_op(grpc_call_element *elem,
     calld->on_done_recv_initial_metadata = op->recv_initial_metadata_ready;
     op->recv_initial_metadata_ready = &calld->server_on_recv_initial_metadata;
     op->recv_idempotent_request = &calld->recv_idempotent_request;
+    op->recv_cacheable_request = &calld->recv_cacheable_request;
   }
 }
 
diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c
index 1942075054..41242684da 100644
--- a/src/core/lib/surface/version.c
+++ b/src/core/lib/surface/version.c
@@ -37,3 +37,5 @@
 #include <grpc/grpc.h>
 
 const char *grpc_version_string(void) { return "1.1.0-dev"; }
+
+const char *grpc_g_stands_for(void) { return "good"; }
diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c
index 8f3e5b5b40..fce591f346 100644
--- a/src/core/lib/transport/static_metadata.c
+++ b/src/core/lib/transport/static_metadata.c
@@ -51,15 +51,15 @@ uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
-    {11, 33, 10, 33, 12, 33, 12, 49, 13, 33, 14, 33, 15, 33, 16, 33, 17, 33,
+    {11, 33, 10, 33, 12, 33, 12, 50, 13, 33, 14, 33, 15, 33, 16, 33, 17, 33,
      19, 33, 20, 33, 21, 33, 22, 33, 23, 33, 24, 33, 25, 33, 26, 33, 27, 33,
      28, 18, 28, 33, 29, 33, 30, 33, 34, 33, 35, 33, 36, 33, 37, 33, 40, 31,
-     40, 32, 40, 48, 40, 53, 40, 54, 40, 55, 40, 56, 42, 31, 42, 48, 42, 53,
-     45, 0,  45, 1,  45, 2,  50, 33, 57, 33, 58, 33, 59, 33, 60, 33, 61, 33,
-     62, 33, 63, 33, 64, 33, 65, 33, 66, 33, 67, 33, 68, 38, 68, 70, 68, 73,
-     69, 81, 69, 82, 71, 33, 72, 33, 74, 33, 75, 33, 76, 33, 77, 33, 78, 39,
-     78, 51, 78, 52, 79, 33, 80, 33, 83, 3,  83, 4,  83, 5,  83, 6,  83, 7,
-     83, 8,  83, 9,  84, 33, 85, 86, 87, 33, 88, 33, 89, 33, 90, 33, 91, 33};
+     40, 32, 40, 49, 40, 54, 40, 55, 40, 56, 40, 57, 42, 31, 42, 49, 42, 54,
+     46, 0,  46, 1,  46, 2,  51, 33, 58, 33, 59, 33, 60, 33, 61, 33, 62, 33,
+     63, 33, 64, 33, 65, 33, 66, 33, 67, 33, 68, 33, 69, 38, 69, 71, 69, 74,
+     70, 82, 70, 83, 72, 33, 73, 33, 75, 33, 76, 33, 77, 33, 78, 33, 79, 39,
+     79, 52, 79, 53, 80, 33, 81, 33, 84, 3,  84, 4,  84, 5,  84, 6,  84, 7,
+     84, 8,  84, 9,  85, 33, 86, 87, 88, 33, 89, 33, 90, 33, 91, 33, 92, 33};
 
 const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "0",
@@ -107,6 +107,7 @@ const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "grpc-encoding",
     "grpc-internal-encoding-request",
     "grpc-message",
+    "grpc-payload-bin",
     "grpc-status",
     "grpc-timeout",
     "grpc-tracing-bin",
diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h
index b51bacac50..54b6f38be1 100644
--- a/src/core/lib/transport/static_metadata.h
+++ b/src/core/lib/transport/static_metadata.h
@@ -44,7 +44,7 @@
 
 #include "src/core/lib/transport/metadata.h"
 
-#define GRPC_STATIC_MDSTR_COUNT 92
+#define GRPC_STATIC_MDSTR_COUNT 93
 extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 /* "0" */
 #define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0])
@@ -136,101 +136,103 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (&grpc_static_mdstr_table[43])
 /* "grpc-message" */
 #define GRPC_MDSTR_GRPC_MESSAGE (&grpc_static_mdstr_table[44])
+/* "grpc-payload-bin" */
+#define GRPC_MDSTR_GRPC_PAYLOAD_BIN (&grpc_static_mdstr_table[45])
 /* "grpc-status" */
-#define GRPC_MDSTR_GRPC_STATUS (&grpc_static_mdstr_table[45])
+#define GRPC_MDSTR_GRPC_STATUS (&grpc_static_mdstr_table[46])
 /* "grpc-timeout" */
-#define GRPC_MDSTR_GRPC_TIMEOUT (&grpc_static_mdstr_table[46])
+#define GRPC_MDSTR_GRPC_TIMEOUT (&grpc_static_mdstr_table[47])
 /* "grpc-tracing-bin" */
-#define GRPC_MDSTR_GRPC_TRACING_BIN (&grpc_static_mdstr_table[47])
+#define GRPC_MDSTR_GRPC_TRACING_BIN (&grpc_static_mdstr_table[48])
 /* "gzip" */
-#define GRPC_MDSTR_GZIP (&grpc_static_mdstr_table[48])
+#define GRPC_MDSTR_GZIP (&grpc_static_mdstr_table[49])
 /* "gzip, deflate" */
-#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (&grpc_static_mdstr_table[49])
+#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (&grpc_static_mdstr_table[50])
 /* "host" */
-#define GRPC_MDSTR_HOST (&grpc_static_mdstr_table[50])
+#define GRPC_MDSTR_HOST (&grpc_static_mdstr_table[51])
 /* "http" */
-#define GRPC_MDSTR_HTTP (&grpc_static_mdstr_table[51])
+#define GRPC_MDSTR_HTTP (&grpc_static_mdstr_table[52])
 /* "https" */
-#define GRPC_MDSTR_HTTPS (&grpc_static_mdstr_table[52])
+#define GRPC_MDSTR_HTTPS (&grpc_static_mdstr_table[53])
 /* "identity" */
-#define GRPC_MDSTR_IDENTITY (&grpc_static_mdstr_table[53])
+#define GRPC_MDSTR_IDENTITY (&grpc_static_mdstr_table[54])
 /* "identity,deflate" */
-#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (&grpc_static_mdstr_table[54])
+#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (&grpc_static_mdstr_table[55])
 /* "identity,deflate,gzip" */
 #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \
-  (&grpc_static_mdstr_table[55])
+  (&grpc_static_mdstr_table[56])
 /* "identity,gzip" */
-#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (&grpc_static_mdstr_table[56])
+#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (&grpc_static_mdstr_table[57])
 /* "if-match" */
-#define GRPC_MDSTR_IF_MATCH (&grpc_static_mdstr_table[57])
+#define GRPC_MDSTR_IF_MATCH (&grpc_static_mdstr_table[58])
 /* "if-modified-since" */
-#define GRPC_MDSTR_IF_MODIFIED_SINCE (&grpc_static_mdstr_table[58])
+#define GRPC_MDSTR_IF_MODIFIED_SINCE (&grpc_static_mdstr_table[59])
 /* "if-none-match" */
-#define GRPC_MDSTR_IF_NONE_MATCH (&grpc_static_mdstr_table[59])
+#define GRPC_MDSTR_IF_NONE_MATCH (&grpc_static_mdstr_table[60])
 /* "if-range" */
-#define GRPC_MDSTR_IF_RANGE (&grpc_static_mdstr_table[60])
+#define GRPC_MDSTR_IF_RANGE (&grpc_static_mdstr_table[61])
 /* "if-unmodified-since" */
-#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (&grpc_static_mdstr_table[61])
+#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (&grpc_static_mdstr_table[62])
 /* "last-modified" */
-#define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[62])
+#define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[63])
 /* "link" */
-#define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[63])
+#define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[64])
 /* "load-reporting-initial" */
-#define GRPC_MDSTR_LOAD_REPORTING_INITIAL (&grpc_static_mdstr_table[64])
+#define GRPC_MDSTR_LOAD_REPORTING_INITIAL (&grpc_static_mdstr_table[65])
 /* "load-reporting-trailing" */
-#define GRPC_MDSTR_LOAD_REPORTING_TRAILING (&grpc_static_mdstr_table[65])
+#define GRPC_MDSTR_LOAD_REPORTING_TRAILING (&grpc_static_mdstr_table[66])
 /* "location" */
-#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[66])
+#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[67])
 /* "max-forwards" */
-#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[67])
+#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[68])
 /* ":method" */
-#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[68])
+#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[69])
 /* ":path" */
-#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[69])
+#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[70])
 /* "POST" */
-#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[70])
+#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[71])
 /* "proxy-authenticate" */
-#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[71])
+#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[72])
 /* "proxy-authorization" */
-#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[72])
+#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[73])
 /* "PUT" */
-#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[73])
+#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[74])
 /* "range" */
-#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[74])
+#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[75])
 /* "referer" */
-#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[75])
+#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[76])
 /* "refresh" */
-#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[76])
+#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[77])
 /* "retry-after" */
-#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[77])
+#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[78])
 /* ":scheme" */
-#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[78])
+#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[79])
 /* "server" */
-#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[79])
+#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[80])
 /* "set-cookie" */
-#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[80])
+#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[81])
 /* "/" */
-#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[81])
+#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[82])
 /* "/index.html" */
-#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[82])
+#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[83])
 /* ":status" */
-#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[83])
+#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[84])
 /* "strict-transport-security" */
-#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[84])
+#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[85])
 /* "te" */
-#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[85])
+#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[86])
 /* "trailers" */
-#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[86])
+#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[87])
 /* "transfer-encoding" */
-#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[87])
+#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[88])
 /* "user-agent" */
-#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[88])
+#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[89])
 /* "vary" */
-#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[89])
+#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[90])
 /* "via" */
-#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[90])
+#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[91])
 /* "www-authenticate" */
-#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[91])
+#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[92])
 
 #define GRPC_STATIC_MDELEM_COUNT 81
 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h
index 1705e6e582..26ed6cb839 100644
--- a/src/core/lib/transport/transport.h
+++ b/src/core/lib/transport/transport.h
@@ -124,6 +124,7 @@ typedef struct grpc_transport_stream_op {
   /** Receive initial metadata from the stream, into provided metadata batch. */
   grpc_metadata_batch *recv_initial_metadata;
   bool *recv_idempotent_request;
+  bool *recv_cacheable_request;
   /** Should be enqueued when initial metadata is ready to be processed. */
   grpc_closure *recv_initial_metadata_ready;
 
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 45bb858e2e..1a014da3b8 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -35,10 +35,9 @@
 
 #include <grpc++/impl/service_type.h>
 #include <grpc++/server.h>
-#include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
+#include <grpc/support/useful.h>
 
-#include "include/grpc/support/useful.h"
 #include "src/cpp/server/thread_pool_interface.h"
 
 namespace grpc {
@@ -154,14 +153,12 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
     (*option)->UpdateArguments(&args);
     (*option)->UpdatePlugins(&plugins_);
   }
-  if (!thread_pool) {
-    for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
-      if ((*plugin)->has_sync_methods()) {
-        thread_pool.reset(CreateDefaultThreadPool());
-        has_sync_methods = true;
-        break;
-      }
+  for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
+    if (!thread_pool && (*plugin)->has_sync_methods()) {
+      thread_pool.reset(CreateDefaultThreadPool());
+      has_sync_methods = true;
     }
+    (*plugin)->UpdateChannelArguments(&args);
   }
   if (max_message_size_ > 0) {
     args.SetInt(GRPC_ARG_MAX_MESSAGE_LENGTH, max_message_size_);
diff --git a/src/objective-c/RxLibrary/transformations/GRXMappingWriter.m b/src/objective-c/RxLibrary/transformations/GRXMappingWriter.m
index f3242e4fa9..6ee1545d25 100644
--- a/src/objective-c/RxLibrary/transformations/GRXMappingWriter.m
+++ b/src/objective-c/RxLibrary/transformations/GRXMappingWriter.m
@@ -33,10 +33,6 @@
 
 #import "GRXMappingWriter.h"
 
-static id (^kIdentity)(id value) = ^id(id value) {
-  return value;
-};
-
 @interface GRXForwardingWriter () <GRXWriteable>
 @end
 
@@ -51,7 +47,9 @@ static id (^kIdentity)(id value) = ^id(id value) {
 // Designated initializer
 - (instancetype)initWithWriter:(GRXWriter *)writer map:(id (^)(id value))map {
   if ((self = [super initWithWriter:writer])) {
-    _map = map ?: kIdentity;
+    _map = map ?: ^id(id value) {
+      return value;
+    };
   }
   return self;
 }
diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c
index 66ca1513ed..31c59fe5ad 100644
--- a/src/php/ext/grpc/call.c
+++ b/src/php/ext/grpc/call.c
@@ -164,6 +164,9 @@ bool create_metadata_array(zval *array, grpc_metadata_array *metadata) {
     if (key_type1 != HASH_KEY_IS_STRING) {
       return false;
     }
+    if (!grpc_header_key_is_legal(key1, strlen(key1))) {
+      return false;
+    }
     inner_array_hash = Z_ARRVAL_P(inner_array);
     PHP_GRPC_HASH_FOREACH_VAL_START(inner_array_hash, value)
       if (Z_TYPE_P(value) != IS_STRING) {
diff --git a/src/php/ext/grpc/call_credentials.c b/src/php/ext/grpc/call_credentials.c
index 6921a5df17..25c92c91fe 100644
--- a/src/php/ext/grpc/call_credentials.c
+++ b/src/php/ext/grpc/call_credentials.c
@@ -192,24 +192,16 @@ void plugin_get_metadata(void *ptr, grpc_auth_metadata_context context,
   /* call the user callback function */
   zend_call_function(state->fci, state->fci_cache TSRMLS_CC);
 
-  if (Z_TYPE_P(retval) != IS_ARRAY) {
-    zend_throw_exception(spl_ce_InvalidArgumentException,
-                         "plugin callback must return metadata array",
-                         1 TSRMLS_CC);
-    return;
-  }
-
+  grpc_status_code code = GRPC_STATUS_OK;
   grpc_metadata_array metadata;
-  if (!create_metadata_array(retval, &metadata)) {
-    zend_throw_exception(spl_ce_InvalidArgumentException,
-                         "invalid metadata", 1 TSRMLS_CC);
+
+  if (Z_TYPE_P(retval) != IS_ARRAY) {
+    code = GRPC_STATUS_INVALID_ARGUMENT;
+  } else if (!create_metadata_array(retval, &metadata)) {
     grpc_metadata_array_destroy(&metadata);
-    return;
+    code = GRPC_STATUS_INVALID_ARGUMENT;
   }
 
-  /* TODO: handle error */
-  grpc_status_code code = GRPC_STATUS_OK;
-
   /* Pass control back to core */
   cb(user_data, metadata.metadata, metadata.count, code, NULL);
 }
diff --git a/src/php/lib/Grpc/AbstractCall.php b/src/php/lib/Grpc/AbstractCall.php
index c86d298805..c58ee56742 100644
--- a/src/php/lib/Grpc/AbstractCall.php
+++ b/src/php/lib/Grpc/AbstractCall.php
@@ -34,8 +34,15 @@
 
 namespace Grpc;
 
+/**
+ * Class AbstractCall
+ * @package Grpc
+ */
 abstract class AbstractCall
 {
+    /**
+     * @var Call
+     */
     protected $call;
     protected $deserialize;
     protected $metadata;
@@ -51,13 +58,15 @@ abstract class AbstractCall
      *                              the response
      * @param array    $options     Call options (optional)
      */
-    public function __construct(Channel $channel,
-                                $method,
-                                $deserialize,
-                                $options = [])
-    {
-        if (isset($options['timeout']) &&
-            is_numeric($timeout = $options['timeout'])) {
+    public function __construct(
+        Channel $channel,
+        $method,
+        $deserialize,
+        $options = []
+    ) {
+        if (array_key_exists('timeout', $options) &&
+            is_numeric($timeout = $options['timeout'])
+        ) {
             $now = Timeval::now();
             $delta = new Timeval($timeout);
             $deadline = $now->add($delta);
@@ -68,17 +77,19 @@ abstract class AbstractCall
         $this->deserialize = $deserialize;
         $this->metadata = null;
         $this->trailing_metadata = null;
-        if (isset($options['call_credentials_callback']) &&
+        if (array_key_exists('call_credentials_callback', $options) &&
             is_callable($call_credentials_callback =
-                        $options['call_credentials_callback'])) {
+                $options['call_credentials_callback'])
+        ) {
             $call_credentials = CallCredentials::createFromPlugin(
-                $call_credentials_callback);
+                $call_credentials_callback
+            );
             $this->call->setCredentials($call_credentials);
         }
     }
 
     /**
-     * @return The metadata sent by the server.
+     * @return mixed The metadata sent by the server.
      */
     public function getMetadata()
     {
@@ -86,7 +97,7 @@ abstract class AbstractCall
     }
 
     /**
-     * @return The trailing metadata sent by the server.
+     * @return mixed The trailing metadata sent by the server.
      */
     public function getTrailingMetadata()
     {
@@ -114,7 +125,7 @@ abstract class AbstractCall
      *
      * @param string $value The binary value to deserialize
      *
-     * @return The deserialized value
+     * @return mixed The deserialized value
      */
     protected function deserializeResponse($value)
     {
diff --git a/src/php/lib/Grpc/BidiStreamingCall.php b/src/php/lib/Grpc/BidiStreamingCall.php
index c2fdb94b86..5d6ecfb275 100644
--- a/src/php/lib/Grpc/BidiStreamingCall.php
+++ b/src/php/lib/Grpc/BidiStreamingCall.php
@@ -45,7 +45,7 @@ class BidiStreamingCall extends AbstractCall
      *
      * @param array $metadata Metadata to send with the call, if applicable
      */
-    public function start($metadata = [])
+    public function start(array $metadata = [])
     {
         $this->call->startBatch([
             OP_SEND_INITIAL_METADATA => $metadata,
@@ -55,7 +55,7 @@ class BidiStreamingCall extends AbstractCall
     /**
      * Reads the next value from the server.
      *
-     * @return The next value from the server, or null if there is none
+     * @return mixed The next value from the server, or null if there is none
      */
     public function read()
     {
@@ -82,7 +82,7 @@ class BidiStreamingCall extends AbstractCall
     public function write($data, $options = [])
     {
         $message_array = ['message' => $data->serialize()];
-        if (isset($options['flags'])) {
+        if (array_key_exists('flags', $options)) {
             $message_array['flags'] = $options['flags'];
         }
         $this->call->startBatch([
@@ -103,8 +103,8 @@ class BidiStreamingCall extends AbstractCall
     /**
      * Wait for the server to send the status, and return it.
      *
-     * @return object The status object, with integer $code, string $details,
-     *                and array $metadata members
+     * @return \stdClass The status object, with integer $code, string $details,
+     *                   and array $metadata members
      */
     public function getStatus()
     {
diff --git a/src/php/lib/Grpc/ClientStreamingCall.php b/src/php/lib/Grpc/ClientStreamingCall.php
index 4050f7ed06..c96e26d033 100644
--- a/src/php/lib/Grpc/ClientStreamingCall.php
+++ b/src/php/lib/Grpc/ClientStreamingCall.php
@@ -60,10 +60,10 @@ class ClientStreamingCall extends AbstractCall
      * @param array      $options an array of options, possible keys:
      *                            'flags' => a number
      */
-    public function write($data, $options = [])
+    public function write($data, array $options = [])
     {
         $message_array = ['message' => $data->serialize()];
-        if (isset($options['flags'])) {
+        if (array_key_exists('flags', $options)) {
             $message_array['flags'] = $options['flags'];
         }
         $this->call->startBatch([
@@ -74,7 +74,7 @@ class ClientStreamingCall extends AbstractCall
     /**
      * Wait for the server to respond with data and a status.
      *
-     * @return [response data, status]
+     * @return array [response data, status]
      */
     public function wait()
     {
diff --git a/src/php/lib/Grpc/ServerStreamingCall.php b/src/php/lib/Grpc/ServerStreamingCall.php
index ba89d9f972..5f0d42c528 100644
--- a/src/php/lib/Grpc/ServerStreamingCall.php
+++ b/src/php/lib/Grpc/ServerStreamingCall.php
@@ -36,14 +36,14 @@ namespace Grpc;
 
 /**
  * Represents an active call that sends a single message and then gets a stream
- * of reponses.
+ * of responses.
  */
 class ServerStreamingCall extends AbstractCall
 {
     /**
      * Start the call.
      *
-     * @param $data The data to send
+     * @param mixed $data     The data to send
      * @param array $metadata Metadata to send with the call, if applicable
      * @param array $options  an array of options, possible keys:
      *                        'flags' => a number
@@ -51,7 +51,7 @@ class ServerStreamingCall extends AbstractCall
     public function start($data, $metadata = [], $options = [])
     {
         $message_array = ['message' => $data->serialize()];
-        if (isset($options['flags'])) {
+        if (array_key_exists('flags', $options)) {
             $message_array['flags'] = $options['flags'];
         }
         $event = $this->call->startBatch([
@@ -64,7 +64,7 @@ class ServerStreamingCall extends AbstractCall
     }
 
     /**
-     * @return An iterator of response values
+     * @return mixed An iterator of response values
      */
     public function responses()
     {
@@ -82,8 +82,8 @@ class ServerStreamingCall extends AbstractCall
     /**
      * Wait for the server to send the status, and return it.
      *
-     * @return object The status object, with integer $code, string $details,
-     *                and array $metadata members
+     * @return \stdClass The status object, with integer $code, string $details,
+     *                   and array $metadata members
      */
     public function getStatus()
     {
diff --git a/src/php/lib/Grpc/UnaryCall.php b/src/php/lib/Grpc/UnaryCall.php
index a71b05dc93..487b89ebc1 100644
--- a/src/php/lib/Grpc/UnaryCall.php
+++ b/src/php/lib/Grpc/UnaryCall.php
@@ -43,7 +43,7 @@ class UnaryCall extends AbstractCall
     /**
      * Start the call.
      *
-     * @param $data The data to send
+     * @param mixed $data     The data to send
      * @param array $metadata Metadata to send with the call, if applicable
      * @param array $options  an array of options, possible keys:
      *                        'flags' => a number
@@ -66,7 +66,7 @@ class UnaryCall extends AbstractCall
     /**
      * Wait for the server to respond with data and a status.
      *
-     * @return [response data, status]
+     * @return array [response data, status]
      */
     public function wait()
     {
diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php
index bf40549a04..c94ba61296 100755
--- a/src/php/tests/interop/interop_client.php
+++ b/src/php/tests/interop/interop_client.php
@@ -54,6 +54,15 @@ function hardAssert($value, $error_message)
     }
 }
 
+function hardAssertIfStatusOk($status)
+{
+    if ($status->code !== Grpc\STATUS_OK) {
+        echo "Call did not complete successfully. Status object:\n";
+        var_dump($status);
+        exit(1);
+    }
+}
+
 /**
  * Run the empty_unary test.
  *
@@ -62,7 +71,7 @@ function hardAssert($value, $error_message)
 function emptyUnary($stub)
 {
     list($result, $status) = $stub->EmptyCall(new grpc\testing\EmptyMessage())->wait();
-    hardAssert($status->code === Grpc\STATUS_OK, 'Call did not complete successfully');
+    hardAssertIfStatusOk($status);
     hardAssert($result !== null, 'Call completed with a null response');
 }
 
@@ -105,7 +114,7 @@ function performLargeUnary($stub, $fillUsername = false, $fillOauthScope = false
     }
 
     list($result, $status) = $stub->UnaryCall($request, [], $options)->wait();
-    hardAssert($status->code === Grpc\STATUS_OK, 'Call did not complete successfully');
+    hardAssertIfStatusOk($status);
     hardAssert($result !== null, 'Call returned a null response');
     $payload = $result->getPayload();
     hardAssert($payload->getType() === grpc\testing\PayloadType::COMPRESSABLE,
@@ -197,7 +206,12 @@ function updateAuthMetadataCallback($context)
     $methodName = $context->method_name;
     $auth_credentials = ApplicationDefaultCredentials::getCredentials();
 
-    return $auth_credentials->updateMetadata($metadata = [], $authUri);
+    $metadata = [];
+    $result = $auth_credentials->updateMetadata([], $authUri);
+    foreach ($result as $key => $value) {
+      $metadata[strtolower($key)] = $value;
+    }
+    return $metadata;
 }
 
 /**
@@ -242,7 +256,7 @@ function clientStreaming($stub)
         $call->write($request);
     }
     list($result, $status) = $call->wait();
-    hardAssert($status->code === Grpc\STATUS_OK, 'Call did not complete successfully');
+    hardAssertIfStatusOk($status);
     hardAssert($result->getAggregatedPayloadSize() === 74922,
               'aggregated_payload_size was incorrect');
 }
@@ -275,8 +289,7 @@ function serverStreaming($stub)
                 'Response '.$i.' had the wrong length');
         $i += 1;
     }
-    hardAssert($call->getStatus()->code === Grpc\STATUS_OK,
-             'Call did not complete successfully');
+    hardAssertIfStatusOk($call->getStatus());
 }
 
 /**
@@ -312,8 +325,7 @@ function pingPong($stub)
     }
     $call->writesDone();
     hardAssert($call->read() === null, 'Server returned too many responses');
-    hardAssert($call->getStatus()->code === Grpc\STATUS_OK,
-              'Call did not complete successfully');
+    hardAssertIfStatusOk($call->getStatus());
 }
 
 /**
@@ -326,8 +338,7 @@ function emptyStream($stub)
     $call = $stub->FullDuplexCall();
     $call->writesDone();
     hardAssert($call->read() === null, 'Server returned too many responses');
-    hardAssert($call->getStatus()->code === Grpc\STATUS_OK,
-             'Call did not complete successfully');
+    hardAssertIfStatusOk($call->getStatus());
 }
 
 /**
@@ -419,8 +430,7 @@ function customMetadata($stub)
                'Incorrect initial metadata value');
 
     list($result, $status) = $call->wait();
-    hardAssert($status->code === Grpc\STATUS_OK,
-               'Call did not complete successfully');
+    hardAssertIfStatusOk($status);
 
     $trailing_metadata = $call->getTrailingMetadata();
     hardAssert(array_key_exists($ECHO_TRAILING_KEY, $trailing_metadata),
@@ -435,8 +445,7 @@ function customMetadata($stub)
     $streaming_call->write($streaming_request);
     $streaming_call->writesDone();
 
-    hardAssert($streaming_call->getStatus()->code === Grpc\STATUS_OK,
-               'Call did not complete successfully');
+    hardAssertIfStatusOk($streaming_call->getStatus());
 
     $streaming_trailing_metadata = $streaming_call->getTrailingMetadata();
     hardAssert(array_key_exists($ECHO_TRAILING_KEY,
diff --git a/src/php/tests/unit_tests/CallCredentials2Test.php b/src/php/tests/unit_tests/CallCredentials2Test.php
index a57e2b9b4e..b3b98a22ca 100644
--- a/src/php/tests/unit_tests/CallCredentials2Test.php
+++ b/src/php/tests/unit_tests/CallCredentials2Test.php
@@ -132,4 +132,69 @@ class CallCredentials2Test extends PHPUnit_Framework_TestCase
         unset($call);
         unset($server_call);
     }
+
+    public function invalidKeyCallbackFunc($context)
+    {
+        $this->assertTrue(is_string($context->service_url));
+        $this->assertTrue(is_string($context->method_name));
+
+        return ['K1' => ['v1']];
+    }
+
+    public function testCallbackWithInvalidKey()
+    {
+        $deadline = Grpc\Timeval::infFuture();
+        $status_text = 'xyz';
+        $call = new Grpc\Call($this->channel,
+                              '/abc/dummy_method',
+                              $deadline,
+                              $this->host_override);
+
+        $call_credentials = Grpc\CallCredentials::createFromPlugin(
+            array($this, 'invalidKeyCallbackFunc'));
+        $call->setCredentials($call_credentials);
+
+        $event = $call->startBatch([
+            Grpc\OP_SEND_INITIAL_METADATA => [],
+            Grpc\OP_SEND_CLOSE_FROM_CLIENT => true,
+            Grpc\OP_RECV_STATUS_ON_CLIENT => true,
+        ]);
+
+        $this->assertTrue($event->send_metadata);
+        $this->assertTrue($event->send_close);
+        $this->assertTrue($event->status->code == Grpc\STATUS_UNAUTHENTICATED);
+    }
+
+    public function invalidReturnCallbackFunc($context)
+    {
+        $this->assertTrue(is_string($context->service_url));
+        $this->assertTrue(is_string($context->method_name));
+
+        return "a string";
+    }
+
+    public function testCallbackWithInvalidReturnValue()
+    {
+        $deadline = Grpc\Timeval::infFuture();
+        $status_text = 'xyz';
+        $call = new Grpc\Call($this->channel,
+                              '/abc/dummy_method',
+                              $deadline,
+                              $this->host_override);
+
+        $call_credentials = Grpc\CallCredentials::createFromPlugin(
+            array($this, 'invalidReturnCallbackFunc'));
+        $call->setCredentials($call_credentials);
+
+        $event = $call->startBatch([
+            Grpc\OP_SEND_INITIAL_METADATA => [],
+            Grpc\OP_SEND_CLOSE_FROM_CLIENT => true,
+            Grpc\OP_RECV_STATUS_ON_CLIENT => true,
+        ]);
+
+        $this->assertTrue($event->send_metadata);
+        $this->assertTrue($event->send_close);
+        $this->assertTrue($event->status->code == Grpc\STATUS_UNAUTHENTICATED);
+    }
+
 }
diff --git a/src/php/tests/unit_tests/CallCredentials3Test.php b/src/php/tests/unit_tests/CallCredentials3Test.php
deleted file mode 100644
index 8f5e109bf5..0000000000
--- a/src/php/tests/unit_tests/CallCredentials3Test.php
+++ /dev/null
@@ -1,135 +0,0 @@
-<?php
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-class CallCredentials3Test extends PHPUnit_Framework_TestCase
-{
-    public function setUp()
-    {
-        $this->credentials = Grpc\ChannelCredentials::createSsl(
-            file_get_contents(dirname(__FILE__).'/../data/ca.pem'));
-        $server_credentials = Grpc\ServerCredentials::createSsl(
-            null,
-            file_get_contents(dirname(__FILE__).'/../data/server1.key'),
-            file_get_contents(dirname(__FILE__).'/../data/server1.pem'));
-        $this->server = new Grpc\Server();
-        $this->port = $this->server->addSecureHttp2Port('0.0.0.0:0',
-                                              $server_credentials);
-        $this->server->start();
-        $this->host_override = 'foo.test.google.fr';
-        $this->channel = new Grpc\Channel(
-            'localhost:'.$this->port,
-            [
-            'grpc.ssl_target_name_override' => $this->host_override,
-            'grpc.default_authority' => $this->host_override,
-            'credentials' => $this->credentials,
-            ]
-        );
-    }
-
-    public function tearDown()
-    {
-        unset($this->channel);
-        unset($this->server);
-    }
-
-    public function callbackFunc($context)
-    {
-        $this->assertTrue(is_string($context->service_url));
-        $this->assertTrue(is_string($context->method_name));
-
-        return ['k1' => ['v1'], 'k2' => ['v2']];
-    }
-
-    public function testCreateFromPlugin()
-    {
-        $deadline = Grpc\Timeval::infFuture();
-        $status_text = 'xyz';
-        $call = new Grpc\Call($this->channel,
-                              '/abc/dummy_method',
-                              $deadline,
-                              $this->host_override);
-
-        $call_credentials = Grpc\CallCredentials::createFromPlugin(
-            [$this, 'callbackFunc']);
-        $call->setCredentials($call_credentials);
-
-        $event = $call->startBatch([
-            Grpc\OP_SEND_INITIAL_METADATA => [],
-            Grpc\OP_SEND_CLOSE_FROM_CLIENT => true,
-        ]);
-
-        $this->assertTrue($event->send_metadata);
-        $this->assertTrue($event->send_close);
-
-        $event = $this->server->requestCall();
-
-        $this->assertTrue(is_array($event->metadata));
-        $metadata = $event->metadata;
-        $this->assertTrue(array_key_exists('k1', $metadata));
-        $this->assertTrue(array_key_exists('k2', $metadata));
-        $this->assertSame($metadata['k1'], ['v1']);
-        $this->assertSame($metadata['k2'], ['v2']);
-
-        $this->assertSame('/abc/dummy_method', $event->method);
-        $server_call = $event->call;
-
-        $event = $server_call->startBatch([
-            Grpc\OP_SEND_INITIAL_METADATA => [],
-            Grpc\OP_SEND_STATUS_FROM_SERVER => [
-                'metadata' => [],
-                'code' => Grpc\STATUS_OK,
-                'details' => $status_text,
-            ],
-            Grpc\OP_RECV_CLOSE_ON_SERVER => true,
-        ]);
-
-        $this->assertTrue($event->send_metadata);
-        $this->assertTrue($event->send_status);
-        $this->assertFalse($event->cancelled);
-
-        $event = $call->startBatch([
-            Grpc\OP_RECV_INITIAL_METADATA => true,
-            Grpc\OP_RECV_STATUS_ON_CLIENT => true,
-        ]);
-
-        $this->assertSame([], $event->metadata);
-        $status = $event->status;
-        $this->assertSame([], $status->metadata);
-        $this->assertSame(Grpc\STATUS_OK, $status->code);
-        $this->assertSame($status_text, $status->details);
-
-        unset($call);
-        unset($server_call);
-    }
-}
diff --git a/src/php/tests/unit_tests/CallTest.php b/src/php/tests/unit_tests/CallTest.php
index d736f51546..1205f0cd8e 100644
--- a/src/php/tests/unit_tests/CallTest.php
+++ b/src/php/tests/unit_tests/CallTest.php
@@ -113,7 +113,7 @@ class CallTest extends PHPUnit_Framework_TestCase
     /**
      * @expectedException InvalidArgumentException
      */
-    public function testInvalidMetadataKey()
+    public function testInvalidStartBatchKey()
     {
         $batch = [
             'invalid' => ['key1' => 'value1'],
@@ -124,6 +124,28 @@ class CallTest extends PHPUnit_Framework_TestCase
     /**
      * @expectedException InvalidArgumentException
      */
+    public function testInvalidMetadataStrKey()
+    {
+        $batch = [
+            Grpc\OP_SEND_INITIAL_METADATA => ['Key' => ['value1', 'value2']],
+        ];
+        $result = $this->call->startBatch($batch);
+    }
+
+    /**
+     * @expectedException InvalidArgumentException
+     */
+    public function testInvalidMetadataIntKey()
+    {
+        $batch = [
+            Grpc\OP_SEND_INITIAL_METADATA => [1 => ['value1', 'value2']],
+        ];
+        $result = $this->call->startBatch($batch);
+    }
+
+    /**
+     * @expectedException InvalidArgumentException
+     */
     public function testInvalidMetadataInnerValue()
     {
         $batch = [
diff --git a/src/proto/census/trace_context.options b/src/proto/census/trace_context.options
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/proto/census/trace_context.options
diff --git a/src/proto/census/trace_context.proto b/src/proto/census/trace_context.proto
new file mode 100644
index 0000000000..a5d5a9595d
--- /dev/null
+++ b/src/proto/census/trace_context.proto
@@ -0,0 +1,48 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package google.trace;
+
+// A TraceId uniquely represents a single Trace. It is a 128-bit nonce.
+message TraceId {
+  fixed64 hi = 1;
+  fixed64 lo = 2;
+}
+
+// Tracing information that is propagated with RPC's.
+message TraceContext {
+  // Trace identifer. Must be present.
+  TraceId trace_id = 1;
+  // ID of parent (client) span. Must be present.
+  fixed64 span_id = 2;
+  // true if this trace is sampled.
+  bool is_sampled = 3;
+}
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index 7ae76f52c1..725134c3a4 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -220,7 +220,6 @@ CORE_SOURCE_FILES = [
   'src/core/ext/client_config/channel_connectivity.c',
   'src/core/ext/client_config/client_channel.c',
   'src/core/ext/client_config/client_channel_factory.c',
-  'src/core/ext/client_config/client_config.c',
   'src/core/ext/client_config/client_config_plugin.c',
   'src/core/ext/client_config/connector.c',
   'src/core/ext/client_config/default_initial_connect_string.c',
@@ -232,8 +231,8 @@ CORE_SOURCE_FILES = [
   'src/core/ext/client_config/resolver.c',
   'src/core/ext/client_config/resolver_factory.c',
   'src/core/ext/client_config/resolver_registry.c',
+  'src/core/ext/client_config/resolver_result.c',
   'src/core/ext/client_config/subchannel.c',
-  'src/core/ext/client_config/subchannel_call_holder.c',
   'src/core/ext/client_config/subchannel_index.c',
   'src/core/ext/client_config/uri_parser.c',
   'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
@@ -255,6 +254,7 @@ CORE_SOURCE_FILES = [
   'src/core/ext/census/base_resources.c',
   'src/core/ext/census/context.c',
   'src/core/ext/census/gen/census.pb.c',
+  'src/core/ext/census/gen/trace_context.pb.c',
   'src/core/ext/census/grpc_context.c',
   'src/core/ext/census/grpc_filter.c',
   'src/core/ext/census/grpc_plugin.c',
diff --git a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py
index 2f50263730..142387d810 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py
@@ -114,9 +114,6 @@ class TypeSmokeTest(unittest.TestCase):
         lambda ignored_a, ignored_b: None, b'')
     del plugin
 
-  @unittest.skipIf(
-    platform.python_implementation() == "PyPy",
-    'TODO(issue 7672): figure out why this fails on PyPy')
   def testCallCredentialsFromPluginUpDown(self):
     plugin = cygrpc.CredentialsMetadataPlugin(_metadata_plugin_callback, b'')
     call_credentials = cygrpc.call_credentials_metadata_plugin(plugin)
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
index aee57b11aa..9caaf7b783 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
@@ -80,6 +80,7 @@ grpc_register_plugin_type grpc_register_plugin_import;
 grpc_init_type grpc_init_import;
 grpc_shutdown_type grpc_shutdown_import;
 grpc_version_string_type grpc_version_string_import;
+grpc_g_stands_for_type grpc_g_stands_for_import;
 grpc_completion_queue_create_type grpc_completion_queue_create_import;
 grpc_completion_queue_next_type grpc_completion_queue_next_import;
 grpc_completion_queue_pluck_type grpc_completion_queue_pluck_import;
@@ -348,6 +349,7 @@ void grpc_rb_load_imports(HMODULE library) {
   grpc_init_import = (grpc_init_type) GetProcAddress(library, "grpc_init");
   grpc_shutdown_import = (grpc_shutdown_type) GetProcAddress(library, "grpc_shutdown");
   grpc_version_string_import = (grpc_version_string_type) GetProcAddress(library, "grpc_version_string");
+  grpc_g_stands_for_import = (grpc_g_stands_for_type) GetProcAddress(library, "grpc_g_stands_for");
   grpc_completion_queue_create_import = (grpc_completion_queue_create_type) GetProcAddress(library, "grpc_completion_queue_create");
   grpc_completion_queue_next_import = (grpc_completion_queue_next_type) GetProcAddress(library, "grpc_completion_queue_next");
   grpc_completion_queue_pluck_import = (grpc_completion_queue_pluck_type) GetProcAddress(library, "grpc_completion_queue_pluck");
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
index 3bb76fbb97..a2f5b86497 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -191,6 +191,9 @@ extern grpc_shutdown_type grpc_shutdown_import;
 typedef const char *(*grpc_version_string_type)(void);
 extern grpc_version_string_type grpc_version_string_import;
 #define grpc_version_string grpc_version_string_import
+typedef const char *(*grpc_g_stands_for_type)(void);
+extern grpc_g_stands_for_type grpc_g_stands_for_import;
+#define grpc_g_stands_for grpc_g_stands_for_import
 typedef grpc_completion_queue *(*grpc_completion_queue_create_type)(void *reserved);
 extern grpc_completion_queue_create_type grpc_completion_queue_create_import;
 #define grpc_completion_queue_create grpc_completion_queue_create_import