Merge branch 'who-combines-the-combiners' into direct-calls

29 files changed, 407 insertions, 584 deletions

diff --git a/BUILD b/BUILD
index 4adc2b4d1d..61cb622b1b 100644
--- a/BUILD
+++ b/BUILD
@@ -302,7 +302,6 @@ cc_library(
     "src/core/ext/client_config/resolver_registry.h",
     "src/core/ext/client_config/resolver_result.h",
     "src/core/ext/client_config/subchannel.h",
-    "src/core/ext/client_config/subchannel_call_holder.h",
     "src/core/ext/client_config/subchannel_index.h",
     "src/core/ext/client_config/uri_parser.h",
     "src/core/ext/lb_policy/grpclb/grpclb.h",
@@ -479,7 +478,6 @@ cc_library(
     "src/core/ext/client_config/resolver_registry.c",
     "src/core/ext/client_config/resolver_result.c",
     "src/core/ext/client_config/subchannel.c",
-    "src/core/ext/client_config/subchannel_call_holder.c",
     "src/core/ext/client_config/subchannel_index.c",
     "src/core/ext/client_config/uri_parser.c",
     "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c",
@@ -678,7 +676,6 @@ cc_library(
     "src/core/ext/client_config/resolver_registry.h",
     "src/core/ext/client_config/resolver_result.h",
     "src/core/ext/client_config/subchannel.h",
-    "src/core/ext/client_config/subchannel_call_holder.h",
     "src/core/ext/client_config/subchannel_index.h",
     "src/core/ext/client_config/uri_parser.h",
     "src/core/lib/security/context/security_context.h",
@@ -838,7 +835,6 @@ cc_library(
     "src/core/ext/client_config/resolver_registry.c",
     "src/core/ext/client_config/resolver_result.c",
     "src/core/ext/client_config/subchannel.c",
-    "src/core/ext/client_config/subchannel_call_holder.c",
     "src/core/ext/client_config/subchannel_index.c",
     "src/core/ext/client_config/uri_parser.c",
     "src/core/lib/http/httpcli_security_connector.c",
@@ -1033,7 +1029,6 @@ cc_library(
     "src/core/ext/client_config/resolver_registry.h",
     "src/core/ext/client_config/resolver_result.h",
     "src/core/ext/client_config/subchannel.h",
-    "src/core/ext/client_config/subchannel_call_holder.h",
     "src/core/ext/client_config/subchannel_index.h",
     "src/core/ext/client_config/uri_parser.h",
     "src/core/ext/load_reporting/load_reporting.h",
@@ -1185,7 +1180,6 @@ cc_library(
     "src/core/ext/client_config/resolver_registry.c",
     "src/core/ext/client_config/resolver_result.c",
     "src/core/ext/client_config/subchannel.c",
-    "src/core/ext/client_config/subchannel_call_holder.c",
     "src/core/ext/client_config/subchannel_index.c",
     "src/core/ext/client_config/uri_parser.c",
     "src/core/ext/resolver/dns/native/dns_resolver.c",
@@ -2335,7 +2329,6 @@ objc_library(
     "src/core/ext/client_config/resolver_registry.c",
     "src/core/ext/client_config/resolver_result.c",
     "src/core/ext/client_config/subchannel.c",
-    "src/core/ext/client_config/subchannel_call_holder.c",
     "src/core/ext/client_config/subchannel_index.c",
     "src/core/ext/client_config/uri_parser.c",
     "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c",
@@ -2536,7 +2529,6 @@ objc_library(
     "src/core/ext/client_config/resolver_registry.h",
     "src/core/ext/client_config/resolver_result.h",
     "src/core/ext/client_config/subchannel.h",
-    "src/core/ext/client_config/subchannel_call_holder.h",
     "src/core/ext/client_config/subchannel_index.h",
     "src/core/ext/client_config/uri_parser.h",
     "src/core/ext/lb_policy/grpclb/grpclb.h",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8b83334b51..a0d174a1fc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -450,7 +450,6 @@ add_library(grpc
   src/core/ext/client_config/resolver_registry.c
   src/core/ext/client_config/resolver_result.c
   src/core/ext/client_config/subchannel.c
-  src/core/ext/client_config/subchannel_call_holder.c
   src/core/ext/client_config/subchannel_index.c
   src/core/ext/client_config/uri_parser.c
   src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -684,7 +683,6 @@ add_library(grpc_cronet
   src/core/ext/client_config/resolver_registry.c
   src/core/ext/client_config/resolver_result.c
   src/core/ext/client_config/subchannel.c
-  src/core/ext/client_config/subchannel_call_holder.c
   src/core/ext/client_config/subchannel_index.c
   src/core/ext/client_config/uri_parser.c
   src/core/lib/http/httpcli_security_connector.c
@@ -917,7 +915,6 @@ add_library(grpc_unsecure
   src/core/ext/client_config/resolver_registry.c
   src/core/ext/client_config/resolver_result.c
   src/core/ext/client_config/subchannel.c
-  src/core/ext/client_config/subchannel_call_holder.c
   src/core/ext/client_config/subchannel_index.c
   src/core/ext/client_config/uri_parser.c
   src/core/ext/resolver/dns/native/dns_resolver.c
diff --git a/Makefile b/Makefile
index 55f12da927..a8e71984ed 100644
--- a/Makefile
+++ b/Makefile
@@ -2674,7 +2674,6 @@ LIBGRPC_SRC = \
     src/core/ext/client_config/resolver_registry.c \
     src/core/ext/client_config/resolver_result.c \
     src/core/ext/client_config/subchannel.c \
-    src/core/ext/client_config/subchannel_call_holder.c \
     src/core/ext/client_config/subchannel_index.c \
     src/core/ext/client_config/uri_parser.c \
     src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \
@@ -2926,7 +2925,6 @@ LIBGRPC_CRONET_SRC = \
     src/core/ext/client_config/resolver_registry.c \
     src/core/ext/client_config/resolver_result.c \
     src/core/ext/client_config/subchannel.c \
-    src/core/ext/client_config/subchannel_call_holder.c \
     src/core/ext/client_config/subchannel_index.c \
     src/core/ext/client_config/uri_parser.c \
     src/core/lib/http/httpcli_security_connector.c \
@@ -3387,7 +3385,6 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/ext/client_config/resolver_registry.c \
     src/core/ext/client_config/resolver_result.c \
     src/core/ext/client_config/subchannel.c \
-    src/core/ext/client_config/subchannel_call_holder.c \
     src/core/ext/client_config/subchannel_index.c \
     src/core/ext/client_config/uri_parser.c \
     src/core/ext/resolver/dns/native/dns_resolver.c \
diff --git a/binding.gyp b/binding.gyp
index 16d24802bc..67bf1cb260 100644
--- a/binding.gyp
+++ b/binding.gyp
@@ -722,7 +722,6 @@
         'src/core/ext/client_config/resolver_registry.c',
         'src/core/ext/client_config/resolver_result.c',
         'src/core/ext/client_config/subchannel.c',
-        'src/core/ext/client_config/subchannel_call_holder.c',
         'src/core/ext/client_config/subchannel_index.c',
         'src/core/ext/client_config/uri_parser.c',
         'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
diff --git a/build.yaml b/build.yaml
index 6852dca6c8..7484b5db06 100644
--- a/build.yaml
+++ b/build.yaml
@@ -356,7 +356,6 @@ filegroups:
   - src/core/ext/client_config/resolver_registry.h
   - src/core/ext/client_config/resolver_result.h
   - src/core/ext/client_config/subchannel.h
-  - src/core/ext/client_config/subchannel_call_holder.h
   - src/core/ext/client_config/subchannel_index.h
   - src/core/ext/client_config/uri_parser.h
   src:
@@ -376,7 +375,6 @@ filegroups:
   - src/core/ext/client_config/resolver_registry.c
   - src/core/ext/client_config/resolver_result.c
   - src/core/ext/client_config/subchannel.c
-  - src/core/ext/client_config/subchannel_call_holder.c
   - src/core/ext/client_config/subchannel_index.c
   - src/core/ext/client_config/uri_parser.c
   plugin: grpc_client_config
diff --git a/config.m4 b/config.m4
index 39df2dd3f4..ab3c69d9c8 100644
--- a/config.m4
+++ b/config.m4
@@ -241,7 +241,6 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/client_config/resolver_registry.c \
     src/core/ext/client_config/resolver_result.c \
     src/core/ext/client_config/subchannel.c \
-    src/core/ext/client_config/subchannel_call_holder.c \
     src/core/ext/client_config/subchannel_index.c \
     src/core/ext/client_config/uri_parser.c \
     src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index c84b058a11..1fc11709ae 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -393,7 +393,6 @@ Pod::Spec.new do |s|
                       'src/core/ext/client_config/resolver_registry.h',
                       'src/core/ext/client_config/resolver_result.h',
                       'src/core/ext/client_config/subchannel.h',
-                      'src/core/ext/client_config/subchannel_call_holder.h',
                       'src/core/ext/client_config/subchannel_index.h',
                       'src/core/ext/client_config/uri_parser.h',
                       'src/core/ext/lb_policy/grpclb/grpclb.h',
@@ -574,7 +573,6 @@ Pod::Spec.new do |s|
                       'src/core/ext/client_config/resolver_registry.c',
                       'src/core/ext/client_config/resolver_result.c',
                       'src/core/ext/client_config/subchannel.c',
-                      'src/core/ext/client_config/subchannel_call_holder.c',
                       'src/core/ext/client_config/subchannel_index.c',
                       'src/core/ext/client_config/uri_parser.c',
                       'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
@@ -759,7 +757,6 @@ Pod::Spec.new do |s|
                               'src/core/ext/client_config/resolver_registry.h',
                               'src/core/ext/client_config/resolver_result.h',
                               'src/core/ext/client_config/subchannel.h',
-                              'src/core/ext/client_config/subchannel_call_holder.h',
                               'src/core/ext/client_config/subchannel_index.h',
                               'src/core/ext/client_config/uri_parser.h',
                               'src/core/ext/lb_policy/grpclb/grpclb.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index f57b7dca62..3a023c282d 100755
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -312,7 +312,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/client_config/resolver_registry.h )
   s.files += %w( src/core/ext/client_config/resolver_result.h )
   s.files += %w( src/core/ext/client_config/subchannel.h )
-  s.files += %w( src/core/ext/client_config/subchannel_call_holder.h )
   s.files += %w( src/core/ext/client_config/subchannel_index.h )
   s.files += %w( src/core/ext/client_config/uri_parser.h )
   s.files += %w( src/core/ext/lb_policy/grpclb/grpclb.h )
@@ -493,7 +492,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/client_config/resolver_registry.c )
   s.files += %w( src/core/ext/client_config/resolver_result.c )
   s.files += %w( src/core/ext/client_config/subchannel.c )
-  s.files += %w( src/core/ext/client_config/subchannel_call_holder.c )
   s.files += %w( src/core/ext/client_config/subchannel_index.c )
   s.files += %w( src/core/ext/client_config/uri_parser.c )
   s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2.c )
diff --git a/include/grpc++/grpc++.h b/include/grpc++/grpc++.h
index ef07e19975..afb1c555bb 100644
--- a/include/grpc++/grpc++.h
+++ b/include/grpc++/grpc++.h
@@ -51,6 +51,9 @@
 #ifndef GRPCXX_GRPCXX_H
 #define GRPCXX_GRPCXX_H
 
+// Pragma for http://include-what-you-use.org/ tool, tells that following
+// headers are not private for grpc++.h and are part of its interface.
+// IWYU pragma: begin_exports
 #include <grpc/grpc.h>
 
 #include <grpc++/channel.h>
@@ -62,5 +65,6 @@
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
 #include <grpc++/server_posix.h>
+// IWYU pragma: end_exports
 
 #endif  // GRPCXX_GRPCXX_H
diff --git a/include/grpc++/impl/server_builder_plugin.h b/include/grpc++/impl/server_builder_plugin.h
index 1e157efa11..61632e32fa 100644
--- a/include/grpc++/impl/server_builder_plugin.h
+++ b/include/grpc++/impl/server_builder_plugin.h
@@ -41,6 +41,7 @@
 namespace grpc {
 
 class ServerInitializer;
+class ChannelArguments;
 
 class ServerBuilderPlugin {
  public:
@@ -58,6 +59,10 @@ class ServerBuilderPlugin {
   // ServerBuilderOption::UpdatePlugins
   virtual void ChangeArguments(const grpc::string& name, void* value) = 0;
 
+  // UpdateChannelArguments will be called in ServerBuilder::BuildAndStart(),
+  // before the Server instance is created.
+  virtual void UpdateChannelArguments(ChannelArguments* args) {}
+
   virtual bool has_sync_methods() const { return false; }
   virtual bool has_async_methods() const { return false; }
 };
diff --git a/package.xml b/package.xml
index 5bc3010aa2..fed30e9605 100644
--- a/package.xml
+++ b/package.xml
@@ -320,7 +320,6 @@
     <file baseinstalldir="/" name="src/core/ext/client_config/resolver_registry.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/resolver_result.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/subchannel.h" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/client_config/subchannel_call_holder.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/subchannel_index.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/uri_parser.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/lb_policy/grpclb/grpclb.h" role="src" />
@@ -501,7 +500,6 @@
     <file baseinstalldir="/" name="src/core/ext/client_config/resolver_registry.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/resolver_result.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/subchannel.c" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/client_config/subchannel_call_holder.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/subchannel_index.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_config/uri_parser.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/server/insecure/server_chttp2.c" role="src" />
diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c
index 566d3d5ce4..be333f4e0d 100644
--- a/src/core/ext/client_config/client_channel.c
+++ b/src/core/ext/client_config/client_channel.c
@@ -33,6 +33,7 @@
 
 #include "src/core/ext/client_config/client_channel.h"
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -41,10 +42,11 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/ext/client_config/subchannel_call_holder.h"
+#include "src/core/ext/client_config/subchannel.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/channel.h"
@@ -52,13 +54,15 @@
 
 /* Client channel implementation */
 
-typedef grpc_subchannel_call_holder call_data;
+/*************************************************************************
+ * CHANNEL-WIDE FUNCTIONS
+ */
 
 typedef struct client_channel_channel_data {
   /** resolver for this channel */
   grpc_resolver *resolver;
   /** have we started resolving this channel */
-  int started_resolving;
+  bool started_resolving;
 
   /** mutex protecting client configuration, including all
       variables below in this data structure */
@@ -74,7 +78,7 @@ typedef struct client_channel_channel_data {
   /** connectivity state being tracked */
   grpc_connectivity_state_tracker state_tracker;
   /** when an lb_policy arrives, should we try to exit idle */
-  int exit_idle_when_lb_policy_arrives;
+  bool exit_idle_when_lb_policy_arrives;
   /** owning stack */
   grpc_channel_stack *owning_stack;
   /** interested parties (owned) */
@@ -82,10 +86,8 @@ typedef struct client_channel_channel_data {
 } channel_data;
 
 /** We create one watcher for each new lb_policy that is returned from a
-   resolver,
-    to watch for state changes from the lb_policy. When a state change is seen,
-   we
-    update the channel, and create a new watcher */
+    resolver, to watch for state changes from the lb_policy. When a state
+    change is seen, we update the channel, and create a new watcher. */
 typedef struct {
   channel_data *chand;
   grpc_closure on_changed;
@@ -93,22 +95,6 @@ typedef struct {
   grpc_lb_policy *lb_policy;
 } lb_policy_connectivity_watcher;
 
-typedef struct {
-  grpc_closure closure;
-  grpc_call_element *elem;
-} waiting_call;
-
-static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data);
-}
-
-static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
-                                         grpc_call_element *elem,
-                                         grpc_transport_stream_op *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-  grpc_subchannel_call_holder_perform_op(exec_ctx, elem->call_data, op);
-}
-
 static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
                             grpc_lb_policy *lb_policy,
                             grpc_connectivity_state current_state);
@@ -177,13 +163,13 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
                                         &w->on_changed);
 }
 
-static void cc_on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                          grpc_error *error) {
+static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg,
+                                       grpc_error *error) {
   channel_data *chand = arg;
   grpc_lb_policy *lb_policy = NULL;
   grpc_lb_policy *old_lb_policy;
   grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE;
-  int exit_idle = 0;
+  bool exit_idle = false;
   grpc_error *state_error = GRPC_ERROR_CREATE("No load balancing policy");
 
   if (chand->resolver_result != NULL) {
@@ -221,8 +207,8 @@ static void cc_on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg,
   }
   if (lb_policy != NULL && chand->exit_idle_when_lb_policy_arrives) {
     GRPC_LB_POLICY_REF(lb_policy, "exit_idle");
-    exit_idle = 1;
-    chand->exit_idle_when_lb_policy_arrives = 0;
+    exit_idle = true;
+    chand->exit_idle_when_lb_policy_arrives = false;
   }
 
   if (error == GRPC_ERROR_NONE && chand->resolver) {
@@ -330,6 +316,192 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
   gpr_mu_unlock(&chand->mu);
 }
 
+/* Constructor for channel_data */
+static void cc_init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_element *elem,
+                                 grpc_channel_element_args *args) {
+  channel_data *chand = elem->channel_data;
+
+  memset(chand, 0, sizeof(*chand));
+
+  GPR_ASSERT(args->is_last);
+  GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
+
+  gpr_mu_init(&chand->mu);
+  grpc_closure_init(&chand->on_resolver_result_changed,
+                    on_resolver_result_changed, chand);
+  chand->owning_stack = args->channel_stack;
+
+  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
+                               "client_channel");
+  chand->interested_parties = grpc_pollset_set_create();
+}
+
+/* Destructor for channel_data */
+static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx,
+                                    grpc_channel_element *elem) {
+  channel_data *chand = elem->channel_data;
+
+  if (chand->resolver != NULL) {
+    grpc_resolver_shutdown(exec_ctx, chand->resolver);
+    GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
+  }
+  if (chand->lb_policy != NULL) {
+    grpc_pollset_set_del_pollset_set(exec_ctx,
+                                     chand->lb_policy->interested_parties,
+                                     chand->interested_parties);
+    GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
+  }
+  grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
+  grpc_pollset_set_destroy(chand->interested_parties);
+  gpr_mu_destroy(&chand->mu);
+}
+
+/*************************************************************************
+ * PER-CALL FUNCTIONS
+ */
+
+#define GET_CALL(call_data) \
+  ((grpc_subchannel_call *)(gpr_atm_acq_load(&(call_data)->subchannel_call)))
+
+#define CANCELLED_CALL ((grpc_subchannel_call *)1)
+
+typedef enum {
+  GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING,
+  GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL
+} subchannel_creation_phase;
+
+/** Call data.  Holds a pointer to grpc_subchannel_call and the
+    associated machinery to create such a pointer.
+    Handles queueing of stream ops until a call object is ready, waiting
+    for initial metadata before trying to create a call object,
+    and handling cancellation gracefully. */
+typedef struct client_channel_call_data {
+  /** either 0 for no call, 1 for cancelled, or a pointer to a
+      grpc_subchannel_call */
+  gpr_atm subchannel_call;
+
+  gpr_mu mu;
+
+  subchannel_creation_phase creation_phase;
+  grpc_connected_subchannel *connected_subchannel;
+  grpc_polling_entity *pollent;
+
+  grpc_transport_stream_op **waiting_ops;
+  size_t waiting_ops_count;
+  size_t waiting_ops_capacity;
+
+  grpc_closure next_step;
+
+  grpc_call_stack *owning_call;
+} call_data;
+
+static void add_waiting_locked(call_data *calld, grpc_transport_stream_op *op) {
+  GPR_TIMER_BEGIN("add_waiting_locked", 0);
+  if (calld->waiting_ops_count == calld->waiting_ops_capacity) {
+    calld->waiting_ops_capacity = GPR_MAX(3, 2 * calld->waiting_ops_capacity);
+    calld->waiting_ops =
+        gpr_realloc(calld->waiting_ops,
+                    calld->waiting_ops_capacity * sizeof(*calld->waiting_ops));
+  }
+  calld->waiting_ops[calld->waiting_ops_count++] = op;
+  GPR_TIMER_END("add_waiting_locked", 0);
+}
+
+static void fail_locked(grpc_exec_ctx *exec_ctx, call_data *calld,
+                        grpc_error *error) {
+  size_t i;
+  for (i = 0; i < calld->waiting_ops_count; i++) {
+    grpc_transport_stream_op_finish_with_failure(
+        exec_ctx, calld->waiting_ops[i], GRPC_ERROR_REF(error));
+  }
+  calld->waiting_ops_count = 0;
+  GRPC_ERROR_UNREF(error);
+}
+
+typedef struct {
+  grpc_transport_stream_op **ops;
+  size_t nops;
+  grpc_subchannel_call *call;
+} retry_ops_args;
+
+static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
+  retry_ops_args *a = args;
+  size_t i;
+  for (i = 0; i < a->nops; i++) {
+    grpc_subchannel_call_process_op(exec_ctx, a->call, a->ops[i]);
+  }
+  GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops");
+  gpr_free(a->ops);
+  gpr_free(a);
+}
+
+static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) {
+  if (calld->waiting_ops_count == 0) {
+    return;
+  }
+
+  retry_ops_args *a = gpr_malloc(sizeof(*a));
+  a->ops = calld->waiting_ops;
+  a->nops = calld->waiting_ops_count;
+  a->call = GET_CALL(calld);
+  if (a->call == CANCELLED_CALL) {
+    gpr_free(a);
+    fail_locked(exec_ctx, calld, GRPC_ERROR_CANCELLED);
+    return;
+  }
+  calld->waiting_ops = NULL;
+  calld->waiting_ops_count = 0;
+  calld->waiting_ops_capacity = 0;
+  GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
+  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
+                      GRPC_ERROR_NONE, NULL);
+}
+
+static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
+                             grpc_error *error) {
+  call_data *calld = arg;
+  gpr_mu_lock(&calld->mu);
+  GPR_ASSERT(calld->creation_phase ==
+             GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL);
+  calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  if (calld->connected_subchannel == NULL) {
+    gpr_atm_no_barrier_store(&calld->subchannel_call, 1);
+    fail_locked(exec_ctx, calld, GRPC_ERROR_CREATE_REFERENCING(
+                                     "Failed to create subchannel", &error, 1));
+  } else if (1 == gpr_atm_acq_load(&calld->subchannel_call)) {
+    /* already cancelled before subchannel became ready */
+    fail_locked(exec_ctx, calld,
+                GRPC_ERROR_CREATE_REFERENCING(
+                    "Cancelled before creating subchannel", &error, 1));
+  } else {
+    grpc_subchannel_call *subchannel_call = NULL;
+    grpc_error *new_error = grpc_connected_subchannel_create_call(
+        exec_ctx, calld->connected_subchannel, calld->pollent,
+        &subchannel_call);
+    if (new_error != GRPC_ERROR_NONE) {
+      new_error = grpc_error_add_child(new_error, error);
+      subchannel_call = CANCELLED_CALL;
+      fail_locked(exec_ctx, calld, new_error);
+    }
+    gpr_atm_rel_store(&calld->subchannel_call,
+                      (gpr_atm)(uintptr_t)subchannel_call);
+    retry_waiting_locked(exec_ctx, calld);
+  }
+  gpr_mu_unlock(&calld->mu);
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
+}
+
+static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
+  grpc_subchannel_call *subchannel_call = GET_CALL(calld);
+  if (subchannel_call == NULL || subchannel_call == CANCELLED_CALL) {
+    return NULL;
+  } else {
+    return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
+  }
+}
+
 typedef struct {
   grpc_metadata_batch *initial_metadata;
   uint32_t initial_metadata_flags;
@@ -339,11 +511,11 @@ typedef struct {
   grpc_closure closure;
 } continue_picking_args;
 
-static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
-                              grpc_metadata_batch *initial_metadata,
-                              uint32_t initial_metadata_flags,
-                              grpc_connected_subchannel **connected_subchannel,
-                              grpc_closure *on_ready);
+static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                            grpc_metadata_batch *initial_metadata,
+                            uint32_t initial_metadata_flags,
+                            grpc_connected_subchannel **connected_subchannel,
+                            grpc_closure *on_ready);
 
 static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
                              grpc_error *error) {
@@ -352,22 +524,21 @@ static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
     /* cancelled, do nothing */
   } else if (error != GRPC_ERROR_NONE) {
     grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error), NULL);
-  } else if (cc_pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
-                                cpa->initial_metadata_flags,
-                                cpa->connected_subchannel, cpa->on_ready)) {
+  } else if (pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
+                             cpa->initial_metadata_flags,
+                             cpa->connected_subchannel, cpa->on_ready)) {
     grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE, NULL);
   }
   gpr_free(cpa);
 }
 
-static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
-                              grpc_metadata_batch *initial_metadata,
-                              uint32_t initial_metadata_flags,
-                              grpc_connected_subchannel **connected_subchannel,
-                              grpc_closure *on_ready) {
-  GPR_TIMER_BEGIN("cc_pick_subchannel", 0);
+static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                            grpc_metadata_batch *initial_metadata,
+                            uint32_t initial_metadata_flags,
+                            grpc_connected_subchannel **connected_subchannel,
+                            grpc_closure *on_ready) {
+  GPR_TIMER_BEGIN("pick_subchannel", 0);
 
-  grpc_call_element *elem = elemp;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   continue_picking_args *cpa;
@@ -391,23 +562,23 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
       }
     }
     gpr_mu_unlock(&chand->mu);
-    GPR_TIMER_END("cc_pick_subchannel", 0);
-    return 1;
+    GPR_TIMER_END("pick_subchannel", 0);
+    return true;
   }
   if (chand->lb_policy != NULL) {
     grpc_lb_policy *lb_policy = chand->lb_policy;
     int r;
-    GRPC_LB_POLICY_REF(lb_policy, "cc_pick_subchannel");
+    GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel");
     gpr_mu_unlock(&chand->mu);
     r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollent,
                             initial_metadata, initial_metadata_flags,
                             connected_subchannel, on_ready);
-    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "cc_pick_subchannel");
-    GPR_TIMER_END("cc_pick_subchannel", 0);
+    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick_subchannel");
+    GPR_TIMER_END("pick_subchannel", 0);
     return r;
   }
   if (chand->resolver != NULL && !chand->started_resolving) {
-    chand->started_resolving = 1;
+    chand->started_resolving = true;
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
     grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result,
                        &chand->on_resolver_result_changed);
@@ -428,66 +599,143 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
   }
   gpr_mu_unlock(&chand->mu);
 
-  GPR_TIMER_END("cc_pick_subchannel", 0);
-  return 0;
+  GPR_TIMER_END("pick_subchannel", 0);
+  return false;
+}
+
+// The logic here is fairly complicated, due to (a) the fact that we
+// need to handle the case where we receive the send op before the
+// initial metadata op, and (b) the need for efficiency, especially in
+// the streaming case.
+// TODO(ctiller): Explain this more thoroughly.
+static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
+                                         grpc_call_element *elem,
+                                         grpc_transport_stream_op *op) {
+  call_data *calld = elem->call_data;
+  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
+  /* try to (atomically) get the call */
+  grpc_subchannel_call *call = GET_CALL(calld);
+  GPR_TIMER_BEGIN("cc_start_transport_stream_op", 0);
+  if (call == CANCELLED_CALL) {
+    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                 GRPC_ERROR_CANCELLED);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  if (call != NULL) {
+    grpc_subchannel_call_process_op(exec_ctx, call, op);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  /* we failed; lock and figure out what to do */
+  gpr_mu_lock(&calld->mu);
+retry:
+  /* need to recheck that another thread hasn't set the call */
+  call = GET_CALL(calld);
+  if (call == CANCELLED_CALL) {
+    gpr_mu_unlock(&calld->mu);
+    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                 GRPC_ERROR_CANCELLED);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  if (call != NULL) {
+    gpr_mu_unlock(&calld->mu);
+    grpc_subchannel_call_process_op(exec_ctx, call, op);
+    GPR_TIMER_END("cc_start_transport_stream_op", 0);
+    return;
+  }
+  /* if this is a cancellation, then we can raise our cancelled flag */
+  if (op->cancel_error != GRPC_ERROR_NONE) {
+    if (!gpr_atm_rel_cas(&calld->subchannel_call, 0,
+                         (gpr_atm)(uintptr_t)CANCELLED_CALL)) {
+      goto retry;
+    } else {
+      switch (calld->creation_phase) {
+        case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING:
+          fail_locked(exec_ctx, calld, GRPC_ERROR_REF(op->cancel_error));
+          break;
+        case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL:
+          pick_subchannel(exec_ctx, elem, NULL, 0, &calld->connected_subchannel,
+                          NULL);
+          break;
+      }
+      gpr_mu_unlock(&calld->mu);
+      grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                   GRPC_ERROR_CANCELLED);
+      GPR_TIMER_END("cc_start_transport_stream_op", 0);
+      return;
+    }
+  }
+  /* if we don't have a subchannel, try to get one */
+  if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
+      calld->connected_subchannel == NULL &&
+      op->send_initial_metadata != NULL) {
+    calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL;
+    grpc_closure_init(&calld->next_step, subchannel_ready, calld);
+    GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel");
+    if (pick_subchannel(exec_ctx, elem, op->send_initial_metadata,
+                        op->send_initial_metadata_flags,
+                        &calld->connected_subchannel, &calld->next_step)) {
+      calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+      GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
+    }
+  }
+  /* if we've got a subchannel, then let's ask it to create a call */
+  if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
+      calld->connected_subchannel != NULL) {
+    grpc_subchannel_call *subchannel_call = NULL;
+    grpc_error *error = grpc_connected_subchannel_create_call(
+        exec_ctx, calld->connected_subchannel, calld->pollent,
+        &subchannel_call);
+    if (error != GRPC_ERROR_NONE) {
+      subchannel_call = CANCELLED_CALL;
+      fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error));
+      grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error);
+    }
+    gpr_atm_rel_store(&calld->subchannel_call,
+                      (gpr_atm)(uintptr_t)subchannel_call);
+    retry_waiting_locked(exec_ctx, calld);
+    goto retry;
+  }
+  /* nothing to be done but wait */
+  add_waiting_locked(calld, op);
+  gpr_mu_unlock(&calld->mu);
+  GPR_TIMER_END("cc_start_transport_stream_op", 0);
 }
 
 /* Constructor for call_data */
-static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem,
-                                  grpc_call_element_args *args) {
-  grpc_subchannel_call_holder_init(elem->call_data, cc_pick_subchannel, elem,
-                                   args->call_stack);
+static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_element *elem,
+                                     grpc_call_element_args *args) {
+  call_data *calld = elem->call_data;
+  gpr_atm_rel_store(&calld->subchannel_call, 0);
+  gpr_mu_init(&calld->mu);
+  calld->connected_subchannel = NULL;
+  calld->waiting_ops = NULL;
+  calld->waiting_ops_count = 0;
+  calld->waiting_ops_capacity = 0;
+  calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  calld->owning_call = args->call_stack;
+  calld->pollent = NULL;
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for call_data */
-static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              const grpc_call_final_info *final_info,
-                              void *and_free_memory) {
-  grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data);
-  gpr_free(and_free_memory);
-}
-
-/* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
-  channel_data *chand = elem->channel_data;
-
-  memset(chand, 0, sizeof(*chand));
-
-  GPR_ASSERT(args->is_last);
-  GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
-
-  gpr_mu_init(&chand->mu);
-  grpc_closure_init(&chand->on_resolver_result_changed,
-                    cc_on_resolver_result_changed, chand);
-  chand->owning_stack = args->channel_stack;
-
-  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
-                               "client_channel");
-  chand->interested_parties = grpc_pollset_set_create();
-}
-
-/* Destructor for channel_data */
-static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
-                                 grpc_channel_element *elem) {
-  channel_data *chand = elem->channel_data;
-
-  if (chand->resolver != NULL) {
-    grpc_resolver_shutdown(exec_ctx, chand->resolver);
-    GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
-  }
-  if (chand->lb_policy != NULL) {
-    grpc_pollset_set_del_pollset_set(exec_ctx,
-                                     chand->lb_policy->interested_parties,
-                                     chand->interested_parties);
-    GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
+static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_call_element *elem,
+                                 const grpc_call_final_info *final_info,
+                                 void *and_free_memory) {
+  call_data *calld = elem->call_data;
+  grpc_subchannel_call *call = GET_CALL(calld);
+  if (call != NULL && call != CANCELLED_CALL) {
+    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "client_channel_destroy_call");
   }
-  grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
-  grpc_pollset_set_destroy(chand->interested_parties);
-  gpr_mu_destroy(&chand->mu);
+  GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING);
+  gpr_mu_destroy(&calld->mu);
+  GPR_ASSERT(calld->waiting_ops_count == 0);
+  gpr_free(calld->waiting_ops);
+  gpr_free(and_free_memory);
 }
 
 static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
@@ -497,16 +745,20 @@ static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
   calld->pollent = pollent;
 }
 
+/*************************************************************************
+ * EXPORTED SYMBOLS
+ */
+
 const grpc_channel_filter grpc_client_channel_filter = {
     cc_start_transport_stream_op,
     cc_start_transport_op,
     sizeof(call_data),
-    init_call_elem,
+    cc_init_call_elem,
     cc_set_pollset_or_pollset_set,
-    destroy_call_elem,
+    cc_destroy_call_elem,
     sizeof(channel_data),
-    init_channel_elem,
-    destroy_channel_elem,
+    cc_init_channel_elem,
+    cc_destroy_channel_elem,
     cc_get_peer,
     "client-channel",
 };
@@ -523,7 +775,7 @@ void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx,
   GRPC_RESOLVER_REF(resolver, "channel");
   if (!grpc_closure_list_empty(chand->waiting_for_config_closures) ||
       chand->exit_idle_when_lb_policy_arrives) {
-    chand->started_resolving = 1;
+    chand->started_resolving = true;
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
     grpc_resolver_next(exec_ctx, resolver, &chand->resolver_result,
                        &chand->on_resolver_result_changed);
@@ -541,10 +793,10 @@ grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     if (chand->lb_policy != NULL) {
       grpc_lb_policy_exit_idle(exec_ctx, chand->lb_policy);
     } else {
-      chand->exit_idle_when_lb_policy_arrives = 1;
+      chand->exit_idle_when_lb_policy_arrives = true;
       if (!chand->started_resolving && chand->resolver != NULL) {
         GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-        chand->started_resolving = 1;
+        chand->started_resolving = true;
         grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result,
                            &chand->on_resolver_result_changed);
       }
diff --git a/src/core/ext/client_config/subchannel_call_holder.c b/src/core/ext/client_config/subchannel_call_holder.c
deleted file mode 100644
index cda939b4a0..0000000000
--- a/src/core/ext/client_config/subchannel_call_holder.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/ext/client_config/subchannel_call_holder.h"
-
-#include <grpc/support/alloc.h>
-
-#include "src/core/lib/profiling/timers.h"
-
-#define GET_CALL(holder) \
-  ((grpc_subchannel_call *)(gpr_atm_acq_load(&(holder)->subchannel_call)))
-
-#define CANCELLED_CALL ((grpc_subchannel_call *)1)
-
-static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *holder,
-                             grpc_error *error);
-static void retry_ops(grpc_exec_ctx *exec_ctx, void *retry_ops_args,
-                      grpc_error *error);
-
-static void add_waiting_locked(grpc_subchannel_call_holder *holder,
-                               grpc_transport_stream_op *op);
-static void fail_locked(grpc_exec_ctx *exec_ctx,
-                        grpc_subchannel_call_holder *holder, grpc_error *error);
-static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
-                                 grpc_subchannel_call_holder *holder);
-
-void grpc_subchannel_call_holder_init(
-    grpc_subchannel_call_holder *holder,
-    grpc_subchannel_call_holder_pick_subchannel pick_subchannel,
-    void *pick_subchannel_arg, grpc_call_stack *owning_call) {
-  gpr_atm_rel_store(&holder->subchannel_call, 0);
-  holder->pick_subchannel = pick_subchannel;
-  holder->pick_subchannel_arg = pick_subchannel_arg;
-  gpr_mu_init(&holder->mu);
-  holder->connected_subchannel = NULL;
-  holder->waiting_ops = NULL;
-  holder->waiting_ops_count = 0;
-  holder->waiting_ops_capacity = 0;
-  holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-  holder->owning_call = owning_call;
-  holder->pollent = NULL;
-}
-
-void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
-                                         grpc_subchannel_call_holder *holder) {
-  grpc_subchannel_call *call = GET_CALL(holder);
-  if (call != NULL && call != CANCELLED_CALL) {
-    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "holder");
-  }
-  GPR_ASSERT(holder->creation_phase ==
-             GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING);
-  gpr_mu_destroy(&holder->mu);
-  GPR_ASSERT(holder->waiting_ops_count == 0);
-  gpr_free(holder->waiting_ops);
-}
-
-// The logic here is fairly complicated, due to (a) the fact that we
-// need to handle the case where we receive the send op before the
-// initial metadata op, and (b) the need for efficiency, especially in
-// the streaming case.
-// TODO(ctiller): Explain this more thoroughly.
-void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
-                                            grpc_subchannel_call_holder *holder,
-                                            grpc_transport_stream_op *op) {
-  /* try to (atomically) get the call */
-  grpc_subchannel_call *call = GET_CALL(holder);
-  GPR_TIMER_BEGIN("grpc_subchannel_call_holder_perform_op", 0);
-  if (call == CANCELLED_CALL) {
-    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
-                                                 GRPC_ERROR_CANCELLED);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  if (call != NULL) {
-    grpc_subchannel_call_process_op(exec_ctx, call, op);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  /* we failed; lock and figure out what to do */
-  gpr_mu_lock(&holder->mu);
-retry:
-  /* need to recheck that another thread hasn't set the call */
-  call = GET_CALL(holder);
-  if (call == CANCELLED_CALL) {
-    gpr_mu_unlock(&holder->mu);
-    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
-                                                 GRPC_ERROR_CANCELLED);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  if (call != NULL) {
-    gpr_mu_unlock(&holder->mu);
-    grpc_subchannel_call_process_op(exec_ctx, call, op);
-    GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-    return;
-  }
-  /* if this is a cancellation, then we can raise our cancelled flag */
-  if (op->cancel_error != GRPC_ERROR_NONE) {
-    if (!gpr_atm_rel_cas(&holder->subchannel_call, 0,
-                         (gpr_atm)(uintptr_t)CANCELLED_CALL)) {
-      goto retry;
-    } else {
-      switch (holder->creation_phase) {
-        case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING:
-          fail_locked(exec_ctx, holder, GRPC_ERROR_REF(op->cancel_error));
-          break;
-        case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL:
-          holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, NULL,
-                                  0, &holder->connected_subchannel, NULL);
-          break;
-      }
-      gpr_mu_unlock(&holder->mu);
-      grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
-                                                   GRPC_ERROR_CANCELLED);
-      GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-      return;
-    }
-  }
-  /* if we don't have a subchannel, try to get one */
-  if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
-      holder->connected_subchannel == NULL &&
-      op->send_initial_metadata != NULL) {
-    holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL;
-    grpc_closure_init(&holder->next_step, subchannel_ready, holder);
-    GRPC_CALL_STACK_REF(holder->owning_call, "pick_subchannel");
-    if (holder->pick_subchannel(
-            exec_ctx, holder->pick_subchannel_arg, op->send_initial_metadata,
-            op->send_initial_metadata_flags, &holder->connected_subchannel,
-            &holder->next_step)) {
-      holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-      GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
-    }
-  }
-  /* if we've got a subchannel, then let's ask it to create a call */
-  if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
-      holder->connected_subchannel != NULL) {
-    grpc_subchannel_call *subchannel_call = NULL;
-    grpc_error *error = grpc_connected_subchannel_create_call(
-        exec_ctx, holder->connected_subchannel, holder->pollent,
-        &subchannel_call);
-    if (error != GRPC_ERROR_NONE) {
-      subchannel_call = CANCELLED_CALL;
-      fail_locked(exec_ctx, holder, GRPC_ERROR_REF(error));
-      grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error);
-    }
-    gpr_atm_rel_store(&holder->subchannel_call,
-                      (gpr_atm)(uintptr_t)subchannel_call);
-    retry_waiting_locked(exec_ctx, holder);
-    goto retry;
-  }
-  /* nothing to be done but wait */
-  add_waiting_locked(holder, op);
-  gpr_mu_unlock(&holder->mu);
-  GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
-}
-
-static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
-                             grpc_error *error) {
-  grpc_subchannel_call_holder *holder = arg;
-  gpr_mu_lock(&holder->mu);
-  GPR_ASSERT(holder->creation_phase ==
-             GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL);
-  holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-  if (holder->connected_subchannel == NULL) {
-    gpr_atm_no_barrier_store(&holder->subchannel_call, 1);
-    fail_locked(exec_ctx, holder,
-                GRPC_ERROR_CREATE_REFERENCING("Failed to create subchannel",
-                                              &error, 1));
-  } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) {
-    /* already cancelled before subchannel became ready */
-    fail_locked(exec_ctx, holder,
-                GRPC_ERROR_CREATE_REFERENCING(
-                    "Cancelled before creating subchannel", &error, 1));
-  } else {
-    grpc_subchannel_call *subchannel_call = NULL;
-    grpc_error *new_error = grpc_connected_subchannel_create_call(
-        exec_ctx, holder->connected_subchannel, holder->pollent,
-        &subchannel_call);
-    if (new_error != GRPC_ERROR_NONE) {
-      new_error = grpc_error_add_child(new_error, error);
-      subchannel_call = CANCELLED_CALL;
-      fail_locked(exec_ctx, holder, new_error);
-    }
-    gpr_atm_rel_store(&holder->subchannel_call,
-                      (gpr_atm)(uintptr_t)subchannel_call);
-    retry_waiting_locked(exec_ctx, holder);
-  }
-  gpr_mu_unlock(&holder->mu);
-  GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
-}
-
-typedef struct {
-  grpc_transport_stream_op **ops;
-  size_t nops;
-  grpc_subchannel_call *call;
-} retry_ops_args;
-
-static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
-                                 grpc_subchannel_call_holder *holder) {
-  if (holder->waiting_ops_count == 0) {
-    return;
-  }
-
-  retry_ops_args *a = gpr_malloc(sizeof(*a));
-  a->ops = holder->waiting_ops;
-  a->nops = holder->waiting_ops_count;
-  a->call = GET_CALL(holder);
-  if (a->call == CANCELLED_CALL) {
-    gpr_free(a);
-    fail_locked(exec_ctx, holder, GRPC_ERROR_CANCELLED);
-    return;
-  }
-  holder->waiting_ops = NULL;
-  holder->waiting_ops_count = 0;
-  holder->waiting_ops_capacity = 0;
-  GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
-  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
-                      GRPC_ERROR_NONE, NULL);
-}
-
-static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
-  retry_ops_args *a = args;
-  size_t i;
-  for (i = 0; i < a->nops; i++) {
-    grpc_subchannel_call_process_op(exec_ctx, a->call, a->ops[i]);
-  }
-  GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops");
-  gpr_free(a->ops);
-  gpr_free(a);
-}
-
-static void add_waiting_locked(grpc_subchannel_call_holder *holder,
-                               grpc_transport_stream_op *op) {
-  GPR_TIMER_BEGIN("add_waiting_locked", 0);
-  if (holder->waiting_ops_count == holder->waiting_ops_capacity) {
-    holder->waiting_ops_capacity = GPR_MAX(3, 2 * holder->waiting_ops_capacity);
-    holder->waiting_ops =
-        gpr_realloc(holder->waiting_ops, holder->waiting_ops_capacity *
-                                             sizeof(*holder->waiting_ops));
-  }
-  holder->waiting_ops[holder->waiting_ops_count++] = op;
-  GPR_TIMER_END("add_waiting_locked", 0);
-}
-
-static void fail_locked(grpc_exec_ctx *exec_ctx,
-                        grpc_subchannel_call_holder *holder,
-                        grpc_error *error) {
-  size_t i;
-  for (i = 0; i < holder->waiting_ops_count; i++) {
-    grpc_transport_stream_op_finish_with_failure(
-        exec_ctx, holder->waiting_ops[i], GRPC_ERROR_REF(error));
-  }
-  holder->waiting_ops_count = 0;
-  GRPC_ERROR_UNREF(error);
-}
-
-char *grpc_subchannel_call_holder_get_peer(
-    grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder) {
-  grpc_subchannel_call *subchannel_call = GET_CALL(holder);
-
-  if (subchannel_call == NULL || subchannel_call == CANCELLED_CALL) {
-    return NULL;
-  } else {
-    return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
-  }
-}
diff --git a/src/core/ext/client_config/subchannel_call_holder.h b/src/core/ext/client_config/subchannel_call_holder.h
deleted file mode 100644
index 19b22a2905..0000000000
--- a/src/core/ext/client_config/subchannel_call_holder.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H
-#define GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H
-
-#include "src/core/ext/client_config/subchannel.h"
-#include "src/core/lib/iomgr/polling_entity.h"
-
-/** Pick a subchannel for grpc_subchannel_call_holder;
-    Return 1 if subchannel is available immediately (in which case on_ready
-    should not be called), or 0 otherwise (in which case on_ready should be
-    called when the subchannel is available) */
-typedef int (*grpc_subchannel_call_holder_pick_subchannel)(
-    grpc_exec_ctx *exec_ctx, void *arg, grpc_metadata_batch *initial_metadata,
-    uint32_t initial_metadata_flags,
-    grpc_connected_subchannel **connected_subchannel, grpc_closure *on_ready);
-
-typedef enum {
-  GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING,
-  GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL
-} grpc_subchannel_call_holder_creation_phase;
-
-/** Wrapper for holding a pointer to grpc_subchannel_call, and the
-    associated machinery to create such a pointer.
-    Handles queueing of stream ops until a call object is ready, waiting
-    for initial metadata before trying to create a call object,
-    and handling cancellation gracefully.
-
-    The channel filter uses this as their call_data. */
-typedef struct grpc_subchannel_call_holder {
-  /** either 0 for no call, 1 for cancelled, or a pointer to a
-      grpc_subchannel_call */
-  gpr_atm subchannel_call;
-  /** Helper function to choose the subchannel on which to create
-      the call object. Channel filter delegates to the load
-      balancing policy (once it's ready). */
-  grpc_subchannel_call_holder_pick_subchannel pick_subchannel;
-  void *pick_subchannel_arg;
-
-  gpr_mu mu;
-
-  grpc_subchannel_call_holder_creation_phase creation_phase;
-  grpc_connected_subchannel *connected_subchannel;
-  grpc_polling_entity *pollent;
-
-  grpc_transport_stream_op **waiting_ops;
-  size_t waiting_ops_count;
-  size_t waiting_ops_capacity;
-
-  grpc_closure next_step;
-
-  grpc_call_stack *owning_call;
-} grpc_subchannel_call_holder;
-
-void grpc_subchannel_call_holder_init(
-    grpc_subchannel_call_holder *holder,
-    grpc_subchannel_call_holder_pick_subchannel pick_subchannel,
-    void *pick_subchannel_arg, grpc_call_stack *owning_call);
-void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
-                                         grpc_subchannel_call_holder *holder);
-
-void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
-                                            grpc_subchannel_call_holder *holder,
-                                            grpc_transport_stream_op *op);
-char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx,
-                                           grpc_subchannel_call_holder *holder);
-
-#endif /* GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H */
diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c
index a3def8affb..981d3348ce 100644
--- a/src/core/lib/iomgr/combiner.c
+++ b/src/core/lib/iomgr/combiner.c
@@ -235,6 +235,9 @@ bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) {
       gpr_log(GPR_DEBUG, "C:%p finish old_state=%" PRIdPTR, lock, old_state));
   lock->time_to_execute_final_list = false;
   switch (old_state) {
+    default:
+      // we have multiple queued work items: just continue executing them
+      break;
     case 5:  // we're down to one queued item: if it's the final list we
     case 4:  // should do that
       if (!grpc_closure_list_empty(lock->final_list)) {
diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h
index 80ed33c2a7..5f3e5f94b3 100644
--- a/src/core/lib/iomgr/combiner.h
+++ b/src/core/lib/iomgr/combiner.h
@@ -41,7 +41,7 @@
 #include "src/core/lib/support/mpscq.h"
 
 // Provides serialized access to some resource.
-// Each action queued on an aelock is executed serially in a borrowed thread.
+// Each action queued on a combiner is executed serially in a borrowed thread.
 // The actual thread executing actions may change over time (but there will only
 // every be one at a time).
 
@@ -55,11 +55,12 @@ void grpc_combiner_execute(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
                            grpc_closure *closure, grpc_error *error,
                            bool covered_by_poller);
 // Execute \a action within the lock just prior to unlocking.
-// if \a hint_async_break is additionally set, the combiner is tries to trip
-// through the workqueue between finishing the primary queue of combined
-// closures and executing the finally list.
+// if \a hint_async_break is true, the combiner is tries to hand execution to
+// another thread before finishing the primary queue of combined closures and
+// executing the finally list.
+// Deprecation warning: \a hint_async_break will be removed in a future version
 // Takes a very slow and round-about path if not called from a
-// grpc_combiner_execute closure
+// grpc_combiner_execute closure.
 void grpc_combiner_execute_finally(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
                                    grpc_closure *closure, grpc_error *error);
 
diff --git a/src/core/lib/support/mpscq.c b/src/core/lib/support/mpscq.c
index 25b055b172..cdd6335f82 100644
--- a/src/core/lib/support/mpscq.c
+++ b/src/core/lib/support/mpscq.c
@@ -38,7 +38,7 @@
 void gpr_mpscq_init(gpr_mpscq *q) {
   gpr_atm_no_barrier_store(&q->head, (gpr_atm)&q->stub);
   q->tail = &q->stub;
-  gpr_atm_no_barrier_store(&q->stub.next, 0);
+  gpr_atm_no_barrier_store(&q->stub.next, (gpr_atm)NULL);
 }
 
 void gpr_mpscq_destroy(gpr_mpscq *q) {
@@ -47,16 +47,17 @@ void gpr_mpscq_destroy(gpr_mpscq *q) {
 }
 
 void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) {
-  gpr_atm_no_barrier_store(&n->next, 0);
+  gpr_atm_no_barrier_store(&n->next, (gpr_atm)NULL);
   gpr_mpscq_node *prev =
       (gpr_mpscq_node *)gpr_atm_full_xchg(&q->head, (gpr_atm)n);
-  gpr_atm_rel_store(&prev->next, (gpr_atm)n);
+  gpr_atm_no_barrier_store(&prev->next, (gpr_atm)n);
 }
 
 gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) {
   gpr_mpscq_node *tail = q->tail;
   gpr_mpscq_node *next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next);
   if (tail == &q->stub) {
+    // indicates the list is actually (ephemerally) empty
     if (next == NULL) return NULL;
     q->tail = next;
     tail = next;
@@ -68,7 +69,8 @@ gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) {
   }
   gpr_mpscq_node *head = (gpr_mpscq_node *)gpr_atm_acq_load(&q->head);
   if (tail != head) {
-    return 0;
+    // indicates a retry is in order: we're still adding
+    return NULL;
   }
   gpr_mpscq_push(q, &q->stub);
   next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next);
@@ -76,5 +78,6 @@ gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) {
     q->tail = next;
     return tail;
   }
+  // indicates a retry is in order: we're still adding
   return NULL;
 }
diff --git a/src/core/lib/support/mpscq.h b/src/core/lib/support/mpscq.h
index 1201edceb1..977a117952 100644
--- a/src/core/lib/support/mpscq.h
+++ b/src/core/lib/support/mpscq.h
@@ -41,8 +41,8 @@
 // implementation from Dmitry Vyukov here:
 // http://www.1024cores.net/home/lock-free-algorithms/queues/intrusive-mpsc-node-based-queue
 
-// List node (include this in a data structure and dangle the rest of the
-// interesting bits off the end)
+// List node (include this in a data structure at the top, and add application
+// fields after it - to simulate inheritance)
 typedef struct gpr_mpscq_node { gpr_atm next; } gpr_mpscq_node;
 
 // Actual queue type
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 87a526fe98..1a014da3b8 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -153,14 +153,12 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
     (*option)->UpdateArguments(&args);
     (*option)->UpdatePlugins(&plugins_);
   }
-  if (!thread_pool) {
-    for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
-      if ((*plugin)->has_sync_methods()) {
-        thread_pool.reset(CreateDefaultThreadPool());
-        has_sync_methods = true;
-        break;
-      }
+  for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
+    if (!thread_pool && (*plugin)->has_sync_methods()) {
+      thread_pool.reset(CreateDefaultThreadPool());
+      has_sync_methods = true;
     }
+    (*plugin)->UpdateChannelArguments(&args);
   }
   if (max_message_size_ > 0) {
     args.SetInt(GRPC_ARG_MAX_MESSAGE_LENGTH, max_message_size_);
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index 360c33219a..1ae412e2c5 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -235,7 +235,6 @@ CORE_SOURCE_FILES = [
   'src/core/ext/client_config/resolver_registry.c',
   'src/core/ext/client_config/resolver_result.c',
   'src/core/ext/client_config/subchannel.c',
-  'src/core/ext/client_config/subchannel_call_holder.c',
   'src/core/ext/client_config/subchannel_index.c',
   'src/core/ext/client_config/uri_parser.c',
   'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
diff --git a/test/core/iomgr/combiner_test.c b/test/core/iomgr/combiner_test.c
index cfb6159b17..2ea4e5dd14 100644
--- a/test/core/iomgr/combiner_test.c
+++ b/test/core/iomgr/combiner_test.c
@@ -80,7 +80,6 @@ typedef struct {
 
 static void check_one(grpc_exec_ctx *exec_ctx, void *a, grpc_error *error) {
   ex_args *args = a;
-  // gpr_log(GPR_DEBUG, "*%p=%d; step %d", args->ctr, *args->ctr, args->value);
   GPR_ASSERT(*args->ctr == args->value - 1);
   *args->ctr = args->value;
   gpr_free(a);
@@ -100,6 +99,8 @@ static void execute_many_loop(void *a) {
                             false);
       grpc_exec_ctx_flush(&exec_ctx);
     }
+    // sleep for a little bit, to test a combiner draining and another thread
+    // picking it up
     gpr_sleep_until(GRPC_TIMEOUT_MILLIS_TO_DEADLINE(100));
   }
   grpc_exec_ctx_finish(&exec_ctx);
diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc
index f06053ca23..4af00cdc96 100644
--- a/test/cpp/util/grpc_tool.cc
+++ b/test/cpp/util/grpc_tool.cc
@@ -57,7 +57,7 @@ DEFINE_bool(remotedb, true, "Use server types to parse and format messages");
 DEFINE_string(metadata, "",
               "Metadata to send to server, in the form of key1:val1:key2:val2");
 DEFINE_string(proto_path, ".", "Path to look for the proto file.");
-DEFINE_string(proto_file, "", "Name of the proto file.");
+DEFINE_string(protofiles, "", "Name of the proto file.");
 DEFINE_bool(binary_input, false, "Input in binary format");
 DEFINE_bool(binary_output, false, "Output in binary format");
 DEFINE_string(infile, "", "Input file (default is stdin)");
@@ -71,9 +71,9 @@ class GrpcTool {
   explicit GrpcTool();
   virtual ~GrpcTool() {}
 
-  bool Help(int argc, const char** argv, CliCredentials cred,
+  bool Help(int argc, const char** argv, const CliCredentials& cred,
             GrpcToolOutputCallback callback);
-  bool CallMethod(int argc, const char** argv, CliCredentials cred,
+  bool CallMethod(int argc, const char** argv, const CliCredentials& cred,
                   GrpcToolOutputCallback callback);
   // TODO(zyc): implement the following methods
   // bool ListServices(int argc, const char** argv, GrpcToolOutputCallback
@@ -101,7 +101,7 @@ class GrpcTool {
 };
 
 template <typename T>
-std::function<bool(GrpcTool*, int, const char**, const CliCredentials,
+std::function<bool(GrpcTool*, int, const char**, const CliCredentials&,
                    GrpcToolOutputCallback)>
 BindWith5Args(T&& func) {
   return std::bind(std::forward<T>(func), std::placeholders::_1,
@@ -156,7 +156,7 @@ void PrintMetadata(const T& m, const grpc::string& message) {
 
 struct Command {
   const char* command;
-  std::function<bool(GrpcTool*, int, const char**, const CliCredentials,
+  std::function<bool(GrpcTool*, int, const char**, const CliCredentials&,
                      GrpcToolOutputCallback)>
       function;
   int min_args;
@@ -201,7 +201,7 @@ const Command* FindCommand(const grpc::string& name) {
 }
 }  // namespace
 
-int GrpcToolMainLib(int argc, const char** argv, const CliCredentials cred,
+int GrpcToolMainLib(int argc, const char** argv, const CliCredentials& cred,
                     GrpcToolOutputCallback callback) {
   if (argc < 2) {
     Usage("No command specified");
@@ -238,7 +238,7 @@ void GrpcTool::CommandUsage(const grpc::string& usage) const {
   }
 }
 
-bool GrpcTool::Help(int argc, const char** argv, const CliCredentials cred,
+bool GrpcTool::Help(int argc, const char** argv, const CliCredentials& cred,
                     GrpcToolOutputCallback callback) {
   CommandUsage(
       "Print help\n"
@@ -258,7 +258,7 @@ bool GrpcTool::Help(int argc, const char** argv, const CliCredentials cred,
 }
 
 bool GrpcTool::CallMethod(int argc, const char** argv,
-                          const CliCredentials cred,
+                          const CliCredentials& cred,
                           GrpcToolOutputCallback callback) {
   CommandUsage(
       "Call method\n"
@@ -267,10 +267,10 @@ bool GrpcTool::CallMethod(int argc, const char** argv,
       "    <service>                ; Exported service name\n"
       "    <method>                 ; Method name\n"
       "    <request>                ; Text protobuffer (overrides infile)\n"
-      "    --proto_file             ; Comma separated proto files used as a"
+      "    --protofiles             ; Comma separated proto files used as a"
       " fallback when parsing request/response\n"
       "    --proto_path             ; The search path of proto files, valid"
-      " only when --proto_file is given\n"
+      " only when --protofiles is given\n"
       "    --metadata               ; The metadata to be sent to the server\n"
       "    --infile                 ; Input filename (defaults to stdin)\n"
       "    --outfile                ; Output filename (defaults to stdout)\n"
@@ -310,7 +310,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv,
   if (!FLAGS_binary_input || !FLAGS_binary_output) {
     parser.reset(
         new grpc::testing::ProtoFileParser(FLAGS_remotedb ? channel : nullptr,
-                                           FLAGS_proto_path, FLAGS_proto_file));
+                                           FLAGS_proto_path, FLAGS_protofiles));
     if (parser->HasError()) {
       return false;
     }
diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h
index 3da86937c2..df3680258b 100644
--- a/test/cpp/util/grpc_tool.h
+++ b/test/cpp/util/grpc_tool.h
@@ -45,7 +45,7 @@ namespace testing {
 
 typedef std::function<bool(const grpc::string &)> GrpcToolOutputCallback;
 
-int GrpcToolMainLib(int argc, const char **argv, CliCredentials cred,
+int GrpcToolMainLib(int argc, const char **argv, const CliCredentials &cred,
                     GrpcToolOutputCallback callback);
 
 }  // namespace testing
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 09d48546bc..3ffd5e152c 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -929,7 +929,6 @@ src/core/ext/client_config/resolver_factory.h \
 src/core/ext/client_config/resolver_registry.h \
 src/core/ext/client_config/resolver_result.h \
 src/core/ext/client_config/subchannel.h \
-src/core/ext/client_config/subchannel_call_holder.h \
 src/core/ext/client_config/subchannel_index.h \
 src/core/ext/client_config/uri_parser.h \
 src/core/ext/lb_policy/grpclb/grpclb.h \
@@ -1110,7 +1109,6 @@ src/core/ext/client_config/resolver_factory.c \
 src/core/ext/client_config/resolver_registry.c \
 src/core/ext/client_config/resolver_result.c \
 src/core/ext/client_config/subchannel.c \
-src/core/ext/client_config/subchannel_call_holder.c \
 src/core/ext/client_config/subchannel_index.c \
 src/core/ext/client_config/uri_parser.c \
 src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 96a1171fdb..1945d1c417 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -6141,7 +6141,6 @@
       "src/core/ext/client_config/resolver_registry.h", 
       "src/core/ext/client_config/resolver_result.h", 
       "src/core/ext/client_config/subchannel.h", 
-      "src/core/ext/client_config/subchannel_call_holder.h", 
       "src/core/ext/client_config/subchannel_index.h", 
       "src/core/ext/client_config/uri_parser.h"
     ], 
@@ -6177,8 +6176,6 @@
       "src/core/ext/client_config/resolver_result.h", 
       "src/core/ext/client_config/subchannel.c", 
       "src/core/ext/client_config/subchannel.h", 
-      "src/core/ext/client_config/subchannel_call_holder.c", 
-      "src/core/ext/client_config/subchannel_call_holder.h", 
       "src/core/ext/client_config/subchannel_index.c", 
       "src/core/ext/client_config/subchannel_index.h", 
       "src/core/ext/client_config/uri_parser.c", 
diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj
index efa8a98c49..98873ca8d7 100644
--- a/vsprojects/vcxproj/grpc/grpc.vcxproj
+++ b/vsprojects/vcxproj/grpc/grpc.vcxproj
@@ -438,7 +438,6 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\resolver_registry.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\resolver_result.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\uri_parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\lb_policy\grpclb\grpclb.h" />
@@ -780,8 +779,6 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.c">
-    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\uri_parser.c">
diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters
index f7b7aad962..f3d7958f7c 100644
--- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters
@@ -478,9 +478,6 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.c">
       <Filter>src\core\ext\client_config</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.c">
-      <Filter>src\core\ext\client_config</Filter>
-    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.c">
       <Filter>src\core\ext\client_config</Filter>
     </ClCompile>
@@ -1088,9 +1085,6 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.h">
       <Filter>src\core\ext\client_config</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.h">
-      <Filter>src\core\ext\client_config</Filter>
-    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.h">
       <Filter>src\core\ext\client_config</Filter>
     </ClInclude>
diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj
index 941aa11c01..0ea880833a 100644
--- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj
+++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj
@@ -404,7 +404,6 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\resolver_registry.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\resolver_result.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\uri_parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\load_reporting\load_reporting.h" />
@@ -696,8 +695,6 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.c">
-    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\uri_parser.c">
diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters
index d5bf7b47da..7549a9609d 100644
--- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters
@@ -403,9 +403,6 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.c">
       <Filter>src\core\ext\client_config</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.c">
-      <Filter>src\core\ext\client_config</Filter>
-    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.c">
       <Filter>src\core\ext\client_config</Filter>
     </ClCompile>
@@ -926,9 +923,6 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel.h">
       <Filter>src\core\ext\client_config</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_call_holder.h">
-      <Filter>src\core\ext\client_config</Filter>
-    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\client_config\subchannel_index.h">
       <Filter>src\core\ext\client_config</Filter>
     </ClInclude>