Merge github.com:grpc/grpc into big_data

174 files changed, 3892 insertions, 2180 deletions

diff --git a/src/core/census/context.h b/src/core/census/context.h
index e45409a6b8..700bcf86cf 100644
--- a/src/core/census/context.h
+++ b/src/core/census/context.h
@@ -36,10 +36,12 @@
 
 #include <grpc/census.h>
 
+#define GRPC_CENSUS_MAX_ON_THE_WIRE_TAG_BYTES 2048
+
 /* census_context is the in-memory representation of information needed to
  * maintain tracing, RPC statistics and resource usage information. */
 struct census_context {
-  census_tag_set *tags;  /* Opaque data structure for census tags. */
+  census_tag_set *tags; /* Opaque data structure for census tags. */
 };
 
 #endif /* GRPC_INTERNAL_CORE_CENSUS_CONTEXT_H */
diff --git a/src/core/census/grpc_filter.c b/src/core/census/grpc_filter.c
index 8f18cd503e..4529ae9bd7 100644
--- a/src/core/census/grpc_filter.c
+++ b/src/core/census/grpc_filter.c
@@ -115,8 +115,11 @@ static void server_mutate_op(grpc_call_element *elem,
 static void server_start_transport_op(grpc_exec_ctx *exec_ctx,
                                       grpc_call_element *elem,
                                       grpc_transport_stream_op *op) {
+  /* TODO(ctiller): this code fails. I don't know why. I expect it's
+                    incomplete, and someone should look at it soon.
+
   call_data *calld = elem->call_data;
-  GPR_ASSERT((calld->op_id.upper != 0) || (calld->op_id.lower != 0));
+  GPR_ASSERT((calld->op_id.upper != 0) || (calld->op_id.lower != 0)); */
   server_mutate_op(elem, op);
   grpc_call_next_op(exec_ctx, elem, op);
 }
diff --git a/src/core/channel/channel_stack.c b/src/core/channel/channel_stack.c
index 7f7fbf420f..5e09a050ee 100644
--- a/src/core/channel/channel_stack.c
+++ b/src/core/channel/channel_stack.c
@@ -101,11 +101,12 @@ grpc_call_element *grpc_call_stack_element(grpc_call_stack *call_stack,
   return CALL_ELEMS_FROM_STACK(call_stack) + index;
 }
 
-void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx,
+void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
+                             grpc_iomgr_cb_func destroy, void *destroy_arg,
                              const grpc_channel_filter **filters,
-                             size_t filter_count, grpc_channel *master,
+                             size_t filter_count,
                              const grpc_channel_args *channel_args,
-                             grpc_channel_stack *stack) {
+                             const char *name, grpc_channel_stack *stack) {
   size_t call_size =
       ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call_stack)) +
       ROUND_UP_TO_ALIGNMENT_SIZE(filter_count * sizeof(grpc_call_element));
@@ -115,6 +116,8 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx,
   size_t i;
 
   stack->count = filter_count;
+  GRPC_STREAM_REF_INIT(&stack->refcount, initial_refs, destroy, destroy_arg,
+                       name);
   elems = CHANNEL_ELEMS_FROM_STACK(stack);
   user_data =
       ((char *)elems) +
@@ -122,7 +125,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx,
 
   /* init per-filter data */
   for (i = 0; i < filter_count; i++) {
-    args.master = master;
+    args.channel_stack = stack;
     args.channel_args = channel_args;
     args.is_first = i == 0;
     args.is_last = i == (filter_count - 1);
@@ -166,15 +169,15 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx,
   size_t i;
 
   call_stack->count = count;
-  gpr_ref_init(&call_stack->refcount.refs, initial_refs);
-  grpc_closure_init(&call_stack->refcount.destroy, destroy, destroy_arg);
+  GRPC_STREAM_REF_INIT(&call_stack->refcount, initial_refs, destroy,
+                       destroy_arg, "CALL_STACK");
   call_elems = CALL_ELEMS_FROM_STACK(call_stack);
   user_data = ((char *)call_elems) +
               ROUND_UP_TO_ALIGNMENT_SIZE(count * sizeof(grpc_call_element));
 
   /* init per-filter data */
   for (i = 0; i < count; i++) {
-    args.refcount = &call_stack->refcount;
+    args.call_stack = call_stack;
     args.server_transport_data = transport_server_data;
     args.context = context;
     call_elems[i].filter = channel_elems[i].filter;
diff --git a/src/core/channel/channel_stack.h b/src/core/channel/channel_stack.h
index 1db12ead7e..c01050e717 100644
--- a/src/core/channel/channel_stack.h
+++ b/src/core/channel/channel_stack.h
@@ -51,15 +51,18 @@
 typedef struct grpc_channel_element grpc_channel_element;
 typedef struct grpc_call_element grpc_call_element;
 
+typedef struct grpc_channel_stack grpc_channel_stack;
+typedef struct grpc_call_stack grpc_call_stack;
+
 typedef struct {
-  grpc_channel *master;
+  grpc_channel_stack *channel_stack;
   const grpc_channel_args *channel_args;
   int is_first;
   int is_last;
 } grpc_channel_element_args;
 
 typedef struct {
-  grpc_stream_refcount *refcount;
+  grpc_call_stack *call_stack;
   const void *server_transport_data;
   grpc_call_context_element *context;
 } grpc_call_element_args;
@@ -144,23 +147,24 @@ struct grpc_call_element {
 
 /* A channel stack tracks a set of related filters for one channel, and
    guarantees they live within a single malloc() allocation */
-typedef struct {
+struct grpc_channel_stack {
+  grpc_stream_refcount refcount;
   size_t count;
   /* Memory required for a call stack (computed at channel stack
      initialization) */
   size_t call_stack_size;
-} grpc_channel_stack;
+};
 
 /* A call stack tracks a set of related filters for one call, and guarantees
    they live within a single malloc() allocation */
-typedef struct {
+struct grpc_call_stack {
   /* shared refcount for this channel stack.
      MUST be the first element: the underlying code calls destroy
      with the address of the refcount, but higher layers prefer to think
      about the address of the call stack itself. */
   grpc_stream_refcount refcount;
   size_t count;
-} grpc_call_stack;
+};
 
 /* Get a channel element given a channel stack and its index */
 grpc_channel_element *grpc_channel_stack_element(grpc_channel_stack *stack,
@@ -175,11 +179,11 @@ grpc_call_element *grpc_call_stack_element(grpc_call_stack *stack, size_t i);
 size_t grpc_channel_stack_size(const grpc_channel_filter **filters,
                                size_t filter_count);
 /* Initialize a channel stack given some filters */
-void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx,
+void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
+                             grpc_iomgr_cb_func destroy, void *destroy_arg,
                              const grpc_channel_filter **filters,
-                             size_t filter_count, grpc_channel *master,
-                             const grpc_channel_args *args,
-                             grpc_channel_stack *stack);
+                             size_t filter_count, const grpc_channel_args *args,
+                             const char *name, grpc_channel_stack *stack);
 /* Destroy a channel stack */
 void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx,
                                 grpc_channel_stack *stack);
@@ -199,14 +203,23 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
                                  grpc_pollset *pollset);
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
-#define grpc_call_stack_ref(call_stack, reason) \
+#define GRPC_CALL_STACK_REF(call_stack, reason) \
   grpc_stream_ref(&(call_stack)->refcount, reason)
-#define grpc_call_stack_unref(exec_ctx, call_stack, reason) \
+#define GRPC_CALL_STACK_UNREF(exec_ctx, call_stack, reason) \
   grpc_stream_unref(exec_ctx, &(call_stack)->refcount, reason)
+#define GRPC_CHANNEL_STACK_REF(channel_stack, reason) \
+  grpc_stream_ref(&(channel_stack)->refcount, reason)
+#define GRPC_CHANNEL_STACK_UNREF(exec_ctx, channel_stack, reason) \
+  grpc_stream_unref(exec_ctx, &(channel_stack)->refcount, reason)
 #else
-#define grpc_call_stack_ref(call_stack) grpc_stream_ref(&(call_stack)->refcount)
-#define grpc_call_stack_unref(exec_ctx, call_stack) \
+#define GRPC_CALL_STACK_REF(call_stack, reason) \
+  grpc_stream_ref(&(call_stack)->refcount)
+#define GRPC_CALL_STACK_UNREF(exec_ctx, call_stack, reason) \
   grpc_stream_unref(exec_ctx, &(call_stack)->refcount)
+#define GRPC_CHANNEL_STACK_REF(channel_stack, reason) \
+  grpc_stream_ref(&(channel_stack)->refcount)
+#define GRPC_CHANNEL_STACK_UNREF(exec_ctx, channel_stack, reason) \
+  grpc_stream_unref(exec_ctx, &(channel_stack)->refcount)
 #endif
 
 /* Destroy a call stack */
diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c
index 020138bf15..385ae3be9b 100644
--- a/src/core/channel/client_channel.c
+++ b/src/core/channel/client_channel.c
@@ -59,11 +59,6 @@ typedef struct client_channel_channel_data {
   grpc_resolver *resolver;
   /** have we started resolving this channel */
   int started_resolving;
-  /** master channel - the grpc_channel instance that ultimately owns
-      this channel_data via its channel stack.
-      We occasionally use this to bump the refcount on the master channel
-      to keep ourselves alive through an asynchronous operation. */
-  grpc_channel *master;
 
   /** mutex protecting client configuration, including all
       variables below in this data structure */
@@ -81,8 +76,10 @@ typedef struct client_channel_channel_data {
   grpc_connectivity_state_tracker state_tracker;
   /** when an lb_policy arrives, should we try to exit idle */
   int exit_idle_when_lb_policy_arrives;
-  /** pollset_set of interested parties in a new connection */
-  grpc_pollset_set pollset_set;
+  /** owning stack */
+  grpc_channel_stack *owning_stack;
+  /** interested parties */
+  grpc_pollset_set interested_parties;
 } channel_data;
 
 /** We create one watcher for each new lb_policy that is returned from a
@@ -103,9 +100,7 @@ typedef struct {
 } waiting_call;
 
 static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  channel_data *chand = elem->channel_data;
-  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data,
-                                              chand->master);
+  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data);
 }
 
 static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
@@ -121,10 +116,18 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
 
 static void on_lb_policy_state_changed_locked(
     grpc_exec_ctx *exec_ctx, lb_policy_connectivity_watcher *w) {
+  grpc_connectivity_state publish_state = w->state;
   /* check if the notification is for a stale policy */
   if (w->lb_policy != w->chand->lb_policy) return;
 
-  grpc_connectivity_state_set(exec_ctx, &w->chand->state_tracker, w->state,
+  if (publish_state == GRPC_CHANNEL_FATAL_FAILURE &&
+      w->chand->resolver != NULL) {
+    publish_state = GRPC_CHANNEL_TRANSIENT_FAILURE;
+    grpc_resolver_channel_saw_error(exec_ctx, w->chand->resolver);
+    GRPC_LB_POLICY_UNREF(exec_ctx, w->chand->lb_policy, "channel");
+    w->chand->lb_policy = NULL;
+  }
+  grpc_connectivity_state_set(exec_ctx, &w->chand->state_tracker, publish_state,
                               "lb_changed");
   if (w->state != GRPC_CHANNEL_FATAL_FAILURE) {
     watch_lb_policy(exec_ctx, w->chand, w->lb_policy, w->state);
@@ -139,7 +142,7 @@ static void on_lb_policy_state_changed(grpc_exec_ctx *exec_ctx, void *arg,
   on_lb_policy_state_changed_locked(exec_ctx, w);
   gpr_mu_unlock(&w->chand->mu_config);
 
-  GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, w->chand->master, "watch_lb_policy");
+  GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "watch_lb_policy");
   gpr_free(w);
 }
 
@@ -147,7 +150,7 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
                             grpc_lb_policy *lb_policy,
                             grpc_connectivity_state current_state) {
   lb_policy_connectivity_watcher *w = gpr_malloc(sizeof(*w));
-  GRPC_CHANNEL_INTERNAL_REF(chand->master, "watch_lb_policy");
+  GRPC_CHANNEL_STACK_REF(chand->owning_stack, "watch_lb_policy");
 
   w->chand = chand;
   grpc_closure_init(&w->on_changed, on_lb_policy_state_changed, w);
@@ -179,6 +182,11 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
 
   chand->incoming_configuration = NULL;
 
+  if (lb_policy != NULL) {
+    grpc_pollset_set_add_pollset_set(exec_ctx, &lb_policy->interested_parties,
+                                     &chand->interested_parties);
+  }
+
   gpr_mu_lock(&chand->mu_config);
   old_lb_policy = chand->lb_policy;
   chand->lb_policy = lb_policy;
@@ -200,7 +208,7 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
       watch_lb_policy(exec_ctx, chand, lb_policy, state);
     }
     gpr_mu_unlock(&chand->mu_config);
-    GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+    GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
     grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration,
                        &chand->on_config_changed);
     GRPC_RESOLVER_UNREF(exec_ctx, resolver, "channel-next");
@@ -222,7 +230,9 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
   }
 
   if (old_lb_policy != NULL) {
-    grpc_lb_policy_shutdown(exec_ctx, old_lb_policy);
+    grpc_pollset_set_del_pollset_set(exec_ctx,
+                                     &old_lb_policy->interested_parties,
+                                     &chand->interested_parties);
     GRPC_LB_POLICY_UNREF(exec_ctx, old_lb_policy, "channel");
   }
 
@@ -230,20 +240,22 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "config_change");
   }
 
-  GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, chand->master, "resolver");
+  GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->owning_stack, "resolver");
 }
 
 static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                   grpc_channel_element *elem,
                                   grpc_transport_op *op) {
-  grpc_lb_policy *lb_policy = NULL;
   channel_data *chand = elem->channel_data;
   grpc_resolver *destroy_resolver = NULL;
 
   grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, 1);
 
   GPR_ASSERT(op->set_accept_stream == NULL);
-  GPR_ASSERT(op->bind_pollset == NULL);
+  if (op->bind_pollset != NULL) {
+    grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties,
+                                 op->bind_pollset);
+  }
 
   gpr_mu_lock(&chand->mu_config);
   if (op->on_connectivity_state_change != NULL) {
@@ -254,9 +266,14 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
     op->connectivity_state = NULL;
   }
 
-  lb_policy = chand->lb_policy;
-  if (lb_policy) {
-    GRPC_LB_POLICY_REF(lb_policy, "broadcast");
+  if (op->send_ping != NULL) {
+    if (chand->lb_policy == NULL) {
+      grpc_exec_ctx_enqueue(exec_ctx, op->send_ping, 0);
+    } else {
+      grpc_lb_policy_ping_one(exec_ctx, chand->lb_policy, op->send_ping);
+      op->bind_pollset = NULL;
+    }
+    op->send_ping = NULL;
   }
 
   if (op->disconnect && chand->resolver != NULL) {
@@ -265,7 +282,9 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
     destroy_resolver = chand->resolver;
     chand->resolver = NULL;
     if (chand->lb_policy != NULL) {
-      grpc_lb_policy_shutdown(exec_ctx, chand->lb_policy);
+      grpc_pollset_set_del_pollset_set(exec_ctx,
+                                       &chand->lb_policy->interested_parties,
+                                       &chand->interested_parties);
       GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
       chand->lb_policy = NULL;
     }
@@ -276,16 +295,11 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
     grpc_resolver_shutdown(exec_ctx, destroy_resolver);
     GRPC_RESOLVER_UNREF(exec_ctx, destroy_resolver, "channel");
   }
-
-  if (lb_policy) {
-    grpc_lb_policy_broadcast(exec_ctx, lb_policy, op);
-    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "broadcast");
-  }
 }
 
 typedef struct {
   grpc_metadata_batch *initial_metadata;
-  grpc_subchannel **subchannel;
+  grpc_connected_subchannel **connected_subchannel;
   grpc_closure *on_ready;
   grpc_call_element *elem;
   grpc_closure closure;
@@ -293,17 +307,17 @@ typedef struct {
 
 static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
                               grpc_metadata_batch *initial_metadata,
-                              grpc_subchannel **subchannel,
+                              grpc_connected_subchannel **connected_subchannel,
                               grpc_closure *on_ready);
 
 static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg, int success) {
   continue_picking_args *cpa = arg;
   if (!success) {
     grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, 0);
-  } else if (cpa->subchannel == NULL) {
+  } else if (cpa->connected_subchannel == NULL) {
     /* cancelled, do nothing */
   } else if (cc_pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
-                                cpa->subchannel, cpa->on_ready)) {
+                                cpa->connected_subchannel, cpa->on_ready)) {
     grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, 1);
   }
   gpr_free(cpa);
@@ -311,7 +325,7 @@ static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg, int success) {
 
 static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
                               grpc_metadata_batch *initial_metadata,
-                              grpc_subchannel **subchannel,
+                              grpc_connected_subchannel **connected_subchannel,
                               grpc_closure *on_ready) {
   grpc_call_element *elem = elemp;
   channel_data *chand = elem->channel_data;
@@ -319,18 +333,19 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
   continue_picking_args *cpa;
   grpc_closure *closure;
 
-  GPR_ASSERT(subchannel);
+  GPR_ASSERT(connected_subchannel);
 
   gpr_mu_lock(&chand->mu_config);
   if (initial_metadata == NULL) {
     if (chand->lb_policy != NULL) {
-      grpc_lb_policy_cancel_pick(exec_ctx, chand->lb_policy, subchannel);
+      grpc_lb_policy_cancel_pick(exec_ctx, chand->lb_policy,
+                                 connected_subchannel);
     }
     for (closure = chand->waiting_for_config_closures.head; closure != NULL;
          closure = grpc_closure_next(closure)) {
       cpa = closure->cb_arg;
-      if (cpa->subchannel == subchannel) {
-        cpa->subchannel = NULL;
+      if (cpa->connected_subchannel == connected_subchannel) {
+        cpa->connected_subchannel = NULL;
         grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, 0);
       }
     }
@@ -338,21 +353,22 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
     return 1;
   }
   if (chand->lb_policy != NULL) {
-    int r = grpc_lb_policy_pick(exec_ctx, chand->lb_policy, calld->pollset,
-                                initial_metadata, subchannel, on_ready);
+    int r =
+        grpc_lb_policy_pick(exec_ctx, chand->lb_policy, calld->pollset,
+                            initial_metadata, connected_subchannel, on_ready);
     gpr_mu_unlock(&chand->mu_config);
     return r;
   }
   if (chand->resolver != NULL && !chand->started_resolving) {
     chand->started_resolving = 1;
-    GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+    GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
     grpc_resolver_next(exec_ctx, chand->resolver,
                        &chand->incoming_configuration,
                        &chand->on_config_changed);
   }
   cpa = gpr_malloc(sizeof(*cpa));
   cpa->initial_metadata = initial_metadata;
-  cpa->subchannel = subchannel;
+  cpa->connected_subchannel = connected_subchannel;
   cpa->on_ready = on_ready;
   cpa->elem = elem;
   grpc_closure_init(&cpa->closure, continue_picking, cpa);
@@ -364,7 +380,8 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
 /* Constructor for call_data */
 static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                            grpc_call_element_args *args) {
-  grpc_subchannel_call_holder_init(elem->call_data, cc_pick_subchannel, elem);
+  grpc_subchannel_call_holder_init(elem->call_data, cc_pick_subchannel, elem,
+                                   args->call_stack);
 }
 
 /* Destructor for call_data */
@@ -385,12 +402,12 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
 
   gpr_mu_init(&chand->mu_config);
-  chand->master = args->master;
-  grpc_pollset_set_init(&chand->pollset_set);
   grpc_closure_init(&chand->on_config_changed, cc_on_config_changed, chand);
+  chand->owning_stack = args->channel_stack;
 
   grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
                                "client_channel");
+  grpc_pollset_set_init(&chand->interested_parties);
 }
 
 /* Destructor for channel_data */
@@ -403,10 +420,13 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
     GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
   }
   if (chand->lb_policy != NULL) {
+    grpc_pollset_set_del_pollset_set(exec_ctx,
+                                     &chand->lb_policy->interested_parties,
+                                     &chand->interested_parties);
     GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
   }
   grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
-  grpc_pollset_set_destroy(&chand->pollset_set);
+  grpc_pollset_set_destroy(&chand->interested_parties);
   gpr_mu_destroy(&chand->mu_config);
 }
 
@@ -435,7 +455,7 @@ void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx,
   if (!grpc_closure_list_empty(chand->waiting_for_config_closures) ||
       chand->exit_idle_when_lb_policy_arrives) {
     chand->started_resolving = 1;
-    GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+    GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
     grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration,
                        &chand->on_config_changed);
   }
@@ -454,7 +474,7 @@ grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     } else {
       chand->exit_idle_when_lb_policy_arrives = 1;
       if (!chand->started_resolving && chand->resolver != NULL) {
-        GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+        GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
         chand->started_resolving = 1;
         grpc_resolver_next(exec_ctx, chand->resolver,
                            &chand->incoming_configuration,
@@ -466,32 +486,39 @@ grpc_connectivity_state grpc_client_channel_check_connectivity_state(
   return out;
 }
 
+typedef struct {
+  channel_data *chand;
+  grpc_pollset *pollset;
+  grpc_closure *on_complete;
+  grpc_closure my_closure;
+} external_connectivity_watcher;
+
+static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg,
+                                       int iomgr_success) {
+  external_connectivity_watcher *w = arg;
+  grpc_closure *follow_up = w->on_complete;
+  grpc_pollset_set_del_pollset(exec_ctx, &w->chand->interested_parties,
+                               w->pollset);
+  GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack,
+                           "external_connectivity_watcher");
+  gpr_free(w);
+  follow_up->cb(exec_ctx, follow_up->cb_arg, iomgr_success);
+}
+
 void grpc_client_channel_watch_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
+    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
     grpc_connectivity_state *state, grpc_closure *on_complete) {
   channel_data *chand = elem->channel_data;
+  external_connectivity_watcher *w = gpr_malloc(sizeof(*w));
+  w->chand = chand;
+  w->pollset = pollset;
+  w->on_complete = on_complete;
+  grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, pollset);
+  grpc_closure_init(&w->my_closure, on_external_watch_complete, w);
+  GRPC_CHANNEL_STACK_REF(w->chand->owning_stack,
+                         "external_connectivity_watcher");
   gpr_mu_lock(&chand->mu_config);
   grpc_connectivity_state_notify_on_state_change(
-      exec_ctx, &chand->state_tracker, state, on_complete);
+      exec_ctx, &chand->state_tracker, state, &w->my_closure);
   gpr_mu_unlock(&chand->mu_config);
 }
-
-grpc_pollset_set *grpc_client_channel_get_connecting_pollset_set(
-    grpc_channel_element *elem) {
-  channel_data *chand = elem->channel_data;
-  return &chand->pollset_set;
-}
-
-void grpc_client_channel_add_interested_party(grpc_exec_ctx *exec_ctx,
-                                              grpc_channel_element *elem,
-                                              grpc_pollset *pollset) {
-  channel_data *chand = elem->channel_data;
-  grpc_pollset_set_add_pollset(exec_ctx, &chand->pollset_set, pollset);
-}
-
-void grpc_client_channel_del_interested_party(grpc_exec_ctx *exec_ctx,
-                                              grpc_channel_element *elem,
-                                              grpc_pollset *pollset) {
-  channel_data *chand = elem->channel_data;
-  grpc_pollset_set_del_pollset(exec_ctx, &chand->pollset_set, pollset);
-}
diff --git a/src/core/channel/client_channel.h b/src/core/channel/client_channel.h
index 5103f07a43..d9bc4971f1 100644
--- a/src/core/channel/client_channel.h
+++ b/src/core/channel/client_channel.h
@@ -57,17 +57,7 @@ grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect);
 
 void grpc_client_channel_watch_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
+    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
     grpc_connectivity_state *state, grpc_closure *on_complete);
 
-grpc_pollset_set *grpc_client_channel_get_connecting_pollset_set(
-    grpc_channel_element *elem);
-
-void grpc_client_channel_add_interested_party(grpc_exec_ctx *exec_ctx,
-                                              grpc_channel_element *channel,
-                                              grpc_pollset *pollset);
-void grpc_client_channel_del_interested_party(grpc_exec_ctx *exec_ctx,
-                                              grpc_channel_element *channel,
-                                              grpc_pollset *pollset);
-
 #endif /* GRPC_INTERNAL_CORE_CHANNEL_CLIENT_CHANNEL_H */
diff --git a/src/core/channel/client_uchannel.c b/src/core/channel/client_uchannel.c
index 456ffb7371..2c0b07d8bf 100644
--- a/src/core/channel/client_uchannel.c
+++ b/src/core/channel/client_uchannel.c
@@ -58,13 +58,13 @@ typedef struct client_uchannel_channel_data {
       this channel_data via its channel stack.
       We occasionally use this to bump the refcount on the master channel
       to keep ourselves alive through an asynchronous operation. */
-  grpc_channel *master;
+  grpc_channel_stack *owning_stack;
 
   /** connectivity state being tracked */
   grpc_connectivity_state_tracker state_tracker;
 
   /** the subchannel wrapped by the microchannel */
-  grpc_subchannel *subchannel;
+  grpc_connected_subchannel *connected_subchannel;
 
   /** the callback used to stay subscribed to subchannel connectivity
    * notifications */
@@ -84,15 +84,13 @@ static void monitor_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
                               chand->subchannel_connectivity,
                               "uchannel_monitor_subchannel");
-  grpc_subchannel_notify_on_state_change(exec_ctx, chand->subchannel,
-                                         &chand->subchannel_connectivity,
-                                         &chand->connectivity_cb);
+  grpc_connected_subchannel_notify_on_state_change(
+      exec_ctx, chand->connected_subchannel, NULL,
+      &chand->subchannel_connectivity, &chand->connectivity_cb);
 }
 
 static char *cuc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  channel_data *chand = elem->channel_data;
-  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data,
-                                              chand->master);
+  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data);
 }
 
 static void cuc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
@@ -128,11 +126,11 @@ static void cuc_start_transport_op(grpc_exec_ctx *exec_ctx,
 
 static int cuc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
                                grpc_metadata_batch *initial_metadata,
-                               grpc_subchannel **subchannel,
+                               grpc_connected_subchannel **connected_subchannel,
                                grpc_closure *on_ready) {
   channel_data *chand = arg;
   GPR_ASSERT(initial_metadata != NULL);
-  *subchannel = chand->subchannel;
+  *connected_subchannel = chand->connected_subchannel;
   return 1;
 }
 
@@ -140,7 +138,7 @@ static int cuc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
 static void cuc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                                grpc_call_element_args *args) {
   grpc_subchannel_call_holder_init(elem->call_data, cuc_pick_subchannel,
-                                   elem->channel_data);
+                                   elem->channel_data, args->call_stack);
 }
 
 /* Destructor for call_data */
@@ -158,7 +156,7 @@ static void cuc_init_channel_elem(grpc_exec_ctx *exec_ctx,
   grpc_closure_init(&chand->connectivity_cb, monitor_subchannel, chand);
   GPR_ASSERT(args->is_last);
   GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter);
-  chand->master = args->master;
+  chand->owning_stack = args->channel_stack;
   grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
                                "client_uchannel");
   gpr_mu_init(&chand->mu_state);
@@ -168,10 +166,14 @@ static void cuc_init_channel_elem(grpc_exec_ctx *exec_ctx,
 static void cuc_destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_element *elem) {
   channel_data *chand = elem->channel_data;
-  grpc_subchannel_state_change_unsubscribe(exec_ctx, chand->subchannel,
-                                           &chand->connectivity_cb);
+  /* cancel subscription */
+  grpc_connected_subchannel_notify_on_state_change(
+      exec_ctx, chand->connected_subchannel, NULL, NULL,
+      &chand->connectivity_cb);
   grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
   gpr_mu_destroy(&chand->mu_state);
+  GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, chand->connected_subchannel,
+                                  "uchannel");
 }
 
 static void cuc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
@@ -191,23 +193,14 @@ grpc_connectivity_state grpc_client_uchannel_check_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect) {
   channel_data *chand = elem->channel_data;
   grpc_connectivity_state out;
-  out = grpc_connectivity_state_check(&chand->state_tracker);
   gpr_mu_lock(&chand->mu_state);
-  if (out == GRPC_CHANNEL_IDLE && try_to_connect) {
-    grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
-                                GRPC_CHANNEL_CONNECTING,
-                                "uchannel_connecting_changed");
-    chand->subchannel_connectivity = out;
-    grpc_subchannel_notify_on_state_change(exec_ctx, chand->subchannel,
-                                           &chand->subchannel_connectivity,
-                                           &chand->connectivity_cb);
-  }
+  out = grpc_connectivity_state_check(&chand->state_tracker);
   gpr_mu_unlock(&chand->mu_state);
   return out;
 }
 
 void grpc_client_uchannel_watch_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
+    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
     grpc_connectivity_state *state, grpc_closure *on_complete) {
   channel_data *chand = elem->channel_data;
   gpr_mu_lock(&chand->mu_state);
@@ -216,40 +209,11 @@ void grpc_client_uchannel_watch_connectivity_state(
   gpr_mu_unlock(&chand->mu_state);
 }
 
-grpc_pollset_set *grpc_client_uchannel_get_connecting_pollset_set(
-    grpc_channel_element *elem) {
-  channel_data *chand = elem->channel_data;
-  grpc_channel_element *parent_elem;
-  gpr_mu_lock(&chand->mu_state);
-  parent_elem = grpc_channel_stack_last_element(grpc_channel_get_channel_stack(
-      grpc_subchannel_get_master(chand->subchannel)));
-  gpr_mu_unlock(&chand->mu_state);
-  return grpc_client_channel_get_connecting_pollset_set(parent_elem);
-}
-
-void grpc_client_uchannel_add_interested_party(grpc_exec_ctx *exec_ctx,
-                                               grpc_channel_element *elem,
-                                               grpc_pollset *pollset) {
-  grpc_pollset_set *master_pollset_set =
-      grpc_client_uchannel_get_connecting_pollset_set(elem);
-  grpc_pollset_set_add_pollset(exec_ctx, master_pollset_set, pollset);
-}
-
-void grpc_client_uchannel_del_interested_party(grpc_exec_ctx *exec_ctx,
-                                               grpc_channel_element *elem,
-                                               grpc_pollset *pollset) {
-  grpc_pollset_set *master_pollset_set =
-      grpc_client_uchannel_get_connecting_pollset_set(elem);
-  grpc_pollset_set_del_pollset(exec_ctx, master_pollset_set, pollset);
-}
-
 grpc_channel *grpc_client_uchannel_create(grpc_subchannel *subchannel,
                                           grpc_channel_args *args) {
   grpc_channel *channel = NULL;
 #define MAX_FILTERS 3
   const grpc_channel_filter *filters[MAX_FILTERS];
-  grpc_channel *master = grpc_subchannel_get_master(subchannel);
-  char *target = grpc_channel_get_target(master);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   size_t n = 0;
 
@@ -261,19 +225,19 @@ grpc_channel *grpc_client_uchannel_create(grpc_subchannel *subchannel,
   GPR_ASSERT(n <= MAX_FILTERS);
 
   channel =
-      grpc_channel_create_from_filters(&exec_ctx, target, filters, n, args, 1);
+      grpc_channel_create_from_filters(&exec_ctx, NULL, filters, n, args, 1);
 
-  gpr_free(target);
   return channel;
 }
 
-void grpc_client_uchannel_set_subchannel(grpc_channel *uchannel,
-                                         grpc_subchannel *subchannel) {
+void grpc_client_uchannel_set_connected_subchannel(
+    grpc_channel *uchannel, grpc_connected_subchannel *connected_subchannel) {
   grpc_channel_element *elem =
       grpc_channel_stack_last_element(grpc_channel_get_channel_stack(uchannel));
   channel_data *chand = elem->channel_data;
   GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter);
   gpr_mu_lock(&chand->mu_state);
-  chand->subchannel = subchannel;
+  chand->connected_subchannel = connected_subchannel;
+  GRPC_CONNECTED_SUBCHANNEL_REF(connected_subchannel, "uchannel");
   gpr_mu_unlock(&chand->mu_state);
 }
diff --git a/src/core/channel/client_uchannel.h b/src/core/channel/client_uchannel.h
index dfe6695ae3..92a831493c 100644
--- a/src/core/channel/client_uchannel.h
+++ b/src/core/channel/client_uchannel.h
@@ -48,23 +48,13 @@ grpc_connectivity_state grpc_client_uchannel_check_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect);
 
 void grpc_client_uchannel_watch_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
+    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
     grpc_connectivity_state *state, grpc_closure *on_complete);
 
-grpc_pollset_set *grpc_client_uchannel_get_connecting_pollset_set(
-    grpc_channel_element *elem);
-
-void grpc_client_uchannel_add_interested_party(grpc_exec_ctx *exec_ctx,
-                                               grpc_channel_element *channel,
-                                               grpc_pollset *pollset);
-void grpc_client_uchannel_del_interested_party(grpc_exec_ctx *exec_ctx,
-                                               grpc_channel_element *channel,
-                                               grpc_pollset *pollset);
-
 grpc_channel *grpc_client_uchannel_create(grpc_subchannel *subchannel,
                                           grpc_channel_args *args);
 
-void grpc_client_uchannel_set_subchannel(grpc_channel *uchannel,
-                                         grpc_subchannel *subchannel);
+void grpc_client_uchannel_set_connected_subchannel(
+    grpc_channel *uchannel, grpc_connected_subchannel *connected_subchannel);
 
 #endif /* GRPC_INTERNAL_CORE_CHANNEL_CLIENT_MICROCHANNEL_H */
diff --git a/src/core/channel/compress_filter.c b/src/core/channel/compress_filter.c
index d7d1c189fe..cc8e191628 100644
--- a/src/core/channel/compress_filter.c
+++ b/src/core/channel/compress_filter.c
@@ -288,8 +288,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
 
 /* Destructor for channel data */
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
-                                 grpc_channel_element *elem) {
-}
+                                 grpc_channel_element *elem) {}
 
 const grpc_channel_filter grpc_compress_filter = {
     compress_start_transport_stream_op, grpc_channel_next_op, sizeof(call_data),
diff --git a/src/core/channel/connected_channel.c b/src/core/channel/connected_channel.c
index 0e1efd965a..e8eb9dcfc5 100644
--- a/src/core/channel/connected_channel.c
+++ b/src/core/channel/connected_channel.c
@@ -89,9 +89,9 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   int r;
 
   GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
-  r = grpc_transport_init_stream(exec_ctx, chand->transport,
-                                 TRANSPORT_STREAM_FROM_CALL_DATA(calld),
-                                 args->refcount, args->server_transport_data);
+  r = grpc_transport_init_stream(
+      exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld),
+      &args->call_stack->refcount, args->server_transport_data);
   GPR_ASSERT(r == 0);
 }
 
diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c
index bc3a56cbf0..ae8660da92 100644
--- a/src/core/channel/http_server_filter.c
+++ b/src/core/channel/http_server_filter.c
@@ -224,8 +224,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
 
 /* Destructor for channel data */
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
-                                 grpc_channel_element *elem) {
-}
+                                 grpc_channel_element *elem) {}
 
 const grpc_channel_filter grpc_http_server_filter = {
     hs_start_transport_op, grpc_channel_next_op, sizeof(call_data),
diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c
index 7251714519..f5da41f3cd 100644
--- a/src/core/channel/subchannel_call_holder.c
+++ b/src/core/channel/subchannel_call_holder.c
@@ -44,7 +44,6 @@
 
 static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *holder,
                              int success);
-static void call_ready(grpc_exec_ctx *exec_ctx, void *holder, int success);
 static void retry_ops(grpc_exec_ctx *exec_ctx, void *retry_ops_args,
                       int success);
 
@@ -58,16 +57,17 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
 void grpc_subchannel_call_holder_init(
     grpc_subchannel_call_holder *holder,
     grpc_subchannel_call_holder_pick_subchannel pick_subchannel,
-    void *pick_subchannel_arg) {
+    void *pick_subchannel_arg, grpc_call_stack *owning_call) {
   gpr_atm_rel_store(&holder->subchannel_call, 0);
   holder->pick_subchannel = pick_subchannel;
   holder->pick_subchannel_arg = pick_subchannel_arg;
   gpr_mu_init(&holder->mu);
-  holder->subchannel = NULL;
+  holder->connected_subchannel = NULL;
   holder->waiting_ops = NULL;
   holder->waiting_ops_count = 0;
   holder->waiting_ops_capacity = 0;
   holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  holder->owning_call = owning_call;
 }
 
 void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
@@ -125,13 +125,9 @@ retry:
         case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING:
           fail_locked(exec_ctx, holder);
           break;
-        case GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL:
-          grpc_subchannel_cancel_create_call(exec_ctx, holder->subchannel,
-                                             &holder->subchannel_call);
-          break;
         case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL:
           holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, NULL,
-                                  &holder->subchannel, NULL);
+                                  &holder->connected_subchannel, NULL);
           break;
       }
       gpr_mu_unlock(&holder->mu);
@@ -142,28 +138,27 @@ retry:
   }
   /* if we don't have a subchannel, try to get one */
   if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
-      holder->subchannel == NULL && op->send_initial_metadata != NULL) {
+      holder->connected_subchannel == NULL &&
+      op->send_initial_metadata != NULL) {
     holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL;
     grpc_closure_init(&holder->next_step, subchannel_ready, holder);
-    if (holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg,
-                                op->send_initial_metadata, &holder->subchannel,
-                                &holder->next_step)) {
+    GRPC_CALL_STACK_REF(holder->owning_call, "pick_subchannel");
+    if (holder->pick_subchannel(
+            exec_ctx, holder->pick_subchannel_arg, op->send_initial_metadata,
+            &holder->connected_subchannel, &holder->next_step)) {
       holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+      GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
     }
   }
   /* if we've got a subchannel, then let's ask it to create a call */
   if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING &&
-      holder->subchannel != NULL) {
-    holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL;
-    grpc_closure_init(&holder->next_step, call_ready, holder);
-    if (grpc_subchannel_create_call(exec_ctx, holder->subchannel,
-                                    holder->pollset, &holder->subchannel_call,
-                                    &holder->next_step)) {
-      /* got one immediately - continue the op (and any waiting ops) */
-      holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-      retry_waiting_locked(exec_ctx, holder);
-      goto retry;
-    }
+      holder->connected_subchannel != NULL) {
+    gpr_atm_rel_store(
+        &holder->subchannel_call,
+        (gpr_atm)(gpr_uintptr)grpc_connected_subchannel_create_call(
+            exec_ctx, holder->connected_subchannel, holder->pollset));
+    retry_waiting_locked(exec_ctx, holder);
+    goto retry;
   }
   /* nothing to be done but wait */
   add_waiting_locked(holder, op);
@@ -179,36 +174,18 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, int success) {
              GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL);
   call = GET_CALL(holder);
   GPR_ASSERT(call == NULL || call == CANCELLED_CALL);
-  if (holder->subchannel == NULL) {
-    holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
+  if (holder->connected_subchannel == NULL) {
     fail_locked(exec_ctx, holder);
   } else {
-    grpc_closure_init(&holder->next_step, call_ready, holder);
-    if (grpc_subchannel_create_call(exec_ctx, holder->subchannel,
-                                    holder->pollset, &holder->subchannel_call,
-                                    &holder->next_step)) {
-      holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-      /* got one immediately - continue the op (and any waiting ops) */
-      retry_waiting_locked(exec_ctx, holder);
-    }
-  }
-  gpr_mu_unlock(&holder->mu);
-}
-
-static void call_ready(grpc_exec_ctx *exec_ctx, void *arg, int success) {
-  grpc_subchannel_call_holder *holder = arg;
-  GPR_TIMER_BEGIN("call_ready", 0);
-  gpr_mu_lock(&holder->mu);
-  GPR_ASSERT(holder->creation_phase ==
-             GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL);
-  holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
-  if (GET_CALL(holder) != NULL) {
+    gpr_atm_rel_store(
+        &holder->subchannel_call,
+        (gpr_atm)(gpr_uintptr)grpc_connected_subchannel_create_call(
+            exec_ctx, holder->connected_subchannel, holder->pollset));
     retry_waiting_locked(exec_ctx, holder);
-  } else {
-    fail_locked(exec_ctx, holder);
   }
   gpr_mu_unlock(&holder->mu);
-  GPR_TIMER_END("call_ready", 0);
+  GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
 }
 
 typedef struct {
@@ -270,14 +247,13 @@ static void fail_locked(grpc_exec_ctx *exec_ctx,
   holder->waiting_ops_count = 0;
 }
 
-char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx,
-                                           grpc_subchannel_call_holder *holder,
-                                           grpc_channel *master) {
+char *grpc_subchannel_call_holder_get_peer(
+    grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder) {
   grpc_subchannel_call *subchannel_call = GET_CALL(holder);
 
   if (subchannel_call) {
     return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
   } else {
-    return grpc_channel_get_target(master);
+    return NULL;
   }
 }
diff --git a/src/core/channel/subchannel_call_holder.h b/src/core/channel/subchannel_call_holder.h
index bda051c566..9cf72c6cf7 100644
--- a/src/core/channel/subchannel_call_holder.h
+++ b/src/core/channel/subchannel_call_holder.h
@@ -42,12 +42,11 @@
     called when the subchannel is available) */
 typedef int (*grpc_subchannel_call_holder_pick_subchannel)(
     grpc_exec_ctx *exec_ctx, void *arg, grpc_metadata_batch *initial_metadata,
-    grpc_subchannel **subchannel, grpc_closure *on_ready);
+    grpc_connected_subchannel **connected_subchannel, grpc_closure *on_ready);
 
 typedef enum {
   GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING,
-  GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL,
-  GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL
+  GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL
 } grpc_subchannel_call_holder_creation_phase;
 
 /** Wrapper for holding a pointer to grpc_subchannel_call, and the
@@ -71,7 +70,7 @@ typedef struct grpc_subchannel_call_holder {
   gpr_mu mu;
 
   grpc_subchannel_call_holder_creation_phase creation_phase;
-  grpc_subchannel *subchannel;
+  grpc_connected_subchannel *connected_subchannel;
   grpc_pollset *pollset;
 
   grpc_transport_stream_op *waiting_ops;
@@ -79,12 +78,14 @@ typedef struct grpc_subchannel_call_holder {
   size_t waiting_ops_capacity;
 
   grpc_closure next_step;
+
+  grpc_call_stack *owning_call;
 } grpc_subchannel_call_holder;
 
 void grpc_subchannel_call_holder_init(
     grpc_subchannel_call_holder *holder,
     grpc_subchannel_call_holder_pick_subchannel pick_subchannel,
-    void *pick_subchannel_arg);
+    void *pick_subchannel_arg, grpc_call_stack *owning_call);
 void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
                                          grpc_subchannel_call_holder *holder);
 
@@ -92,7 +93,6 @@ void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
                                             grpc_subchannel_call_holder *holder,
                                             grpc_transport_stream_op *op);
 char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx,
-                                           grpc_subchannel_call_holder *holder,
-                                           grpc_channel *master);
+                                           grpc_subchannel_call_holder *holder);
 
 #endif
diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c
index 93312abb00..37de3e9f68 100644
--- a/src/core/client_config/lb_policies/pick_first.c
+++ b/src/core/client_config/lb_policies/pick_first.c
@@ -42,7 +42,7 @@
 typedef struct pending_pick {
   struct pending_pick *next;
   grpc_pollset *pollset;
-  grpc_subchannel **target;
+  grpc_connected_subchannel **target;
   grpc_closure *on_complete;
 } pending_pick;
 
@@ -60,7 +60,7 @@ typedef struct {
   /** the selected channel
       TODO(ctiller): this should be atomically set so we don't
                      need to take a mutex in the common case */
-  grpc_subchannel *selected;
+  grpc_connected_subchannel *selected;
   /** have we started picking? */
   int started_picking;
   /** are we shut down? */
@@ -76,24 +76,6 @@ typedef struct {
   grpc_connectivity_state_tracker state_tracker;
 } pick_first_lb_policy;
 
-static void del_interested_parties_locked(grpc_exec_ctx *exec_ctx,
-                                          pick_first_lb_policy *p) {
-  pending_pick *pp;
-  for (pp = p->pending_picks; pp; pp = pp->next) {
-    grpc_subchannel_del_interested_party(
-        exec_ctx, p->subchannels[p->checking_subchannel], pp->pollset);
-  }
-}
-
-static void add_interested_parties_locked(grpc_exec_ctx *exec_ctx,
-                                          pick_first_lb_policy *p) {
-  pending_pick *pp;
-  for (pp = p->pending_picks; pp; pp = pp->next) {
-    grpc_subchannel_add_interested_party(
-        exec_ctx, p->subchannels[p->checking_subchannel], pp->pollset);
-  }
-}
-
 void pf_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   size_t i;
@@ -102,7 +84,7 @@ void pf_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
     GRPC_SUBCHANNEL_UNREF(exec_ctx, p->subchannels[i], "pick_first");
   }
   if (p->selected) {
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, p->selected, "picked_first");
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, p->selected, "picked_first");
   }
   grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker);
   gpr_free(p->subchannels);
@@ -114,16 +96,26 @@ void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   pending_pick *pp;
   gpr_mu_lock(&p->mu);
-  del_interested_parties_locked(exec_ctx, p);
   p->shutdown = 1;
   pp = p->pending_picks;
   p->pending_picks = NULL;
   grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                               GRPC_CHANNEL_FATAL_FAILURE, "shutdown");
+  /* cancel subscription */
+  if (p->selected != NULL) {
+    grpc_connected_subchannel_notify_on_state_change(
+        exec_ctx, p->selected, NULL, NULL, &p->connectivity_changed);
+  } else {
+    grpc_subchannel_notify_on_state_change(
+        exec_ctx, p->subchannels[p->checking_subchannel], NULL, NULL,
+        &p->connectivity_changed);
+  }
   gpr_mu_unlock(&p->mu);
   while (pp != NULL) {
     pending_pick *next = pp->next;
     *pp->target = NULL;
+    grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties,
+                                 pp->pollset);
     grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 1);
     gpr_free(pp);
     pp = next;
@@ -131,7 +123,7 @@ void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                           grpc_subchannel **target) {
+                           grpc_connected_subchannel **target) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   pending_pick *pp;
   gpr_mu_lock(&p->mu);
@@ -140,8 +132,8 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   while (pp != NULL) {
     pending_pick *next = pp->next;
     if (pp->target == target) {
-      grpc_subchannel_del_interested_party(
-          exec_ctx, p->subchannels[p->checking_subchannel], pp->pollset);
+      grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties,
+                                   pp->pollset);
       *target = NULL;
       grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 0);
       gpr_free(pp);
@@ -158,10 +150,11 @@ static void start_picking(grpc_exec_ctx *exec_ctx, pick_first_lb_policy *p) {
   p->started_picking = 1;
   p->checking_subchannel = 0;
   p->checking_connectivity = GRPC_CHANNEL_IDLE;
-  GRPC_LB_POLICY_REF(&p->base, "pick_first_connectivity");
+  GRPC_LB_POLICY_WEAK_REF(&p->base, "pick_first_connectivity");
   grpc_subchannel_notify_on_state_change(
       exec_ctx, p->subchannels[p->checking_subchannel],
-      &p->checking_connectivity, &p->connectivity_changed);
+      &p->base.interested_parties, &p->checking_connectivity,
+      &p->connectivity_changed);
 }
 
 void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
@@ -174,8 +167,8 @@ void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset,
-            grpc_metadata_batch *initial_metadata, grpc_subchannel **target,
-            grpc_closure *on_complete) {
+            grpc_metadata_batch *initial_metadata,
+            grpc_connected_subchannel **target, grpc_closure *on_complete) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   pending_pick *pp;
   gpr_mu_lock(&p->mu);
@@ -187,8 +180,8 @@ int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_subchannel_add_interested_party(
-        exec_ctx, p->subchannels[p->checking_subchannel], pollset);
+    grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties,
+                                 pollset);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
     pp->pollset = pollset;
@@ -204,25 +197,17 @@ static void destroy_subchannels(grpc_exec_ctx *exec_ctx, void *arg,
                                 int iomgr_success) {
   pick_first_lb_policy *p = arg;
   size_t i;
-  grpc_transport_op op;
   size_t num_subchannels = p->num_subchannels;
   grpc_subchannel **subchannels;
-  grpc_subchannel *exclude_subchannel;
 
   gpr_mu_lock(&p->mu);
   subchannels = p->subchannels;
   p->num_subchannels = 0;
   p->subchannels = NULL;
-  exclude_subchannel = p->selected;
   gpr_mu_unlock(&p->mu);
-  GRPC_LB_POLICY_UNREF(exec_ctx, &p->base, "destroy_subchannels");
+  GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "destroy_subchannels");
 
   for (i = 0; i < num_subchannels; i++) {
-    if (subchannels[i] != exclude_subchannel) {
-      memset(&op, 0, sizeof(op));
-      op.disconnect = 1;
-      grpc_subchannel_process_transport_op(exec_ctx, subchannels[i], &op);
-    }
     GRPC_SUBCHANNEL_UNREF(exec_ctx, subchannels[i], "pick_first");
   }
 
@@ -232,23 +217,28 @@ static void destroy_subchannels(grpc_exec_ctx *exec_ctx, void *arg,
 static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                                     int iomgr_success) {
   pick_first_lb_policy *p = arg;
+  grpc_subchannel *selected_subchannel;
   pending_pick *pp;
 
   gpr_mu_lock(&p->mu);
 
   if (p->shutdown) {
     gpr_mu_unlock(&p->mu);
-    GRPC_LB_POLICY_UNREF(exec_ctx, &p->base, "pick_first_connectivity");
+    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pick_first_connectivity");
     return;
   } else if (p->selected != NULL) {
+    if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      /* if the selected channel goes bad, we're done */
+      p->checking_connectivity = GRPC_CHANNEL_FATAL_FAILURE;
+    }
     grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                 p->checking_connectivity, "selected_changed");
     if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) {
-      grpc_subchannel_notify_on_state_change(exec_ctx, p->selected,
-                                             &p->checking_connectivity,
-                                             &p->connectivity_changed);
+      grpc_connected_subchannel_notify_on_state_change(
+          exec_ctx, p->selected, &p->base.interested_parties,
+          &p->checking_connectivity, &p->connectivity_changed);
     } else {
-      GRPC_LB_POLICY_UNREF(exec_ctx, &p->base, "pick_first_connectivity");
+      GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pick_first_connectivity");
     }
   } else {
   loop:
@@ -256,39 +246,41 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
       case GRPC_CHANNEL_READY:
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                     GRPC_CHANNEL_READY, "connecting_ready");
-        p->selected = p->subchannels[p->checking_subchannel];
-        GRPC_SUBCHANNEL_REF(p->selected, "picked_first");
+        selected_subchannel = p->subchannels[p->checking_subchannel];
+        p->selected =
+            grpc_subchannel_get_connected_subchannel(selected_subchannel);
+        GPR_ASSERT(p->selected);
+        GRPC_CONNECTED_SUBCHANNEL_REF(p->selected, "picked_first");
         /* drop the pick list: we are connected now */
-        GRPC_LB_POLICY_REF(&p->base, "destroy_subchannels");
+        GRPC_LB_POLICY_WEAK_REF(&p->base, "destroy_subchannels");
         grpc_exec_ctx_enqueue(exec_ctx,
                               grpc_closure_create(destroy_subchannels, p), 1);
         /* update any calls that were waiting for a pick */
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
           *pp->target = p->selected;
-          grpc_subchannel_del_interested_party(exec_ctx, p->selected,
-                                               pp->pollset);
+          grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties,
+                                       pp->pollset);
           grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 1);
           gpr_free(pp);
         }
-        grpc_subchannel_notify_on_state_change(exec_ctx, p->selected,
-                                               &p->checking_connectivity,
-                                               &p->connectivity_changed);
+        grpc_connected_subchannel_notify_on_state_change(
+            exec_ctx, p->selected, &p->base.interested_parties,
+            &p->checking_connectivity, &p->connectivity_changed);
         break;
       case GRPC_CHANNEL_TRANSIENT_FAILURE:
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                     GRPC_CHANNEL_TRANSIENT_FAILURE,
                                     "connecting_transient_failure");
-        del_interested_parties_locked(exec_ctx, p);
         p->checking_subchannel =
             (p->checking_subchannel + 1) % p->num_subchannels;
         p->checking_connectivity = grpc_subchannel_check_connectivity(
             p->subchannels[p->checking_subchannel]);
-        add_interested_parties_locked(exec_ctx, p);
         if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) {
           grpc_subchannel_notify_on_state_change(
               exec_ctx, p->subchannels[p->checking_subchannel],
-              &p->checking_connectivity, &p->connectivity_changed);
+              &p->base.interested_parties, &p->checking_connectivity,
+              &p->connectivity_changed);
         } else {
           goto loop;
         }
@@ -300,13 +292,13 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                                     "connecting_changed");
         grpc_subchannel_notify_on_state_change(
             exec_ctx, p->subchannels[p->checking_subchannel],
-            &p->checking_connectivity, &p->connectivity_changed);
+            &p->base.interested_parties, &p->checking_connectivity,
+            &p->connectivity_changed);
         break;
       case GRPC_CHANNEL_FATAL_FAILURE:
-        del_interested_parties_locked(exec_ctx, p);
-        GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel],
-                 p->subchannels[p->num_subchannels - 1]);
         p->num_subchannels--;
+        GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel],
+                 p->subchannels[p->num_subchannels]);
         GRPC_SUBCHANNEL_UNREF(exec_ctx, p->subchannels[p->num_subchannels],
                               "pick_first");
         if (p->num_subchannels == 0) {
@@ -319,7 +311,8 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
             grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 1);
             gpr_free(pp);
           }
-          GRPC_LB_POLICY_UNREF(exec_ctx, &p->base, "pick_first_connectivity");
+          GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base,
+                                    "pick_first_connectivity");
         } else {
           grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                       GRPC_CHANNEL_TRANSIENT_FAILURE,
@@ -327,7 +320,6 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
           p->checking_subchannel %= p->num_subchannels;
           p->checking_connectivity = grpc_subchannel_check_connectivity(
               p->subchannels[p->checking_subchannel]);
-          add_interested_parties_locked(exec_ctx, p);
           goto loop;
         }
     }
@@ -336,39 +328,6 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_mu_unlock(&p->mu);
 }
 
-static void pf_broadcast(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                         grpc_transport_op *op) {
-  pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
-  size_t i;
-  size_t n;
-  grpc_subchannel **subchannels;
-  grpc_subchannel *selected;
-
-  gpr_mu_lock(&p->mu);
-  n = p->num_subchannels;
-  subchannels = gpr_malloc(n * sizeof(*subchannels));
-  selected = p->selected;
-  if (selected) {
-    GRPC_SUBCHANNEL_REF(selected, "pf_broadcast_to_selected");
-  }
-  for (i = 0; i < n; i++) {
-    subchannels[i] = p->subchannels[i];
-    GRPC_SUBCHANNEL_REF(subchannels[i], "pf_broadcast");
-  }
-  gpr_mu_unlock(&p->mu);
-
-  for (i = 0; i < n; i++) {
-    if (selected == subchannels[i]) continue;
-    grpc_subchannel_process_transport_op(exec_ctx, subchannels[i], op);
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, subchannels[i], "pf_broadcast");
-  }
-  if (p->selected) {
-    grpc_subchannel_process_transport_op(exec_ctx, selected, op);
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, selected, "pf_broadcast_to_selected");
-  }
-  gpr_free(subchannels);
-}
-
 static grpc_connectivity_state pf_check_connectivity(grpc_exec_ctx *exec_ctx,
                                                      grpc_lb_policy *pol) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
@@ -389,9 +348,21 @@ void pf_notify_on_state_change(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   gpr_mu_unlock(&p->mu);
 }
 
+void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
+                 grpc_closure *closure) {
+  pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
+  gpr_mu_lock(&p->mu);
+  if (p->selected) {
+    grpc_connected_subchannel_ping(exec_ctx, p->selected, closure);
+  } else {
+    grpc_exec_ctx_enqueue(exec_ctx, closure, 0);
+  }
+  gpr_mu_unlock(&p->mu);
+}
+
 static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = {
-    pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_exit_idle,
-    pf_broadcast, pf_check_connectivity, pf_notify_on_state_change};
+    pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_ping_one, pf_exit_idle,
+    pf_check_connectivity, pf_notify_on_state_change};
 
 static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {}
 
diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/client_config/lb_policies/round_robin.c
index 1ffe32fff2..d487456363 100644
--- a/src/core/client_config/lb_policies/round_robin.c
+++ b/src/core/client_config/lb_policies/round_robin.c
@@ -38,6 +38,8 @@
 #include <grpc/support/alloc.h>
 #include "src/core/transport/connectivity_state.h"
 
+typedef struct round_robin_lb_policy round_robin_lb_policy;
+
 int grpc_lb_round_robin_trace = 0;
 
 /** List of entities waiting for a pick.
@@ -46,7 +48,7 @@ int grpc_lb_round_robin_trace = 0;
 typedef struct pending_pick {
   struct pending_pick *next;
   grpc_pollset *pollset;
-  grpc_subchannel **target;
+  grpc_connected_subchannel **target;
   grpc_closure *on_complete;
 } pending_pick;
 
@@ -58,22 +60,27 @@ typedef struct ready_list {
 } ready_list;
 
 typedef struct {
-  size_t subchannel_idx; /**< Index over p->subchannels */
-  void *p;               /**< round_robin_lb_policy instance */
-} connectivity_changed_cb_arg;
-
-typedef struct {
+  /** index within policy->subchannels */
+  size_t index;
+  /** backpointer to owning policy */
+  round_robin_lb_policy *policy;
+  /** subchannel itself */
+  grpc_subchannel *subchannel;
+  /** notification that connectivity has changed on subchannel */
+  grpc_closure connectivity_changed_closure;
+  /** this subchannels current position in subchannel->ready_list */
+  ready_list *ready_list_node;
+  /** last observed connectivity */
+  grpc_connectivity_state connectivity_state;
+} subchannel_data;
+
+struct round_robin_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
 
   /** all our subchannels */
-  grpc_subchannel **subchannels;
   size_t num_subchannels;
-
-  /** Callbacks, one per subchannel being watched, to be called when their
-   * respective connectivity changes */
-  grpc_closure *connectivity_changed_cbs;
-  connectivity_changed_cb_arg *cb_args;
+  subchannel_data **subchannels;
 
   /** mutex protecting remaining members */
   gpr_mu mu;
@@ -81,8 +88,6 @@ typedef struct {
   int started_picking;
   /** are we shutting down? */
   int shutdown;
-  /** Connectivity state of the subchannels being watched */
-  grpc_connectivity_state *subchannel_connectivity;
   /** List of picks that are waiting on connectivity */
   pending_pick *pending_picks;
 
@@ -93,13 +98,7 @@ typedef struct {
   ready_list ready_list;
   /** Last pick from the ready list. */
   ready_list *ready_list_last_pick;
-
-  /** Subchannel index to ready_list node.
-   *
-   * Kept in order to remove nodes from the ready list associated with a
-   * subchannel */
-  ready_list **subchannel_index_to_readylist_node;
-} round_robin_lb_policy;
+};
 
 /** Returns the next subchannel from the connected list or NULL if the list is
  * empty.
@@ -144,9 +143,9 @@ static void advance_last_picked_locked(round_robin_lb_policy *p) {
 /** Prepends (relative to the root at p->ready_list) the connected subchannel \a
  * csc to the list of ready subchannels. */
 static ready_list *add_connected_sc_locked(round_robin_lb_policy *p,
-                                           grpc_subchannel *csc) {
+                                           grpc_subchannel *sc) {
   ready_list *new_elem = gpr_malloc(sizeof(ready_list));
-  new_elem->subchannel = csc;
+  new_elem->subchannel = sc;
   if (p->ready_list.prev == NULL) {
     /* first element */
     new_elem->next = &p->ready_list;
@@ -160,7 +159,7 @@ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p,
     p->ready_list.prev = new_elem;
   }
   if (grpc_lb_round_robin_trace) {
-    gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (SC %p)", new_elem, csc);
+    gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (SC %p)", new_elem, sc);
   }
   return new_elem;
 }
@@ -200,28 +199,15 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p,
   gpr_free(node);
 }
 
-static void del_interested_parties_locked(grpc_exec_ctx *exec_ctx,
-                                          round_robin_lb_policy *p,
-                                          const size_t subchannel_idx) {
-  pending_pick *pp;
-  for (pp = p->pending_picks; pp; pp = pp->next) {
-    grpc_subchannel_del_interested_party(
-        exec_ctx, p->subchannels[subchannel_idx], pp->pollset);
-  }
-}
-
 void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   size_t i;
   ready_list *elem;
   for (i = 0; i < p->num_subchannels; i++) {
-    del_interested_parties_locked(exec_ctx, p, i);
-  }
-  for (i = 0; i < p->num_subchannels; i++) {
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, p->subchannels[i], "round_robin");
+    subchannel_data *sd = p->subchannels[i];
+    GRPC_SUBCHANNEL_UNREF(exec_ctx, sd->subchannel, "round_robin");
+    gpr_free(sd);
   }
-  gpr_free(p->connectivity_changed_cbs);
-  gpr_free(p->subchannel_connectivity);
 
   grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker);
   gpr_free(p->subchannels);
@@ -237,20 +223,15 @@ void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
     gpr_free(elem);
     elem = tmp;
   }
-  gpr_free(p->subchannel_index_to_readylist_node);
-  gpr_free(p->cb_args);
   gpr_free(p);
 }
 
 void rr_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
-  size_t i;
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   pending_pick *pp;
-  gpr_mu_lock(&p->mu);
+  size_t i;
 
-  for (i = 0; i < p->num_subchannels; i++) {
-    del_interested_parties_locked(exec_ctx, p, i);
-  }
+  gpr_mu_lock(&p->mu);
 
   p->shutdown = 1;
   while ((pp = p->pending_picks)) {
@@ -261,24 +242,26 @@ void rr_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   }
   grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                               GRPC_CHANNEL_FATAL_FAILURE, "shutdown");
+  for (i = 0; i < p->num_subchannels; i++) {
+    subchannel_data *sd = p->subchannels[i];
+    grpc_subchannel_notify_on_state_change(exec_ctx, sd->subchannel, NULL, NULL,
+                                           &sd->connectivity_changed_closure);
+  }
   gpr_mu_unlock(&p->mu);
 }
 
 static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                           grpc_subchannel **target) {
+                           grpc_connected_subchannel **target) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   pending_pick *pp;
-  size_t i;
   gpr_mu_lock(&p->mu);
   pp = p->pending_picks;
   p->pending_picks = NULL;
   while (pp != NULL) {
     pending_pick *next = pp->next;
     if (pp->target == target) {
-      for (i = 0; i < p->num_subchannels; i++) {
-        grpc_subchannel_add_interested_party(exec_ctx, p->subchannels[i],
-                                             pp->pollset);
-      }
+      grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties,
+                                   pp->pollset);
       *target = NULL;
       grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 0);
       gpr_free(pp);
@@ -295,12 +278,16 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) {
   size_t i;
   p->started_picking = 1;
 
+  gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%d", p,
+          p->num_subchannels);
+
   for (i = 0; i < p->num_subchannels; i++) {
-    p->subchannel_connectivity[i] = GRPC_CHANNEL_IDLE;
-    grpc_subchannel_notify_on_state_change(exec_ctx, p->subchannels[i],
-                                           &p->subchannel_connectivity[i],
-                                           &p->connectivity_changed_cbs[i]);
-    GRPC_LB_POLICY_REF(&p->base, "round_robin_connectivity");
+    subchannel_data *sd = p->subchannels[i];
+    sd->connectivity_state = GRPC_CHANNEL_IDLE;
+    grpc_subchannel_notify_on_state_change(
+        exec_ctx, sd->subchannel, &p->base.interested_parties,
+        &sd->connectivity_state, &sd->connectivity_changed_closure);
+    GRPC_LB_POLICY_WEAK_REF(&p->base, "round_robin_connectivity");
   }
 }
 
@@ -314,18 +301,18 @@ void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset,
-            grpc_metadata_batch *initial_metadata, grpc_subchannel **target,
-            grpc_closure *on_complete) {
-  size_t i;
+            grpc_metadata_batch *initial_metadata,
+            grpc_connected_subchannel **target, grpc_closure *on_complete) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   pending_pick *pp;
   ready_list *selected;
   gpr_mu_lock(&p->mu);
   if ((selected = peek_next_connected_locked(p))) {
     gpr_mu_unlock(&p->mu);
-    *target = selected->subchannel;
+    *target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
     if (grpc_lb_round_robin_trace) {
-      gpr_log(GPR_DEBUG, "[RR PICK] TARGET <-- SUBCHANNEL %p (NODE %p)",
+      gpr_log(GPR_DEBUG,
+              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)",
               selected->subchannel, selected);
     }
     /* only advance the last picked pointer if the selection was used */
@@ -335,10 +322,8 @@ int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    for (i = 0; i < p->num_subchannels; i++) {
-      grpc_subchannel_add_interested_party(exec_ctx, p->subchannels[i],
-                                           pollset);
-    }
+    grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties,
+                                 pollset);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
     pp->pollset = pollset;
@@ -352,33 +337,25 @@ int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset,
 
 static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                                     int iomgr_success) {
-  connectivity_changed_cb_arg *cb_arg = arg;
-  round_robin_lb_policy *p = cb_arg->p;
-  /* index over p->subchannels of this cb's subchannel */
-  const size_t this_idx = cb_arg->subchannel_idx;
+  subchannel_data *sd = arg;
+  round_robin_lb_policy *p = sd->policy;
   pending_pick *pp;
   ready_list *selected;
 
   int unref = 0;
 
-  /* connectivity state of this cb's subchannel */
-  grpc_connectivity_state *this_connectivity;
-
   gpr_mu_lock(&p->mu);
 
-  this_connectivity = &p->subchannel_connectivity[this_idx];
-
   if (p->shutdown) {
     unref = 1;
   } else {
-    switch (*this_connectivity) {
+    switch (sd->connectivity_state) {
       case GRPC_CHANNEL_READY:
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                     GRPC_CHANNEL_READY, "connecting_ready");
         /* add the newly connected subchannel to the list of connected ones.
          * Note that it goes to the "end of the line". */
-        p->subchannel_index_to_readylist_node[this_idx] =
-            add_connected_sc_locked(p, p->subchannels[this_idx]);
+        sd->ready_list_node = add_connected_sc_locked(p, sd->subchannel);
         /* at this point we know there's at least one suitable subchannel. Go
          * ahead and pick one and notify the pending suitors in
          * p->pending_picks. This preemtively replicates rr_pick()'s actions. */
@@ -390,60 +367,60 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         }
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
-          *pp->target = selected->subchannel;
+          *pp->target =
+              grpc_subchannel_get_connected_subchannel(selected->subchannel);
           if (grpc_lb_round_robin_trace) {
             gpr_log(GPR_DEBUG,
                     "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)",
                     selected->subchannel, selected);
           }
-          grpc_subchannel_del_interested_party(exec_ctx, selected->subchannel,
-                                               pp->pollset);
+          grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties,
+                                       pp->pollset);
           grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 1);
           gpr_free(pp);
         }
         grpc_subchannel_notify_on_state_change(
-            exec_ctx, p->subchannels[this_idx], this_connectivity,
-            &p->connectivity_changed_cbs[this_idx]);
+            exec_ctx, sd->subchannel, &p->base.interested_parties,
+            &sd->connectivity_state, &sd->connectivity_changed_closure);
         break;
       case GRPC_CHANNEL_CONNECTING:
       case GRPC_CHANNEL_IDLE:
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                    *this_connectivity, "connecting_changed");
+                                    sd->connectivity_state,
+                                    "connecting_changed");
         grpc_subchannel_notify_on_state_change(
-            exec_ctx, p->subchannels[this_idx], this_connectivity,
-            &p->connectivity_changed_cbs[this_idx]);
+            exec_ctx, sd->subchannel, &p->base.interested_parties,
+            &sd->connectivity_state, &sd->connectivity_changed_closure);
         break;
       case GRPC_CHANNEL_TRANSIENT_FAILURE:
-        del_interested_parties_locked(exec_ctx, p, this_idx);
         /* renew state notification */
         grpc_subchannel_notify_on_state_change(
-            exec_ctx, p->subchannels[this_idx], this_connectivity,
-            &p->connectivity_changed_cbs[this_idx]);
+            exec_ctx, sd->subchannel, &p->base.interested_parties,
+            &sd->connectivity_state, &sd->connectivity_changed_closure);
 
         /* remove from ready list if still present */
-        if (p->subchannel_index_to_readylist_node[this_idx] != NULL) {
-          remove_disconnected_sc_locked(
-              p, p->subchannel_index_to_readylist_node[this_idx]);
-          p->subchannel_index_to_readylist_node[this_idx] = NULL;
+        if (sd->ready_list_node != NULL) {
+          remove_disconnected_sc_locked(p, sd->ready_list_node);
+          sd->ready_list_node = NULL;
         }
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                     GRPC_CHANNEL_TRANSIENT_FAILURE,
                                     "connecting_transient_failure");
         break;
       case GRPC_CHANNEL_FATAL_FAILURE:
-        del_interested_parties_locked(exec_ctx, p, this_idx);
-        if (p->subchannel_index_to_readylist_node[this_idx] != NULL) {
-          remove_disconnected_sc_locked(
-              p, p->subchannel_index_to_readylist_node[this_idx]);
-          p->subchannel_index_to_readylist_node[this_idx] = NULL;
+        if (sd->ready_list_node != NULL) {
+          remove_disconnected_sc_locked(p, sd->ready_list_node);
+          sd->ready_list_node = NULL;
         }
 
-        GPR_SWAP(grpc_subchannel *, p->subchannels[this_idx],
-                 p->subchannels[p->num_subchannels - 1]);
         p->num_subchannels--;
-        GRPC_SUBCHANNEL_UNREF(exec_ctx, p->subchannels[p->num_subchannels],
-                              "round_robin");
+        GPR_SWAP(subchannel_data *, p->subchannels[sd->index],
+                 p->subchannels[p->num_subchannels]);
+        GRPC_SUBCHANNEL_UNREF(exec_ctx, sd->subchannel, "round_robin");
+        p->subchannels[sd->index]->index = sd->index;
+        gpr_free(sd);
 
+        unref = 1;
         if (p->num_subchannels == 0) {
           grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                       GRPC_CHANNEL_FATAL_FAILURE,
@@ -454,7 +431,6 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
             grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 1);
             gpr_free(pp);
           }
-          unref = 1;
         } else {
           grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                       GRPC_CHANNEL_TRANSIENT_FAILURE,
@@ -466,31 +442,8 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_mu_unlock(&p->mu);
 
   if (unref) {
-    GRPC_LB_POLICY_UNREF(exec_ctx, &p->base, "round_robin_connectivity");
-  }
-}
-
-static void rr_broadcast(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                         grpc_transport_op *op) {
-  round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
-  size_t i;
-  size_t n;
-  grpc_subchannel **subchannels;
-
-  gpr_mu_lock(&p->mu);
-  n = p->num_subchannels;
-  subchannels = gpr_malloc(n * sizeof(*subchannels));
-  for (i = 0; i < n; i++) {
-    subchannels[i] = p->subchannels[i];
-    GRPC_SUBCHANNEL_REF(subchannels[i], "rr_broadcast");
-  }
-  gpr_mu_unlock(&p->mu);
-
-  for (i = 0; i < n; i++) {
-    grpc_subchannel_process_transport_op(exec_ctx, subchannels[i], op);
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, subchannels[i], "rr_broadcast");
+    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "round_robin_connectivity");
   }
-  gpr_free(subchannels);
 }
 
 static grpc_connectivity_state rr_check_connectivity(grpc_exec_ctx *exec_ctx,
@@ -514,9 +467,25 @@ static void rr_notify_on_state_change(grpc_exec_ctx *exec_ctx,
   gpr_mu_unlock(&p->mu);
 }
 
+static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
+                        grpc_closure *closure) {
+  round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
+  ready_list *selected;
+  grpc_connected_subchannel *target;
+  gpr_mu_lock(&p->mu);
+  if ((selected = peek_next_connected_locked(p))) {
+    gpr_mu_unlock(&p->mu);
+    target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
+    grpc_connected_subchannel_ping(exec_ctx, target, closure);
+  } else {
+    gpr_mu_unlock(&p->mu);
+    grpc_exec_ctx_enqueue(exec_ctx, closure, 0);
+  }
+}
+
 static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = {
-    rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_exit_idle,
-    rr_broadcast, rr_check_connectivity, rr_notify_on_state_change};
+    rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_ping_one, rr_exit_idle,
+    rr_check_connectivity, rr_notify_on_state_change};
 
 static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {}
 
@@ -529,27 +498,22 @@ static grpc_lb_policy *create_round_robin(grpc_lb_policy_factory *factory,
   GPR_ASSERT(args->num_subchannels > 0);
   memset(p, 0, sizeof(*p));
   grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable);
-  p->subchannels =
-      gpr_malloc(sizeof(grpc_subchannel *) * args->num_subchannels);
   p->num_subchannels = args->num_subchannels;
+  p->subchannels = gpr_malloc(sizeof(*p->subchannels) * p->num_subchannels);
+  memset(p->subchannels, 0, sizeof(*p->subchannels) * p->num_subchannels);
   grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
                                "round_robin");
-  memcpy(p->subchannels, args->subchannels,
-         sizeof(grpc_subchannel *) * args->num_subchannels);
 
   gpr_mu_init(&p->mu);
-  p->connectivity_changed_cbs =
-      gpr_malloc(sizeof(grpc_closure) * args->num_subchannels);
-  p->subchannel_connectivity =
-      gpr_malloc(sizeof(grpc_connectivity_state) * args->num_subchannels);
-
-  p->cb_args =
-      gpr_malloc(sizeof(connectivity_changed_cb_arg) * args->num_subchannels);
   for (i = 0; i < args->num_subchannels; i++) {
-    p->cb_args[i].subchannel_idx = i;
-    p->cb_args[i].p = p;
-    grpc_closure_init(&p->connectivity_changed_cbs[i], rr_connectivity_changed,
-                      &p->cb_args[i]);
+    subchannel_data *sd = gpr_malloc(sizeof(*sd));
+    memset(sd, 0, sizeof(*sd));
+    p->subchannels[i] = sd;
+    sd->policy = p;
+    sd->index = i;
+    sd->subchannel = args->subchannels[i];
+    grpc_closure_init(&sd->connectivity_changed_closure,
+                      rr_connectivity_changed, sd);
   }
 
   /* The (dummy node) root of the ready list */
@@ -558,10 +522,6 @@ static grpc_lb_policy *create_round_robin(grpc_lb_policy_factory *factory,
   p->ready_list.next = NULL;
   p->ready_list_last_pick = &p->ready_list;
 
-  p->subchannel_index_to_readylist_node =
-      gpr_malloc(sizeof(grpc_subchannel *) * args->num_subchannels);
-  memset(p->subchannel_index_to_readylist_node, 0,
-         sizeof(grpc_subchannel *) * args->num_subchannels);
   return &p->base;
 }
 
diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c
index 36a2454309..d4672f6b25 100644
--- a/src/core/client_config/lb_policy.c
+++ b/src/core/client_config/lb_policy.c
@@ -33,63 +33,94 @@
 
 #include "src/core/client_config/lb_policy.h"
 
+#define WEAK_REF_BITS 16
+
 void grpc_lb_policy_init(grpc_lb_policy *policy,
                          const grpc_lb_policy_vtable *vtable) {
   policy->vtable = vtable;
-  gpr_ref_init(&policy->refs, 1);
+  gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS);
+  grpc_pollset_set_init(&policy->interested_parties);
 }
 
 #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG
-void grpc_lb_policy_ref(grpc_lb_policy *policy, const char *file, int line,
-                        const char *reason) {
-  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "LB_POLICY:%p   ref %d -> %d %s",
-          policy, (int)policy->refs.count, (int)policy->refs.count + 1, reason);
+#define REF_FUNC_EXTRA_ARGS , const char *file, int line, const char *reason
+#define REF_MUTATE_EXTRA_ARGS REF_FUNC_EXTRA_ARGS, const char *purpose
+#define REF_FUNC_PASS_ARGS(new_reason) , file, line, new_reason
+#define REF_MUTATE_PASS_ARGS(purpose) , file, line, reason, purpose
 #else
-void grpc_lb_policy_ref(grpc_lb_policy *policy) {
+#define REF_FUNC_EXTRA_ARGS
+#define REF_MUTATE_EXTRA_ARGS
+#define REF_FUNC_PASS_ARGS(new_reason)
+#define REF_MUTATE_PASS_ARGS(x)
 #endif
-  gpr_ref(&policy->refs);
-}
 
+static gpr_atm ref_mutate(grpc_lb_policy *c, gpr_atm delta,
+                          int barrier REF_MUTATE_EXTRA_ARGS) {
+  gpr_atm old_val = barrier ? gpr_atm_full_fetch_add(&c->ref_pair, delta)
+                            : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
 #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG
-void grpc_lb_policy_unref(grpc_lb_policy *policy,
-                          grpc_closure_list *closure_list, const char *file,
-                          int line, const char *reason) {
-  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "LB_POLICY:%p unref %d -> %d %s",
-          policy, (int)policy->refs.count, (int)policy->refs.count - 1, reason);
-#else
-void grpc_lb_policy_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy) {
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+          "LB_POLICY: %p % 12s 0x%08x -> 0x%08x [%s]", c, purpose, old_val,
+          old_val + delta, reason);
 #endif
-  if (gpr_unref(&policy->refs)) {
-    policy->vtable->destroy(exec_ctx, policy);
+  return old_val;
+}
+
+void grpc_lb_policy_ref(grpc_lb_policy *policy REF_FUNC_EXTRA_ARGS) {
+  ref_mutate(policy, 1 << WEAK_REF_BITS, 0 REF_MUTATE_PASS_ARGS("STRONG_REF"));
+}
+
+void grpc_lb_policy_unref(grpc_exec_ctx *exec_ctx,
+                          grpc_lb_policy *policy REF_FUNC_EXTRA_ARGS) {
+  gpr_atm old_val =
+      ref_mutate(policy, (gpr_atm)1 - (gpr_atm)(1 << WEAK_REF_BITS),
+                 1 REF_MUTATE_PASS_ARGS("STRONG_UNREF"));
+  gpr_atm mask = ~(gpr_atm)((1 << WEAK_REF_BITS) - 1);
+  gpr_atm check = 1 << WEAK_REF_BITS;
+  if ((old_val & mask) == check) {
+    policy->vtable->shutdown(exec_ctx, policy);
   }
+  grpc_lb_policy_weak_unref(exec_ctx,
+                            policy REF_FUNC_PASS_ARGS("strong-unref"));
 }
 
-void grpc_lb_policy_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy) {
-  policy->vtable->shutdown(exec_ctx, policy);
+void grpc_lb_policy_weak_ref(grpc_lb_policy *policy REF_FUNC_EXTRA_ARGS) {
+  ref_mutate(policy, 1, 0 REF_MUTATE_PASS_ARGS("WEAK_REF"));
+}
+
+void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx,
+                               grpc_lb_policy *policy REF_FUNC_EXTRA_ARGS) {
+  gpr_atm old_val =
+      ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF"));
+  if (old_val == 1) {
+    grpc_pollset_set_destroy(&policy->interested_parties);
+    policy->vtable->destroy(exec_ctx, policy);
+  }
 }
 
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                         grpc_pollset *pollset,
                         grpc_metadata_batch *initial_metadata,
-                        grpc_subchannel **target, grpc_closure *on_complete) {
+                        grpc_connected_subchannel **target,
+                        grpc_closure *on_complete) {
   return policy->vtable->pick(exec_ctx, policy, pollset, initial_metadata,
                               target, on_complete);
 }
 
 void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                                grpc_subchannel **target) {
+                                grpc_connected_subchannel **target) {
   policy->vtable->cancel_pick(exec_ctx, policy, target);
 }
 
-void grpc_lb_policy_broadcast(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                              grpc_transport_op *op) {
-  policy->vtable->broadcast(exec_ctx, policy, op);
-}
-
 void grpc_lb_policy_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy) {
   policy->vtable->exit_idle(exec_ctx, policy);
 }
 
+void grpc_lb_policy_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+                             grpc_closure *closure) {
+  policy->vtable->ping_one(exec_ctx, policy, closure);
+}
+
 void grpc_lb_policy_notify_on_state_change(grpc_exec_ctx *exec_ctx,
                                            grpc_lb_policy *policy,
                                            grpc_connectivity_state *state,
diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h
index a696c3ce64..db5238c8ca 100644
--- a/src/core/client_config/lb_policy.h
+++ b/src/core/client_config/lb_policy.h
@@ -47,7 +47,8 @@ typedef void (*grpc_lb_completion)(void *cb_arg, grpc_subchannel *subchannel,
 
 struct grpc_lb_policy {
   const grpc_lb_policy_vtable *vtable;
-  gpr_refcount refs;
+  gpr_atm ref_pair;
+  grpc_pollset_set interested_parties;
 };
 
 struct grpc_lb_policy_vtable {
@@ -58,17 +59,16 @@ struct grpc_lb_policy_vtable {
   /** implement grpc_lb_policy_pick */
   int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
               grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
-              grpc_subchannel **target, grpc_closure *on_complete);
+              grpc_connected_subchannel **target, grpc_closure *on_complete);
   void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                      grpc_subchannel **target);
+                      grpc_connected_subchannel **target);
+
+  void (*ping_one)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+                   grpc_closure *closure);
 
   /** try to enter a READY connectivity state */
   void (*exit_idle)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 
-  /** broadcast a transport op to all subchannels */
-  void (*broadcast)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                    grpc_transport_op *op);
-
   /** check the current connectivity of the lb_policy */
   grpc_connectivity_state (*check_connectivity)(grpc_exec_ctx *exec_ctx,
                                                 grpc_lb_policy *policy);
@@ -81,29 +81,39 @@ struct grpc_lb_policy_vtable {
                                  grpc_closure *closure);
 };
 
+/*#define GRPC_LB_POLICY_REFCOUNT_DEBUG*/
 #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG
 #define GRPC_LB_POLICY_REF(p, r) \
   grpc_lb_policy_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_LB_POLICY_UNREF(exec_ctx, p, r) \
   grpc_lb_policy_unref((exec_ctx), (p), __FILE__, __LINE__, (r))
+#define GRPC_LB_POLICY_WEAK_REF(p, r) \
+  grpc_lb_policy_weak_ref((p), __FILE__, __LINE__, (r))
+#define GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, p, r) \
+  grpc_lb_policy_weak_unref((exec_ctx), (p), __FILE__, __LINE__, (r))
 void grpc_lb_policy_ref(grpc_lb_policy *policy, const char *file, int line,
                         const char *reason);
 void grpc_lb_policy_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                           const char *file, int line, const char *reason);
+void grpc_lb_policy_weak_ref(grpc_lb_policy *policy, const char *file, int line,
+                             const char *reason);
+void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+                               const char *file, int line, const char *reason);
 #else
 #define GRPC_LB_POLICY_REF(p, r) grpc_lb_policy_ref((p))
 #define GRPC_LB_POLICY_UNREF(cl, p, r) grpc_lb_policy_unref((cl), (p))
+#define GRPC_LB_POLICY_WEAK_REF(p, r) grpc_lb_policy_weak_ref((p))
+#define GRPC_LB_POLICY_WEAK_UNREF(cl, p, r) grpc_lb_policy_weak_unref((cl), (p))
 void grpc_lb_policy_ref(grpc_lb_policy *policy);
 void grpc_lb_policy_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
+void grpc_lb_policy_weak_ref(grpc_lb_policy *policy);
+void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 #endif
 
 /** called by concrete implementations to initialize the base struct */
 void grpc_lb_policy_init(grpc_lb_policy *policy,
                          const grpc_lb_policy_vtable *vtable);
 
-/** Start shutting down (fail any pending picks) */
-void grpc_lb_policy_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
-
 /** Given initial metadata in \a initial_metadata, find an appropriate
     target for this rpc, and 'return' it by calling \a on_complete after setting
     \a target.
@@ -111,13 +121,14 @@ void grpc_lb_policy_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                         grpc_pollset *pollset,
                         grpc_metadata_batch *initial_metadata,
-                        grpc_subchannel **target, grpc_closure *on_complete);
+                        grpc_connected_subchannel **target,
+                        grpc_closure *on_complete);
 
-void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                                grpc_subchannel **target);
+void grpc_lb_policy_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+                             grpc_closure *closure);
 
-void grpc_lb_policy_broadcast(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                              grpc_transport_op *op);
+void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+                                grpc_connected_subchannel **target);
 
 void grpc_lb_policy_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 
diff --git a/src/core/client_config/resolver.c b/src/core/client_config/resolver.c
index 081097eb19..eda01e72ba 100644
--- a/src/core/client_config/resolver.c
+++ b/src/core/client_config/resolver.c
@@ -71,11 +71,8 @@ void grpc_resolver_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
 }
 
 void grpc_resolver_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                     grpc_resolver *resolver,
-                                     struct sockaddr *failing_address,
-                                     int failing_address_len) {
-  resolver->vtable->channel_saw_error(exec_ctx, resolver, failing_address,
-                                      failing_address_len);
+                                     grpc_resolver *resolver) {
+  resolver->vtable->channel_saw_error(exec_ctx, resolver);
 }
 
 void grpc_resolver_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
diff --git a/src/core/client_config/resolver.h b/src/core/client_config/resolver.h
index 7ba0cd5bd4..e612eaf3b3 100644
--- a/src/core/client_config/resolver.h
+++ b/src/core/client_config/resolver.h
@@ -35,8 +35,8 @@
 #define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_H
 
 #include "src/core/client_config/client_config.h"
+#include "src/core/client_config/subchannel.h"
 #include "src/core/iomgr/iomgr.h"
-#include "src/core/iomgr/sockaddr.h"
 
 typedef struct grpc_resolver grpc_resolver;
 typedef struct grpc_resolver_vtable grpc_resolver_vtable;
@@ -51,9 +51,7 @@ struct grpc_resolver {
 struct grpc_resolver_vtable {
   void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver);
   void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver);
-  void (*channel_saw_error)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-                            struct sockaddr *failing_address,
-                            int failing_address_len);
+  void (*channel_saw_error)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver);
   void (*next)(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
                grpc_client_config **target_config, grpc_closure *on_complete);
 };
@@ -81,9 +79,7 @@ void grpc_resolver_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver);
 /** Notification that the channel has seen an error on some address.
     Can be used as a hint that re-resolution is desirable soon. */
 void grpc_resolver_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                     grpc_resolver *resolver,
-                                     struct sockaddr *failing_address,
-                                     int failing_address_len);
+                                     grpc_resolver *resolver);
 
 /** Get the next client config. Called by the channel to fetch a new
     configuration. Expected to set *target_config with a new configuration,
diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/client_config/resolvers/dns_resolver.c
index b40c41544a..28ca30b946 100644
--- a/src/core/client_config/resolvers/dns_resolver.c
+++ b/src/core/client_config/resolvers/dns_resolver.c
@@ -80,9 +80,7 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
                                          dns_resolver *r);
 
 static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
-static void dns_channel_saw_error(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
-                                  struct sockaddr *failing_address,
-                                  int failing_address_len);
+static void dns_channel_saw_error(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
 static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
                      grpc_client_config **target_config,
                      grpc_closure *on_complete);
@@ -102,8 +100,7 @@ static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
 }
 
 static void dns_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                  grpc_resolver *resolver, struct sockaddr *sa,
-                                  int len) {
+                                  grpc_resolver *resolver) {
   dns_resolver *r = (dns_resolver *)resolver;
   gpr_mu_lock(&r->mu);
   if (!r->resolving) {
diff --git a/src/core/client_config/resolvers/sockaddr_resolver.c b/src/core/client_config/resolvers/sockaddr_resolver.c
index 0b017f06c7..81d6627ecc 100644
--- a/src/core/client_config/resolvers/sockaddr_resolver.c
+++ b/src/core/client_config/resolvers/sockaddr_resolver.c
@@ -83,9 +83,7 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
 
 static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
 static void sockaddr_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                       grpc_resolver *r,
-                                       struct sockaddr *failing_address,
-                                       int failing_address_len);
+                                       grpc_resolver *r);
 static void sockaddr_next(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
                           grpc_client_config **target_config,
                           grpc_closure *on_complete);
@@ -107,8 +105,13 @@ static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx,
 }
 
 static void sockaddr_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                       grpc_resolver *resolver,
-                                       struct sockaddr *sa, int len) {}
+                                       grpc_resolver *resolver) {
+  sockaddr_resolver *r = (sockaddr_resolver *)resolver;
+  gpr_mu_lock(&r->mu);
+  r->published = 0;
+  sockaddr_maybe_finish_next_locked(exec_ctx, r);
+  gpr_mu_unlock(&r->mu);
+}
 
 static void sockaddr_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
                           grpc_client_config **target_config,
@@ -344,6 +347,9 @@ static grpc_resolver *sockaddr_create(
   gpr_slice_buffer_destroy(&path_parts);
   gpr_slice_unref(path_slice);
   if (errors_found) {
+    gpr_free(r->lb_policy_name);
+    gpr_free(r->addrs);
+    gpr_free(r->addrs_len);
     gpr_free(r);
     return NULL;
   }
diff --git a/src/core/client_config/resolvers/zookeeper_resolver.c b/src/core/client_config/resolvers/zookeeper_resolver.c
index 136197d4c6..4924ca77d6 100644
--- a/src/core/client_config/resolvers/zookeeper_resolver.c
+++ b/src/core/client_config/resolvers/zookeeper_resolver.c
@@ -96,9 +96,7 @@ static void zookeeper_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
 
 static void zookeeper_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
 static void zookeeper_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                        grpc_resolver *r,
-                                        struct sockaddr *failing_address,
-                                        int failing_address_len);
+                                        grpc_resolver *r);
 static void zookeeper_next(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
                            grpc_client_config **target_config,
                            grpc_closure *on_complete);
@@ -125,8 +123,7 @@ static void zookeeper_shutdown(grpc_exec_ctx *exec_ctx,
 }
 
 static void zookeeper_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                        grpc_resolver *resolver,
-                                        struct sockaddr *sa, int len) {
+                                        grpc_resolver *resolver) {
   zookeeper_resolver *r = (zookeeper_resolver *)resolver;
   gpr_mu_lock(&r->mu);
   if (r->resolving == 0) {
diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c
index 28496ac2c9..afb1cdbd6d 100644
--- a/src/core/client_config/subchannel.c
+++ b/src/core/client_config/subchannel.c
@@ -47,39 +47,44 @@
 #include "src/core/transport/connectivity_state.h"
 #include "src/core/transport/connectivity_state.h"
 
+#define INTERNAL_REF_BITS 16
+#define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1))
+
 #define GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS 20
 #define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 1
 #define GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER 1.6
 #define GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_SUBCHANNEL_RECONNECT_JITTER 0.2
 
-typedef struct {
-  /* all fields protected by subchannel->mu */
-  /** refcount */
-  int refs;
-  /** parent subchannel */
-  grpc_subchannel *subchannel;
-} connection;
+#define GET_CONNECTED_SUBCHANNEL(subchannel, barrier)      \
+  ((grpc_connected_subchannel *)(gpr_atm_##barrier##_load( \
+      &(subchannel)->connected_subchannel)))
 
 typedef struct {
   grpc_closure closure;
-  size_t version;
   grpc_subchannel *subchannel;
   grpc_connectivity_state connectivity_state;
 } state_watcher;
 
-typedef struct waiting_for_connect {
-  struct waiting_for_connect *next;
-  grpc_closure *notify;
-  grpc_pollset *pollset;
-  gpr_atm *target;
+typedef struct external_state_watcher {
   grpc_subchannel *subchannel;
-  grpc_closure continuation;
-} waiting_for_connect;
+  grpc_pollset_set *pollset_set;
+  grpc_closure *notify;
+  grpc_closure closure;
+  struct external_state_watcher *next;
+  struct external_state_watcher *prev;
+} external_state_watcher;
 
 struct grpc_subchannel {
   grpc_connector *connector;
 
+  /** refcount
+      - lower INTERNAL_REF_BITS bits are for internal references:
+        these do not keep the subchannel open.
+      - upper remaining bits are for public references: these do
+        keep the subchannel open */
+  gpr_atm ref_pair;
+
   /** non-transport related channel filters */
   const grpc_channel_filter **filters;
   size_t num_filters;
@@ -88,15 +93,9 @@ struct grpc_subchannel {
   /** address to connect to */
   struct sockaddr *addr;
   size_t addr_len;
+
   /** initial string to send to peer */
   gpr_slice initial_connect_string;
-  /** master channel - the grpc_channel instance that ultimately owns
-      this channel_data via its channel stack.
-      We occasionally use this to bump the refcount on the master channel
-      to keep ourselves alive through an asynchronous operation. */
-  grpc_channel *master;
-  /** have we seen a disconnection? */
-  int disconnected;
 
   /** set during connection */
   grpc_connect_out_args connecting_result;
@@ -105,27 +104,24 @@ struct grpc_subchannel {
   grpc_closure connected;
 
   /** pollset_set tracking who's interested in a connection
-      being setup - owned by the master channel (in particular the
-     client_channel
-      filter there-in) */
-  grpc_pollset_set *pollset_set;
+      being setup */
+  grpc_pollset_set pollset_set;
+
+  /** active connection, or null; of type grpc_connected_subchannel */
+  gpr_atm connected_subchannel;
 
   /** mutex protecting remaining elements */
   gpr_mu mu;
 
-  /** active connection */
-  connection *active;
-  /** version number for the active connection */
-  size_t active_version;
-  /** refcount */
-  int refs;
+  /** have we seen a disconnection? */
+  int disconnected;
   /** are we connecting */
   int connecting;
-  /** things waiting for a connection */
-  waiting_for_connect *waiting;
   /** connectivity state tracking */
   grpc_connectivity_state_tracker state_tracker;
 
+  external_state_watcher root_external_state_watcher;
+
   /** next connect attempt time */
   gpr_timespec next_attempt;
   /** amount to backoff each failure */
@@ -139,151 +135,141 @@ struct grpc_subchannel {
 };
 
 struct grpc_subchannel_call {
-  connection *connection;
+  grpc_connected_subchannel *connection;
 };
 
 #define SUBCHANNEL_CALL_TO_CALL_STACK(call) ((grpc_call_stack *)((call) + 1))
-#define CHANNEL_STACK_FROM_CONNECTION(con) ((grpc_channel_stack *)((con) + 1))
+#define CHANNEL_STACK_FROM_CONNECTION(con) ((grpc_channel_stack *)(con))
 #define CALLSTACK_TO_SUBCHANNEL_CALL(callstack) \
   (((grpc_subchannel_call *)(callstack)) - 1)
 
-static grpc_subchannel_call *create_call(grpc_exec_ctx *exec_ctx,
-                                         connection *con,
-                                         grpc_pollset *pollset);
-static void connectivity_state_changed_locked(grpc_exec_ctx *exec_ctx,
-                                              grpc_subchannel *c,
-                                              const char *reason);
-static grpc_connectivity_state compute_connectivity_locked(grpc_subchannel *c);
 static gpr_timespec compute_connect_deadline(grpc_subchannel *c);
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *subchannel,
                                  int iomgr_success);
 
-static void subchannel_ref_locked(grpc_subchannel *c
-                                      GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-static int subchannel_unref_locked(
-    grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) GRPC_MUST_USE_RESULT;
-static void connection_ref_locked(connection *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-static grpc_subchannel *connection_unref_locked(
-    grpc_exec_ctx *exec_ctx,
-    connection *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) GRPC_MUST_USE_RESULT;
-static void subchannel_destroy(grpc_exec_ctx *exec_ctx, grpc_subchannel *c);
-
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
-#define SUBCHANNEL_REF_LOCKED(p, r) \
-  subchannel_ref_locked((p), __FILE__, __LINE__, (r))
-#define SUBCHANNEL_UNREF_LOCKED(p, r) \
-  subchannel_unref_locked((p), __FILE__, __LINE__, (r))
-#define CONNECTION_REF_LOCKED(p, r) \
-  connection_ref_locked((p), __FILE__, __LINE__, (r))
-#define CONNECTION_UNREF_LOCKED(cl, p, r) \
-  connection_unref_locked((cl), (p), __FILE__, __LINE__, (r))
-#define REF_PASS_ARGS , file, line, reason
-#define REF_PASS_REASON , reason
+#define REF_REASON reason
 #define REF_LOG(name, p)                                                  \
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "%s: %p   ref %d -> %d %s", \
-          (name), (p), (p)->refs, (p)->refs + 1, reason)
+          (name), (p), (p)->refs.count, (p)->refs.count + 1, reason)
 #define UNREF_LOG(name, p)                                                \
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "%s: %p unref %d -> %d %s", \
-          (name), (p), (p)->refs, (p)->refs - 1, reason)
+          (name), (p), (p)->refs.count, (p)->refs.count - 1, reason)
+#define REF_MUTATE_EXTRA_ARGS \
+  GRPC_SUBCHANNEL_REF_EXTRA_ARGS, const char *purpose
+#define REF_MUTATE_PURPOSE(x) , file, line, reason, x
 #else
-#define SUBCHANNEL_REF_LOCKED(p, r) subchannel_ref_locked((p))
-#define SUBCHANNEL_UNREF_LOCKED(p, r) subchannel_unref_locked((p))
-#define CONNECTION_REF_LOCKED(p, r) connection_ref_locked((p))
-#define CONNECTION_UNREF_LOCKED(cl, p, r) connection_unref_locked((cl), (p))
-#define REF_PASS_ARGS
-#define REF_PASS_REASON
+#define REF_REASON ""
 #define REF_LOG(name, p) \
   do {                   \
   } while (0)
 #define UNREF_LOG(name, p) \
   do {                     \
   } while (0)
+#define REF_MUTATE_EXTRA_ARGS
+#define REF_MUTATE_PURPOSE(x)
 #endif
 
 /*
  * connection implementation
  */
 
-static void connection_destroy(grpc_exec_ctx *exec_ctx, connection *c) {
-  GPR_ASSERT(c->refs == 0);
+static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg,
+                               int success) {
+  grpc_connected_subchannel *c = arg;
   grpc_channel_stack_destroy(exec_ctx, CHANNEL_STACK_FROM_CONNECTION(c));
   gpr_free(c);
 }
 
-static void connection_ref_locked(connection *c
-                                      GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  REF_LOG("CONNECTION", c);
-  subchannel_ref_locked(c->subchannel REF_PASS_ARGS);
-  ++c->refs;
+void grpc_connected_subchannel_ref(grpc_connected_subchannel *c
+                                       GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+  GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON);
 }
 
-static grpc_subchannel *connection_unref_locked(
-    grpc_exec_ctx *exec_ctx, connection *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  grpc_subchannel *destroy = NULL;
-  UNREF_LOG("CONNECTION", c);
-  if (subchannel_unref_locked(c->subchannel REF_PASS_ARGS)) {
-    destroy = c->subchannel;
-  }
-  if (--c->refs == 0 && c->subchannel->active != c) {
-    connection_destroy(exec_ctx, c);
-  }
-  return destroy;
+void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx,
+                                     grpc_connected_subchannel *c
+                                         GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+  GRPC_CHANNEL_STACK_UNREF(exec_ctx, CHANNEL_STACK_FROM_CONNECTION(c),
+                           REF_REASON);
 }
 
 /*
  * grpc_subchannel implementation
  */
 
-static void subchannel_ref_locked(grpc_subchannel *c
-                                      GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  REF_LOG("SUBCHANNEL", c);
-  ++c->refs;
+static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg,
+                               int success) {
+  grpc_subchannel *c = arg;
+  gpr_free((void *)c->filters);
+  grpc_channel_args_destroy(c->args);
+  gpr_free(c->addr);
+  gpr_slice_unref(c->initial_connect_string);
+  grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker);
+  grpc_connector_unref(exec_ctx, c->connector);
+  grpc_pollset_set_destroy(&c->pollset_set);
+  gpr_free(c);
 }
 
-static int subchannel_unref_locked(grpc_subchannel *c
-                                       GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  UNREF_LOG("SUBCHANNEL", c);
-  return --c->refs == 0;
+static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta,
+                          int barrier REF_MUTATE_EXTRA_ARGS) {
+  gpr_atm old_val = barrier ? gpr_atm_full_fetch_add(&c->ref_pair, delta)
+                            : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
+#ifdef GRPC_STREAM_REFCOUNT_DEBUG
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+          "SUBCHANNEL: %p % 12s 0x%08x -> 0x%08x [%s]", c, purpose, old_val,
+          old_val + delta, reason);
+#endif
+  return old_val;
 }
 
 void grpc_subchannel_ref(grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  gpr_mu_lock(&c->mu);
-  subchannel_ref_locked(c REF_PASS_ARGS);
-  gpr_mu_unlock(&c->mu);
+  gpr_atm old_refs;
+  old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS),
+                        0 REF_MUTATE_PURPOSE("STRONG_REF"));
+  GPR_ASSERT((old_refs & STRONG_REF_MASK) != 0);
 }
 
-void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx,
-                           grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  int destroy;
-  gpr_mu_lock(&c->mu);
-  destroy = subchannel_unref_locked(c REF_PASS_ARGS);
-  gpr_mu_unlock(&c->mu);
-  if (destroy) subchannel_destroy(exec_ctx, c);
+void grpc_subchannel_weak_ref(grpc_subchannel *c
+                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+  gpr_atm old_refs;
+  old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF"));
+  GPR_ASSERT(old_refs != 0);
 }
 
-static void subchannel_destroy(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
-  if (c->active != NULL) {
-    connection_destroy(exec_ctx, c->active);
+static void disconnect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
+  grpc_connected_subchannel *con;
+  gpr_mu_lock(&c->mu);
+  GPR_ASSERT(!c->disconnected);
+  c->disconnected = 1;
+  grpc_connector_shutdown(exec_ctx, c->connector);
+  con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
+  if (con != NULL) {
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, con, "connection");
+    gpr_atm_no_barrier_store(&c->connected_subchannel, 0xdeadbeef);
   }
-  gpr_free((void *)c->filters);
-  grpc_channel_args_destroy(c->args);
-  gpr_free(c->addr);
-  gpr_slice_unref(c->initial_connect_string);
-  grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker);
-  grpc_connector_unref(exec_ctx, c->connector);
-  gpr_free(c);
+  gpr_mu_unlock(&c->mu);
 }
 
-void grpc_subchannel_add_interested_party(grpc_exec_ctx *exec_ctx,
-                                          grpc_subchannel *c,
-                                          grpc_pollset *pollset) {
-  grpc_pollset_set_add_pollset(exec_ctx, c->pollset_set, pollset);
+void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx,
+                           grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+  gpr_atm old_refs;
+  old_refs = ref_mutate(c, (gpr_atm)1 - (gpr_atm)(1 << INTERNAL_REF_BITS),
+                        1 REF_MUTATE_PURPOSE("STRONG_UNREF"));
+  if ((old_refs & STRONG_REF_MASK) == (1 << INTERNAL_REF_BITS)) {
+    disconnect(exec_ctx, c);
+  }
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "strong-unref");
 }
 
-void grpc_subchannel_del_interested_party(grpc_exec_ctx *exec_ctx,
-                                          grpc_subchannel *c,
-                                          grpc_pollset *pollset) {
-  grpc_pollset_set_del_pollset(exec_ctx, c->pollset_set, pollset);
+void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
+                                grpc_subchannel *c
+                                    GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+  gpr_atm old_refs;
+  old_refs = ref_mutate(c, -(gpr_atm)1, 1 REF_MUTATE_PURPOSE("WEAK_UNREF"));
+  if (old_refs == 1) {
+    grpc_exec_ctx_enqueue(exec_ctx, grpc_closure_create(subchannel_destroy, c),
+                          1);
+  }
 }
 
 static gpr_uint32 random_seed() {
@@ -293,10 +279,8 @@ static gpr_uint32 random_seed() {
 grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
                                         grpc_subchannel_args *args) {
   grpc_subchannel *c = gpr_malloc(sizeof(*c));
-  grpc_channel_element *parent_elem = grpc_channel_stack_last_element(
-      grpc_channel_get_channel_stack(args->master));
   memset(c, 0, sizeof(*c));
-  c->refs = 1;
+  gpr_atm_no_barrier_store(&c->ref_pair, 1 << INTERNAL_REF_BITS);
   c->connector = connector;
   grpc_connector_ref(c->connector);
   c->num_filters = args->filter_count;
@@ -305,13 +289,14 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
          sizeof(grpc_channel_filter *) * c->num_filters);
   c->addr = gpr_malloc(args->addr_len);
   memcpy(c->addr, args->addr, args->addr_len);
+  grpc_pollset_set_init(&c->pollset_set);
   c->addr_len = args->addr_len;
   grpc_set_initial_connect_string(&c->addr, &c->addr_len,
                                   &c->initial_connect_string);
   c->args = grpc_channel_args_copy(args->args);
-  c->master = args->master;
-  c->pollset_set = grpc_client_channel_get_connecting_pollset_set(parent_elem);
   c->random = random_seed();
+  c->root_external_state_watcher.next = c->root_external_state_watcher.prev =
+      &c->root_external_state_watcher;
   grpc_closure_init(&c->connected, subchannel_connected, c);
   grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE,
                                "subchannel");
@@ -319,70 +304,18 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
   return c;
 }
 
-static void cancel_waiting_calls(grpc_exec_ctx *exec_ctx,
-                                 grpc_subchannel *subchannel,
-                                 int iomgr_success) {
-  waiting_for_connect *w4c;
-  gpr_mu_lock(&subchannel->mu);
-  w4c = subchannel->waiting;
-  subchannel->waiting = NULL;
-  gpr_mu_unlock(&subchannel->mu);
-  while (w4c != NULL) {
-    waiting_for_connect *next = w4c->next;
-    grpc_subchannel_del_interested_party(exec_ctx, w4c->subchannel,
-                                         w4c->pollset);
-    if (w4c->notify) {
-      w4c->notify->cb(exec_ctx, w4c->notify->cb_arg, iomgr_success);
-    }
-
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, w4c->subchannel, "waiting_for_connect");
-    gpr_free(w4c);
-
-    w4c = next;
-  }
-}
-
-void grpc_subchannel_cancel_create_call(grpc_exec_ctx *exec_ctx,
-                                        grpc_subchannel *subchannel,
-                                        gpr_atm *target) {
-  waiting_for_connect *w4c;
-  int unref_count = 0;
-  gpr_mu_lock(&subchannel->mu);
-  w4c = subchannel->waiting;
-  subchannel->waiting = NULL;
-  while (w4c != NULL) {
-    waiting_for_connect *next = w4c->next;
-    if (w4c->target == target) {
-      grpc_subchannel_del_interested_party(exec_ctx, w4c->subchannel,
-                                           w4c->pollset);
-      grpc_exec_ctx_enqueue(exec_ctx, w4c->notify, 0);
-
-      unref_count++;
-      gpr_free(w4c);
-    } else {
-      w4c->next = subchannel->waiting;
-      subchannel->waiting = w4c;
-    }
-
-    w4c = next;
-  }
-  gpr_mu_unlock(&subchannel->mu);
-
-  while (unref_count-- > 0) {
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, subchannel, "waiting_for_connect");
-  }
-}
-
 static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   grpc_connect_in_args args;
 
-  args.interested_parties = c->pollset_set;
+  args.interested_parties = &c->pollset_set;
   args.addr = c->addr;
   args.addr_len = c->addr_len;
   args.deadline = compute_connect_deadline(c);
   args.channel_args = c->args;
   args.initial_connect_string = c->initial_connect_string;
 
+  grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
+                              GRPC_CHANNEL_CONNECTING, "state_change");
   grpc_connector_connect(exec_ctx, c->connector, &args, &c->connecting_result,
                          &c->connected);
 }
@@ -395,66 +328,6 @@ static void start_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   continue_connect(exec_ctx, c);
 }
 
-static void continue_creating_call(grpc_exec_ctx *exec_ctx, void *arg,
-                                   int iomgr_success) {
-  int call_creation_finished_ok;
-  waiting_for_connect *w4c = arg;
-  grpc_subchannel_del_interested_party(exec_ctx, w4c->subchannel, w4c->pollset);
-  call_creation_finished_ok = grpc_subchannel_create_call(
-      exec_ctx, w4c->subchannel, w4c->pollset, w4c->target, w4c->notify);
-  GPR_ASSERT(call_creation_finished_ok == 1);
-  w4c->notify->cb(exec_ctx, w4c->notify->cb_arg, iomgr_success);
-  GRPC_SUBCHANNEL_UNREF(exec_ctx, w4c->subchannel, "waiting_for_connect");
-  gpr_free(w4c);
-}
-
-int grpc_subchannel_create_call(grpc_exec_ctx *exec_ctx, grpc_subchannel *c,
-                                grpc_pollset *pollset, gpr_atm *target,
-                                grpc_closure *notify) {
-  connection *con;
-  grpc_subchannel_call *call;
-  GPR_TIMER_BEGIN("grpc_subchannel_create_call", 0);
-  gpr_mu_lock(&c->mu);
-  if (c->active != NULL) {
-    con = c->active;
-    CONNECTION_REF_LOCKED(con, "call");
-    gpr_mu_unlock(&c->mu);
-
-    call = create_call(exec_ctx, con, pollset);
-    if (!gpr_atm_rel_cas(target, 0, (gpr_atm)(gpr_uintptr)call)) {
-      GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "failed to set");
-    }
-    GPR_TIMER_END("grpc_subchannel_create_call", 0);
-    return 1;
-  } else {
-    waiting_for_connect *w4c = gpr_malloc(sizeof(*w4c));
-    w4c->next = c->waiting;
-    w4c->notify = notify;
-    w4c->pollset = pollset;
-    w4c->target = target;
-    w4c->subchannel = c;
-    /* released when clearing w4c */
-    SUBCHANNEL_REF_LOCKED(c, "waiting_for_connect");
-    grpc_closure_init(&w4c->continuation, continue_creating_call, w4c);
-    c->waiting = w4c;
-    grpc_subchannel_add_interested_party(exec_ctx, c, pollset);
-    if (!c->connecting) {
-      c->connecting = 1;
-      connectivity_state_changed_locked(exec_ctx, c, "create_call");
-      /* released by connection */
-      SUBCHANNEL_REF_LOCKED(c, "connecting");
-      GRPC_CHANNEL_INTERNAL_REF(c->master, "connecting");
-      gpr_mu_unlock(&c->mu);
-
-      start_connect(exec_ctx, c);
-    } else {
-      gpr_mu_unlock(&c->mu);
-    }
-    GPR_TIMER_END("grpc_subchannel_create_call", 0);
-    return 0;
-  }
-}
-
 grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel *c) {
   grpc_connectivity_state state;
   gpr_mu_lock(&c->mu);
@@ -463,153 +336,149 @@ grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel *c) {
   return state;
 }
 
-void grpc_subchannel_notify_on_state_change(grpc_exec_ctx *exec_ctx,
-                                            grpc_subchannel *c,
-                                            grpc_connectivity_state *state,
-                                            grpc_closure *notify) {
+static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg,
+                                           int success) {
+  external_state_watcher *w = arg;
+  grpc_closure *follow_up = w->notify;
+  if (w->pollset_set != NULL) {
+    grpc_pollset_set_del_pollset_set(exec_ctx, &w->subchannel->pollset_set,
+                                     w->pollset_set);
+  }
+  gpr_mu_lock(&w->subchannel->mu);
+  w->next->prev = w->prev;
+  w->prev->next = w->next;
+  gpr_mu_unlock(&w->subchannel->mu);
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, w->subchannel, "external_state_watcher");
+  gpr_free(w);
+  follow_up->cb(exec_ctx, follow_up->cb_arg, success);
+}
+
+void grpc_subchannel_notify_on_state_change(
+    grpc_exec_ctx *exec_ctx, grpc_subchannel *c,
+    grpc_pollset_set *interested_parties, grpc_connectivity_state *state,
+    grpc_closure *notify) {
   int do_connect = 0;
-  gpr_mu_lock(&c->mu);
-  if (grpc_connectivity_state_notify_on_state_change(
-          exec_ctx, &c->state_tracker, state, notify)) {
-    do_connect = 1;
-    c->connecting = 1;
-    /* released by connection */
-    SUBCHANNEL_REF_LOCKED(c, "connecting");
-    GRPC_CHANNEL_INTERNAL_REF(c->master, "connecting");
-    connectivity_state_changed_locked(exec_ctx, c, "state_change");
+  external_state_watcher *w;
+
+  if (state == NULL) {
+    gpr_mu_lock(&c->mu);
+    for (w = c->root_external_state_watcher.next;
+         w != &c->root_external_state_watcher; w = w->next) {
+      if (w->notify == notify) {
+        grpc_connectivity_state_notify_on_state_change(
+            exec_ctx, &c->state_tracker, NULL, &w->closure);
+      }
+    }
+    gpr_mu_unlock(&c->mu);
+  } else {
+    w = gpr_malloc(sizeof(*w));
+    w->subchannel = c;
+    w->pollset_set = interested_parties;
+    w->notify = notify;
+    grpc_closure_init(&w->closure, on_external_state_watcher_done, w);
+    if (interested_parties != NULL) {
+      grpc_pollset_set_add_pollset_set(exec_ctx, &c->pollset_set,
+                                       interested_parties);
+    }
+    GRPC_SUBCHANNEL_WEAK_REF(c, "external_state_watcher");
+    gpr_mu_lock(&c->mu);
+    w->next = &c->root_external_state_watcher;
+    w->prev = w->next->prev;
+    w->next->prev = w->prev->next = w;
+    if (grpc_connectivity_state_notify_on_state_change(
+            exec_ctx, &c->state_tracker, state, &w->closure)) {
+      do_connect = 1;
+      c->connecting = 1;
+      /* released by connection */
+      GRPC_SUBCHANNEL_WEAK_REF(c, "connecting");
+    }
+    gpr_mu_unlock(&c->mu);
   }
-  gpr_mu_unlock(&c->mu);
 
   if (do_connect) {
     start_connect(exec_ctx, c);
   }
 }
 
-int grpc_subchannel_state_change_unsubscribe(grpc_exec_ctx *exec_ctx,
-                                             grpc_subchannel *c,
-                                             grpc_closure *subscribed_notify) {
-  int success;
-  gpr_mu_lock(&c->mu);
-  success = grpc_connectivity_state_change_unsubscribe(
-      exec_ctx, &c->state_tracker, subscribed_notify);
-  gpr_mu_unlock(&c->mu);
-  return success;
+void grpc_connected_subchannel_process_transport_op(
+    grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *con,
+    grpc_transport_op *op) {
+  grpc_channel_stack *channel_stack = CHANNEL_STACK_FROM_CONNECTION(con);
+  grpc_channel_element *top_elem = grpc_channel_stack_element(channel_stack, 0);
+  top_elem->filter->start_transport_op(exec_ctx, top_elem, op);
 }
 
-void grpc_subchannel_process_transport_op(grpc_exec_ctx *exec_ctx,
-                                          grpc_subchannel *c,
-                                          grpc_transport_op *op) {
-  connection *con = NULL;
-  grpc_subchannel *destroy;
-  int cancel_alarm = 0;
-  gpr_mu_lock(&c->mu);
-  if (c->active != NULL) {
-    con = c->active;
-    CONNECTION_REF_LOCKED(con, "transport-op");
-  }
-  if (op->disconnect) {
-    c->disconnected = 1;
-    connectivity_state_changed_locked(exec_ctx, c, "disconnect");
-    if (c->have_alarm) {
-      cancel_alarm = 1;
-    }
-  }
-  gpr_mu_unlock(&c->mu);
+static void subchannel_on_child_state_changed(grpc_exec_ctx *exec_ctx, void *p,
+                                              int iomgr_success) {
+  state_watcher *sw = p;
+  grpc_subchannel *c = sw->subchannel;
+  gpr_mu *mu = &c->mu;
 
-  if (con != NULL) {
-    grpc_channel_stack *channel_stack = CHANNEL_STACK_FROM_CONNECTION(con);
-    grpc_channel_element *top_elem =
-        grpc_channel_stack_element(channel_stack, 0);
-    top_elem->filter->start_transport_op(exec_ctx, top_elem, op);
+  gpr_mu_lock(mu);
 
-    gpr_mu_lock(&c->mu);
-    destroy = CONNECTION_UNREF_LOCKED(exec_ctx, con, "transport-op");
-    gpr_mu_unlock(&c->mu);
-    if (destroy) {
-      subchannel_destroy(exec_ctx, destroy);
+  /* if we failed just leave this closure */
+  if (iomgr_success) {
+    if (sw->connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      /* any errors on a subchannel ==> we're done, create a new one */
+      sw->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE;
+    }
+    grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
+                                sw->connectivity_state, "reflect_child");
+    if (sw->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) {
+      grpc_connected_subchannel_notify_on_state_change(
+          exec_ctx, GET_CONNECTED_SUBCHANNEL(c, no_barrier), NULL,
+          &sw->connectivity_state, &sw->closure);
+      GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
+      sw = NULL;
     }
   }
 
-  if (cancel_alarm) {
-    grpc_timer_cancel(exec_ctx, &c->alarm);
-  }
-
-  if (op->disconnect) {
-    grpc_connector_shutdown(exec_ctx, c->connector);
-  }
+  gpr_mu_unlock(mu);
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "state_watcher");
+  gpr_free(sw);
 }
 
-static void on_state_changed(grpc_exec_ctx *exec_ctx, void *p,
-                             int iomgr_success) {
-  state_watcher *sw = p;
-  grpc_subchannel *c = sw->subchannel;
-  gpr_mu *mu = &c->mu;
-  int destroy;
+static void connected_subchannel_state_op(grpc_exec_ctx *exec_ctx,
+                                          grpc_connected_subchannel *con,
+                                          grpc_pollset_set *interested_parties,
+                                          grpc_connectivity_state *state,
+                                          grpc_closure *closure) {
   grpc_transport_op op;
   grpc_channel_element *elem;
-  connection *destroy_connection = NULL;
-
-  gpr_mu_lock(mu);
-
-  /* if we failed or there is a version number mismatch, just leave
-     this closure */
-  if (!iomgr_success || sw->subchannel->active_version != sw->version) {
-    goto done;
-  }
+  memset(&op, 0, sizeof(op));
+  op.connectivity_state = state;
+  op.on_connectivity_state_change = closure;
+  op.bind_pollset_set = interested_parties;
+  elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(con), 0);
+  elem->filter->start_transport_op(exec_ctx, elem, &op);
+}
 
-  switch (sw->connectivity_state) {
-    case GRPC_CHANNEL_CONNECTING:
-    case GRPC_CHANNEL_READY:
-    case GRPC_CHANNEL_IDLE:
-      /* all is still good: keep watching */
-      memset(&op, 0, sizeof(op));
-      op.connectivity_state = &sw->connectivity_state;
-      op.on_connectivity_state_change = &sw->closure;
-      elem = grpc_channel_stack_element(
-          CHANNEL_STACK_FROM_CONNECTION(c->active), 0);
-      elem->filter->start_transport_op(exec_ctx, elem, &op);
-      /* early out */
-      gpr_mu_unlock(mu);
-      return;
-    case GRPC_CHANNEL_FATAL_FAILURE:
-    case GRPC_CHANNEL_TRANSIENT_FAILURE:
-      /* things have gone wrong, deactivate and enter idle */
-      if (sw->subchannel->active->refs == 0) {
-        destroy_connection = sw->subchannel->active;
-      }
-      sw->subchannel->active = NULL;
-      grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
-                                  c->disconnected
-                                      ? GRPC_CHANNEL_FATAL_FAILURE
-                                      : GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                  "connection_failed");
-      break;
-  }
+void grpc_connected_subchannel_notify_on_state_change(
+    grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *con,
+    grpc_pollset_set *interested_parties, grpc_connectivity_state *state,
+    grpc_closure *closure) {
+  connected_subchannel_state_op(exec_ctx, con, interested_parties, state,
+                                closure);
+}
 
-done:
-  connectivity_state_changed_locked(exec_ctx, c, "transport_state_changed");
-  destroy = SUBCHANNEL_UNREF_LOCKED(c, "state_watcher");
-  gpr_free(sw);
-  gpr_mu_unlock(mu);
-  if (destroy) {
-    subchannel_destroy(exec_ctx, c);
-  }
-  if (destroy_connection != NULL) {
-    connection_destroy(exec_ctx, destroy_connection);
-  }
+void grpc_connected_subchannel_ping(grpc_exec_ctx *exec_ctx,
+                                    grpc_connected_subchannel *con,
+                                    grpc_closure *closure) {
+  grpc_transport_op op;
+  grpc_channel_element *elem;
+  memset(&op, 0, sizeof(op));
+  op.send_ping = closure;
+  elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(con), 0);
+  elem->filter->start_transport_op(exec_ctx, elem, &op);
 }
 
 static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   size_t channel_stack_size;
-  connection *con;
+  grpc_connected_subchannel *con;
   grpc_channel_stack *stk;
   size_t num_filters;
   const grpc_channel_filter **filters;
-  waiting_for_connect *w4c;
-  grpc_transport_op op;
-  state_watcher *sw;
-  connection *destroy_connection = NULL;
-  grpc_channel_element *elem;
+  state_watcher *sw_subchannel;
 
   /* build final filter list */
   num_filters = c->num_filters + c->connecting_result.num_filters + 1;
@@ -621,74 +490,52 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
 
   /* construct channel stack */
   channel_stack_size = grpc_channel_stack_size(filters, num_filters);
-  con = gpr_malloc(sizeof(connection) + channel_stack_size);
-  stk = (grpc_channel_stack *)(con + 1);
-  con->refs = 0;
-  con->subchannel = c;
-  grpc_channel_stack_init(exec_ctx, filters, num_filters, c->master, c->args,
-                          stk);
+  con = gpr_malloc(channel_stack_size);
+  stk = CHANNEL_STACK_FROM_CONNECTION(con);
+  grpc_channel_stack_init(exec_ctx, 1, connection_destroy, con, filters,
+                          num_filters, c->args, "CONNECTED_SUBCHANNEL", stk);
   grpc_connected_channel_bind_transport(stk, c->connecting_result.transport);
   gpr_free((void *)c->connecting_result.filters);
   memset(&c->connecting_result, 0, sizeof(c->connecting_result));
 
   /* initialize state watcher */
-  sw = gpr_malloc(sizeof(*sw));
-  grpc_closure_init(&sw->closure, on_state_changed, sw);
-  sw->subchannel = c;
-  sw->connectivity_state = GRPC_CHANNEL_READY;
+  sw_subchannel = gpr_malloc(sizeof(*sw_subchannel));
+  sw_subchannel->subchannel = c;
+  sw_subchannel->connectivity_state = GRPC_CHANNEL_READY;
+  grpc_closure_init(&sw_subchannel->closure, subchannel_on_child_state_changed,
+                    sw_subchannel);
 
   gpr_mu_lock(&c->mu);
 
   if (c->disconnected) {
     gpr_mu_unlock(&c->mu);
-    gpr_free(sw);
+    gpr_free(sw_subchannel);
     gpr_free((void *)filters);
     grpc_channel_stack_destroy(exec_ctx, stk);
-    GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->master, "connecting");
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, c, "connecting");
+    gpr_free(con);
+    GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
     return;
   }
 
   /* publish */
-  if (c->active != NULL && c->active->refs == 0) {
-    destroy_connection = c->active;
-  }
-  c->active = con;
-  c->active_version++;
-  sw->version = c->active_version;
+  GPR_ASSERT(gpr_atm_no_barrier_cas(&c->connected_subchannel, 0, (gpr_atm)con));
   c->connecting = 0;
 
-  /* watch for changes; subchannel ref for connecting is donated
+  /* setup subchannel watching connected subchannel for changes; subchannel ref
+     for connecting is donated
      to the state watcher */
-  memset(&op, 0, sizeof(op));
-  op.connectivity_state = &sw->connectivity_state;
-  op.on_connectivity_state_change = &sw->closure;
-  op.bind_pollset_set = c->pollset_set;
-  SUBCHANNEL_REF_LOCKED(c, "state_watcher");
-  GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->master, "connecting");
-  GPR_ASSERT(!SUBCHANNEL_UNREF_LOCKED(c, "connecting"));
-  elem =
-      grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(c->active), 0);
-  elem->filter->start_transport_op(exec_ctx, elem, &op);
+  GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
+  grpc_connected_subchannel_notify_on_state_change(
+      exec_ctx, con, &c->pollset_set, &sw_subchannel->connectivity_state,
+      &sw_subchannel->closure);
 
   /* signal completion */
-  connectivity_state_changed_locked(exec_ctx, c, "connected");
-  w4c = c->waiting;
-  c->waiting = NULL;
+  grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_READY,
+                              "connected");
 
   gpr_mu_unlock(&c->mu);
-
-  while (w4c != NULL) {
-    waiting_for_connect *next = w4c->next;
-    grpc_exec_ctx_enqueue(exec_ctx, &w4c->continuation, 1);
-    w4c = next;
-  }
-
   gpr_free((void *)filters);
-
-  if (destroy_connection != NULL) {
-    connection_destroy(exec_ctx, destroy_connection);
-  }
 }
 
 /* Generate a random number between 0 and 1. */
@@ -742,29 +589,31 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, int iomgr_success) {
   if (c->disconnected) {
     iomgr_success = 0;
   }
-  connectivity_state_changed_locked(exec_ctx, c, "alarm");
   gpr_mu_unlock(&c->mu);
   if (iomgr_success) {
     update_reconnect_parameters(c);
     continue_connect(exec_ctx, c);
   } else {
-    cancel_waiting_calls(exec_ctx, c, iomgr_success);
-    GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->master, "connecting");
-    GRPC_SUBCHANNEL_UNREF(exec_ctx, c, "connecting");
+    GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   }
 }
 
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
                                  int iomgr_success) {
   grpc_subchannel *c = arg;
+
   if (c->connecting_result.transport != NULL) {
     publish_transport(exec_ctx, c);
+  } else if (c->disconnected) {
+    GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   } else {
     gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
     gpr_mu_lock(&c->mu);
     GPR_ASSERT(!c->have_alarm);
     c->have_alarm = 1;
-    connectivity_state_changed_locked(exec_ctx, c, "connect_failed");
+    grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
+                                GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                "connect_failed");
     grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now);
     gpr_mu_unlock(&c->mu);
   }
@@ -781,29 +630,6 @@ static gpr_timespec compute_connect_deadline(grpc_subchannel *c) {
                                                           : min_deadline;
 }
 
-static grpc_connectivity_state compute_connectivity_locked(grpc_subchannel *c) {
-  if (c->disconnected) {
-    return GRPC_CHANNEL_FATAL_FAILURE;
-  }
-  if (c->connecting) {
-    if (c->have_alarm) {
-      return GRPC_CHANNEL_TRANSIENT_FAILURE;
-    }
-    return GRPC_CHANNEL_CONNECTING;
-  }
-  if (c->active) {
-    return GRPC_CHANNEL_READY;
-  }
-  return GRPC_CHANNEL_IDLE;
-}
-
-static void connectivity_state_changed_locked(grpc_exec_ctx *exec_ctx,
-                                              grpc_subchannel *c,
-                                              const char *reason) {
-  grpc_connectivity_state current = compute_connectivity_locked(c);
-  grpc_connectivity_state_set(exec_ctx, &c->state_tracker, current, reason);
-}
-
 /*
  * grpc_subchannel_call implementation
  */
@@ -811,37 +637,22 @@ static void connectivity_state_changed_locked(grpc_exec_ctx *exec_ctx,
 static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call,
                                     int success) {
   grpc_subchannel_call *c = call;
-  gpr_mu *mu = &c->connection->subchannel->mu;
-  grpc_subchannel *destroy;
   GPR_TIMER_BEGIN("grpc_subchannel_call_unref.destroy", 0);
   grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c));
-  gpr_mu_lock(mu);
-  destroy = CONNECTION_UNREF_LOCKED(exec_ctx, c->connection, "call");
-  gpr_mu_unlock(mu);
+  GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, c->connection, "subchannel_call");
   gpr_free(c);
-  if (destroy != NULL) {
-    subchannel_destroy(exec_ctx, destroy);
-  }
   GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0);
 }
 
 void grpc_subchannel_call_ref(grpc_subchannel_call *c
                                   GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-#ifdef GRPC_STREAM_REFCOUNT_DEBUG
-  grpc_call_stack_ref(SUBCHANNEL_CALL_TO_CALL_STACK(c), reason);
-#else
-  grpc_call_stack_ref(SUBCHANNEL_CALL_TO_CALL_STACK(c));
-#endif
+  GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON);
 }
 
 void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel_call *c
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-#ifdef GRPC_STREAM_REFCOUNT_DEBUG
-  grpc_call_stack_unref(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), reason);
-#else
-  grpc_call_stack_unref(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c));
-#endif
+  GRPC_CALL_STACK_UNREF(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON);
 }
 
 char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx,
@@ -859,24 +670,26 @@ void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
   top_elem->filter->start_transport_stream_op(exec_ctx, top_elem, op);
 }
 
-static grpc_subchannel_call *create_call(grpc_exec_ctx *exec_ctx,
-                                         connection *con,
-                                         grpc_pollset *pollset) {
+grpc_connected_subchannel *grpc_subchannel_get_connected_subchannel(
+    grpc_subchannel *c) {
+  return GET_CONNECTED_SUBCHANNEL(c, acq);
+}
+
+grpc_subchannel_call *grpc_connected_subchannel_create_call(
+    grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *con,
+    grpc_pollset *pollset) {
   grpc_channel_stack *chanstk = CHANNEL_STACK_FROM_CONNECTION(con);
   grpc_subchannel_call *call =
       gpr_malloc(sizeof(grpc_subchannel_call) + chanstk->call_stack_size);
   grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(call);
   call->connection = con;
+  GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call");
   grpc_call_stack_init(exec_ctx, chanstk, 1, subchannel_call_destroy, call,
                        NULL, NULL, callstk);
   grpc_call_stack_set_pollset(exec_ctx, callstk, pollset);
   return call;
 }
 
-grpc_channel *grpc_subchannel_get_master(grpc_subchannel *subchannel) {
-  return subchannel->master;
-}
-
 grpc_call_stack *grpc_subchannel_call_get_call_stack(
     grpc_subchannel_call *subchannel_call) {
   return SUBCHANNEL_CALL_TO_CALL_STACK(subchannel_call);
diff --git a/src/core/client_config/subchannel.h b/src/core/client_config/subchannel.h
index 85ea3739e4..57c7c9dc67 100644
--- a/src/core/client_config/subchannel.h
+++ b/src/core/client_config/subchannel.h
@@ -41,6 +41,7 @@
 /** A (sub-)channel that knows how to connect to exactly one target
     address. Provides a target for load balancing. */
 typedef struct grpc_subchannel grpc_subchannel;
+typedef struct grpc_connected_subchannel grpc_connected_subchannel;
 typedef struct grpc_subchannel_call grpc_subchannel_call;
 typedef struct grpc_subchannel_args grpc_subchannel_args;
 
@@ -49,6 +50,14 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
   grpc_subchannel_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_UNREF(cl, p, r) \
   grpc_subchannel_unref((cl), (p), __FILE__, __LINE__, (r))
+#define GRPC_SUBCHANNEL_WEAK_REF(p, r) \
+  grpc_subchannel_weak_ref((p), __FILE__, __LINE__, (r))
+#define GRPC_SUBCHANNEL_WEAK_UNREF(cl, p, r) \
+  grpc_subchannel_weak_unref((cl), (p), __FILE__, __LINE__, (r))
+#define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) \
+  grpc_connected_subchannel_ref((p), __FILE__, __LINE__, (r))
+#define GRPC_CONNECTED_SUBCHANNEL_UNREF(cl, p, r) \
+  grpc_connected_subchannel_unref((cl), (p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_CALL_REF(p, r) \
   grpc_subchannel_call_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_CALL_UNREF(cl, p, r) \
@@ -58,6 +67,12 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
 #else
 #define GRPC_SUBCHANNEL_REF(p, r) grpc_subchannel_ref((p))
 #define GRPC_SUBCHANNEL_UNREF(cl, p, r) grpc_subchannel_unref((cl), (p))
+#define GRPC_SUBCHANNEL_WEAK_REF(p, r) grpc_subchannel_weak_ref((p))
+#define GRPC_SUBCHANNEL_WEAK_UNREF(cl, p, r) \
+  grpc_subchannel_weak_unref((cl), (p))
+#define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) grpc_connected_subchannel_ref((p))
+#define GRPC_CONNECTED_SUBCHANNEL_UNREF(cl, p, r) \
+  grpc_connected_subchannel_unref((cl), (p))
 #define GRPC_SUBCHANNEL_CALL_REF(p, r) grpc_subchannel_call_ref((p))
 #define GRPC_SUBCHANNEL_CALL_UNREF(cl, p, r) \
   grpc_subchannel_call_unref((cl), (p))
@@ -69,33 +84,31 @@ void grpc_subchannel_ref(grpc_subchannel *channel
 void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx,
                            grpc_subchannel *channel
                                GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_subchannel_weak_ref(grpc_subchannel *channel
+                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
+                                grpc_subchannel *channel
+                                    GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_connected_subchannel_ref(grpc_connected_subchannel *channel
+                                       GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx,
+                                     grpc_connected_subchannel *channel
+                                         GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_ref(grpc_subchannel_call *call
                                   GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel_call *call
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 
-/** construct a subchannel call (possibly asynchronously).
- *
- * If the returned status is 1, the call will return immediately and \a target
- * will point to a connected \a subchannel_call instance. Note that \a notify
- * will \em not be invoked in this case.
- * Otherwise, if the returned status is 0, the subchannel call will be created
- * asynchronously, invoking the \a notify callback upon completion. */
-int grpc_subchannel_create_call(grpc_exec_ctx *exec_ctx,
-                                grpc_subchannel *subchannel,
-                                grpc_pollset *pollset, gpr_atm *target,
-                                grpc_closure *notify);
-
-/** cancel \a call in the waiting state. */
-void grpc_subchannel_cancel_create_call(grpc_exec_ctx *exec_ctx,
-                                        grpc_subchannel *subchannel,
-                                        gpr_atm *target);
+/** construct a subchannel call */
+grpc_subchannel_call *grpc_connected_subchannel_create_call(
+    grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *connected_subchannel,
+    grpc_pollset *pollset);
 
 /** process a transport level op */
-void grpc_subchannel_process_transport_op(grpc_exec_ctx *exec_ctx,
-                                          grpc_subchannel *subchannel,
-                                          grpc_transport_op *op);
+void grpc_connected_subchannel_process_transport_op(
+    grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *subchannel,
+    grpc_transport_op *op);
 
 /** poll the current connectivity state of a channel */
 grpc_connectivity_state grpc_subchannel_check_connectivity(
@@ -103,26 +116,22 @@ grpc_connectivity_state grpc_subchannel_check_connectivity(
 
 /** call notify when the connectivity state of a channel changes from *state.
     Updates *state with the new state of the channel */
-void grpc_subchannel_notify_on_state_change(grpc_exec_ctx *exec_ctx,
-                                            grpc_subchannel *channel,
-                                            grpc_connectivity_state *state,
-                                            grpc_closure *notify);
-
-/** Remove \a subscribed_notify from the list of closures to be called on a
- * state change if present, returning 1. Otherwise, nothing is done and return
- * 0. */
-int grpc_subchannel_state_change_unsubscribe(grpc_exec_ctx *exec_ctx,
-                                             grpc_subchannel *channel,
-                                             grpc_closure *subscribed_notify);
-
-/** express interest in \a channel's activities through \a pollset. */
-void grpc_subchannel_add_interested_party(grpc_exec_ctx *exec_ctx,
-                                          grpc_subchannel *channel,
-                                          grpc_pollset *pollset);
-/** stop following \a channel's activity through \a pollset. */
-void grpc_subchannel_del_interested_party(grpc_exec_ctx *exec_ctx,
-                                          grpc_subchannel *channel,
-                                          grpc_pollset *pollset);
+void grpc_subchannel_notify_on_state_change(
+    grpc_exec_ctx *exec_ctx, grpc_subchannel *channel,
+    grpc_pollset_set *interested_parties, grpc_connectivity_state *state,
+    grpc_closure *notify);
+void grpc_connected_subchannel_notify_on_state_change(
+    grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *channel,
+    grpc_pollset_set *interested_parties, grpc_connectivity_state *state,
+    grpc_closure *notify);
+void grpc_connected_subchannel_ping(grpc_exec_ctx *exec_ctx,
+                                    grpc_connected_subchannel *channel,
+                                    grpc_closure *notify);
+
+/** retrieve the grpc_connected_subchannel - or NULL if called before
+    the subchannel becomes connected */
+grpc_connected_subchannel *grpc_subchannel_get_connected_subchannel(
+    grpc_subchannel *subchannel);
 
 /** continue processing a transport op */
 void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
@@ -147,15 +156,10 @@ struct grpc_subchannel_args {
   /** Address to connect to */
   struct sockaddr *addr;
   size_t addr_len;
-  /** master channel */
-  grpc_channel *master;
 };
 
 /** create a subchannel given a connector */
 grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
                                         grpc_subchannel_args *args);
 
-/** Return the master channel associated with the subchannel */
-grpc_channel *grpc_subchannel_get_master(grpc_subchannel *subchannel);
-
 #endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_H */
diff --git a/src/core/compression/algorithm.c b/src/core/compression/algorithm.c
index 73d91fa8ea..8e4e5c91d4 100644
--- a/src/core/compression/algorithm.c
+++ b/src/core/compression/algorithm.c
@@ -119,8 +119,8 @@ grpc_mdelem *grpc_compression_encoding_mdelem(
       return GRPC_MDELEM_GRPC_ENCODING_DEFLATE;
     case GRPC_COMPRESS_GZIP:
       return GRPC_MDELEM_GRPC_ENCODING_GZIP;
-    case GRPC_COMPRESS_ALGORITHMS_COUNT:
-      return NULL;
+    default:
+      break;
   }
   return NULL;
 }
@@ -139,25 +139,9 @@ grpc_compression_algorithm grpc_compression_algorithm_for_level(
     case GRPC_COMPRESS_LEVEL_HIGH:
       return GRPC_COMPRESS_DEFLATE;
     default:
-      /* we shouldn't be making it here */
-      abort();
-      return GRPC_COMPRESS_NONE;
-  }
-}
-
-grpc_compression_level grpc_compression_level_for_algorithm(
-    grpc_compression_algorithm algorithm) {
-  grpc_compression_level clevel;
-  GRPC_API_TRACE("grpc_compression_level_for_algorithm(algorithm=%d)", 1,
-                 ((int)algorithm));
-  for (clevel = GRPC_COMPRESS_LEVEL_NONE; clevel < GRPC_COMPRESS_LEVEL_COUNT;
-       ++clevel) {
-    if (grpc_compression_algorithm_for_level(clevel) == algorithm) {
-      return clevel;
-    }
+      break;
   }
-  abort();
-  return GRPC_COMPRESS_LEVEL_NONE;
+  GPR_UNREACHABLE_CODE(return GRPC_COMPRESS_NONE);
 }
 
 void grpc_compression_options_init(grpc_compression_options *opts) {
diff --git a/src/core/compression/message_compress.c b/src/core/compression/message_compress.c
index a723c47819..edc21a9eb7 100644
--- a/src/core/compression/message_compress.c
+++ b/src/core/compression/message_compress.c
@@ -69,8 +69,8 @@ static int zlib_body(z_stream* zs, gpr_slice_buffer* input,
         zs->next_out = GPR_SLICE_START_PTR(outbuf);
       }
       r = flate(zs, flush);
-      if (r == Z_STREAM_ERROR) {
-        gpr_log(GPR_INFO, "zlib: stream error");
+      if (r < 0 && r != Z_BUF_ERROR /* not fatal */) {
+        gpr_log(GPR_INFO, "zlib error (%d)", r);
         goto error;
       }
     } while (zs->avail_out == 0);
@@ -91,13 +91,11 @@ error:
   return 0;
 }
 
-static void *zalloc_gpr(void* opaque, unsigned int items, unsigned int size) {
+static void* zalloc_gpr(void* opaque, unsigned int items, unsigned int size) {
   return gpr_malloc(items * size);
 }
 
-static void zfree_gpr(void* opaque, void *address) {
-  gpr_free(address);
-}
+static void zfree_gpr(void* opaque, void* address) { gpr_free(address); }
 
 static int zlib_compress(gpr_slice_buffer* input, gpr_slice_buffer* output,
                          int gzip) {
@@ -111,10 +109,7 @@ static int zlib_compress(gpr_slice_buffer* input, gpr_slice_buffer* output,
   zs.zfree = zfree_gpr;
   r = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | (gzip ? 16 : 0),
                    8, Z_DEFAULT_STRATEGY);
-  if (r != Z_OK) {
-    gpr_log(GPR_ERROR, "deflateInit2 returns %d", r);
-    return 0;
-  }
+  GPR_ASSERT(r == Z_OK);
   r = zlib_body(&zs, input, output, deflate) && output->length < input->length;
   if (!r) {
     for (i = count_before; i < output->count; i++) {
@@ -138,10 +133,7 @@ static int zlib_decompress(gpr_slice_buffer* input, gpr_slice_buffer* output,
   zs.zalloc = zalloc_gpr;
   zs.zfree = zfree_gpr;
   r = inflateInit2(&zs, 15 | (gzip ? 16 : 0));
-  if (r != Z_OK) {
-    gpr_log(GPR_ERROR, "inflateInit2 returns %d", r);
-    return 0;
-  }
+  GPR_ASSERT(r == Z_OK);
   r = zlib_body(&zs, input, output, inflate);
   if (!r) {
     for (i = count_before; i < output->count; i++) {
@@ -163,7 +155,7 @@ static int copy(gpr_slice_buffer* input, gpr_slice_buffer* output) {
 }
 
 static int compress_inner(grpc_compression_algorithm algorithm,
-                   gpr_slice_buffer* input, gpr_slice_buffer* output) {
+                          gpr_slice_buffer* input, gpr_slice_buffer* output) {
   switch (algorithm) {
     case GRPC_COMPRESS_NONE:
       /* the fallback path always needs to be send uncompressed: we simply
diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c
index a87f1aa87b..b5cd8d8d2a 100644
--- a/src/core/httpcli/httpcli.c
+++ b/src/core/httpcli/httpcli.c
@@ -53,6 +53,7 @@ typedef struct {
   size_t next_address;
   grpc_endpoint *ep;
   char *host;
+  char *ssl_host_override;
   gpr_timespec deadline;
   int have_read_byte;
   const grpc_httpcli_handshaker *handshaker;
@@ -106,6 +107,7 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
   }
   gpr_slice_unref(req->request_text);
   gpr_free(req->host);
+  gpr_free(req->ssl_host_override);
   grpc_iomgr_unregister_object(&req->iomgr_obj);
   gpr_slice_buffer_destroy(&req->incoming);
   gpr_slice_buffer_destroy(&req->outgoing);
@@ -180,8 +182,10 @@ static void on_connected(grpc_exec_ctx *exec_ctx, void *arg, int success) {
     next_address(exec_ctx, req);
     return;
   }
-  req->handshaker->handshake(exec_ctx, req, req->ep, req->host,
-                             on_handshake_done);
+  req->handshaker->handshake(
+      exec_ctx, req, req->ep,
+      req->ssl_host_override ? req->ssl_host_override : req->host,
+      on_handshake_done);
 }
 
 static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req) {
@@ -231,6 +235,7 @@ static void internal_request_begin(
   gpr_slice_buffer_init(&req->outgoing);
   grpc_iomgr_register_object(&req->iomgr_obj, name);
   req->host = gpr_strdup(request->host);
+  req->ssl_host_override = gpr_strdup(request->ssl_host_override);
 
   grpc_pollset_set_add_pollset(exec_ctx, &req->context->pollset_set,
                                req->pollset);
diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h
index 6469c2f03e..30875d71f1 100644
--- a/src/core/httpcli/httpcli.h
+++ b/src/core/httpcli/httpcli.h
@@ -74,6 +74,8 @@ extern const grpc_httpcli_handshaker grpc_httpcli_ssl;
 typedef struct grpc_httpcli_request {
   /* The host name to connect to */
   char *host;
+  /* The host to verify in the SSL handshake (or NULL) */
+  char *ssl_host_override;
   /* The path of the resource to fetch */
   char *path;
   /* Additional headers: count and key/values; the following are supplied
diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c
index 2be0ea235f..00710d83bd 100644
--- a/src/core/iomgr/fd_posix.c
+++ b/src/core/iomgr/fd_posix.c
@@ -43,6 +43,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
 #define CLOSURE_NOT_READY ((grpc_closure *)0)
@@ -158,7 +159,10 @@ void grpc_fd_global_shutdown(void) {
 
 grpc_fd *grpc_fd_create(int fd, const char *name) {
   grpc_fd *r = alloc_fd(fd);
-  grpc_iomgr_register_object(&r->iomgr_object, name);
+  char *name2;
+  gpr_asprintf(&name2, "%s fd=%d", name, fd);
+  grpc_iomgr_register_object(&r->iomgr_object, name2);
+  gpr_free(name2);
 #ifdef GRPC_FD_REF_COUNT_DEBUG
   gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name);
 #endif
diff --git a/src/core/iomgr/fd_posix.h b/src/core/iomgr/fd_posix.h
index d628ef3aaf..df4eb64d4c 100644
--- a/src/core/iomgr/fd_posix.h
+++ b/src/core/iomgr/fd_posix.h
@@ -170,6 +170,7 @@ void grpc_fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd);
 void grpc_fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd);
 
 /* Reference counting for fds */
+/*#define GRPC_FD_REF_COUNT_DEBUG*/
 #ifdef GRPC_FD_REF_COUNT_DEBUG
 void grpc_fd_ref(grpc_fd *fd, const char *reason, const char *file, int line);
 void grpc_fd_unref(grpc_fd *fd, const char *reason, const char *file, int line);
diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c
index 1f1bf47e98..6e31efa013 100644
--- a/src/core/iomgr/pollset_multipoller_with_epoll.c
+++ b/src/core/iomgr/pollset_multipoller_with_epoll.c
@@ -123,26 +123,6 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static void multipoll_with_epoll_pollset_del_fd(grpc_exec_ctx *exec_ctx,
-                                                grpc_pollset *pollset,
-                                                grpc_fd *fd,
-                                                int and_unlock_pollset) {
-  pollset_hdr *h = pollset->data.ptr;
-  int err;
-
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-
-  /* Note that this can race with concurrent poll, but that should be fine since
-   * at worst it creates a spurious read event on a reused grpc_fd object. */
-  err = epoll_ctl(h->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL);
-  if (err < 0) {
-    gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd->fd,
-            strerror(errno));
-  }
-}
-
 /* TODO(klempner): We probably want to turn this down a bit */
 #define GRPC_EPOLL_MAX_EVENTS 1000
 
@@ -235,7 +215,7 @@ static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) {
 }
 
 static const grpc_pollset_vtable multipoll_with_epoll_pollset = {
-    multipoll_with_epoll_pollset_add_fd, multipoll_with_epoll_pollset_del_fd,
+    multipoll_with_epoll_pollset_add_fd,
     multipoll_with_epoll_pollset_maybe_work_and_unlock,
     multipoll_with_epoll_pollset_finish_shutdown,
     multipoll_with_epoll_pollset_destroy};
diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
index 09f04b64b9..b619b8c3db 100644
--- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c
+++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
@@ -82,23 +82,6 @@ exit:
   }
 }
 
-static void multipoll_with_poll_pollset_del_fd(grpc_exec_ctx *exec_ctx,
-                                               grpc_pollset *pollset,
-                                               grpc_fd *fd,
-                                               int and_unlock_pollset) {
-  /* will get removed next poll cycle */
-  pollset_hdr *h = pollset->data.ptr;
-  if (h->del_count == h->del_capacity) {
-    h->del_capacity = GPR_MAX(h->del_capacity + 8, h->del_count * 3 / 2);
-    h->dels = gpr_realloc(h->dels, sizeof(grpc_fd *) * h->del_capacity);
-  }
-  h->dels[h->del_count++] = fd;
-  GRPC_FD_REF(fd, "multipoller_del");
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-}
-
 static void multipoll_with_poll_pollset_maybe_work_and_unlock(
     grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
     gpr_timespec deadline, gpr_timespec now) {
@@ -212,7 +195,7 @@ static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
 }
 
 static const grpc_pollset_vtable multipoll_with_poll_pollset = {
-    multipoll_with_poll_pollset_add_fd, multipoll_with_poll_pollset_del_fd,
+    multipoll_with_poll_pollset_add_fd,
     multipoll_with_poll_pollset_maybe_work_and_unlock,
     multipoll_with_poll_pollset_finish_shutdown,
     multipoll_with_poll_pollset_destroy};
diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c
index 0a5577baea..9195344758 100644
--- a/src/core/iomgr/pollset_posix.c
+++ b/src/core/iomgr/pollset_posix.c
@@ -232,21 +232,7 @@ void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   gpr_mu_lock(&pollset->mu);
   pollset->vtable->add_fd(exec_ctx, pollset, fd, 1);
 /* the following (enabled only in debug) will reacquire and then release
-   our lock - meaning that if the unlocking flag passed to del_fd above is
-   not respected, the code will deadlock (in a way that we have a chance of
-   debugging) */
-#ifndef NDEBUG
-  gpr_mu_lock(&pollset->mu);
-  gpr_mu_unlock(&pollset->mu);
-#endif
-}
-
-void grpc_pollset_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                         grpc_fd *fd) {
-  gpr_mu_lock(&pollset->mu);
-  pollset->vtable->del_fd(exec_ctx, pollset, fd, 1);
-/* the following (enabled only in debug) will reacquire and then release
-   our lock - meaning that if the unlocking flag passed to del_fd above is
+   our lock - meaning that if the unlocking flag passed to add_fd above is
    not respected, the code will deadlock (in a way that we have a chance of
    debugging) */
 #ifndef NDEBUG
@@ -547,19 +533,6 @@ exit:
   }
 }
 
-static void basic_pollset_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                 grpc_fd *fd, int and_unlock_pollset) {
-  GPR_ASSERT(fd);
-  if (fd == pollset->data.ptr) {
-    GRPC_FD_UNREF(pollset->data.ptr, "basicpoll");
-    pollset->data.ptr = NULL;
-  }
-
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-}
-
 static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
                                                 grpc_pollset *pollset,
                                                 grpc_pollset_worker *worker,
@@ -651,9 +624,8 @@ static void basic_pollset_destroy(grpc_pollset *pollset) {
 }
 
 static const grpc_pollset_vtable basic_pollset = {
-    basic_pollset_add_fd, basic_pollset_del_fd,
-    basic_pollset_maybe_work_and_unlock, basic_pollset_destroy,
-    basic_pollset_destroy};
+    basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock,
+    basic_pollset_destroy, basic_pollset_destroy};
 
 static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) {
   pollset->vtable = &basic_pollset;
diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h
index e4593728bd..29de4a2026 100644
--- a/src/core/iomgr/pollset_posix.h
+++ b/src/core/iomgr/pollset_posix.h
@@ -86,8 +86,6 @@ typedef struct grpc_pollset {
 struct grpc_pollset_vtable {
   void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                  struct grpc_fd *fd, int and_unlock_pollset);
-  void (*del_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                 struct grpc_fd *fd, int and_unlock_pollset);
   void (*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                                 grpc_pollset_worker *worker,
                                 gpr_timespec deadline, gpr_timespec now);
@@ -100,10 +98,6 @@ struct grpc_pollset_vtable {
 /* Add an fd to a pollset */
 void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                          struct grpc_fd *fd);
-/* Force remove an fd from a pollset (normally they are removed on the next
-   poll after an fd is orphaned) */
-void grpc_pollset_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                         struct grpc_fd *fd);
 
 /* Returns the fd to listen on for kicks */
 int grpc_kick_read_fd(grpc_pollset *p);
diff --git a/src/core/iomgr/pollset_set.h b/src/core/iomgr/pollset_set.h
index 0fdcba01a4..09c04438f7 100644
--- a/src/core/iomgr/pollset_set.h
+++ b/src/core/iomgr/pollset_set.h
@@ -49,13 +49,19 @@
 #include "src/core/iomgr/pollset_set_windows.h"
 #endif
 
-void grpc_pollset_set_init(grpc_pollset_set* pollset_set);
-void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set);
-void grpc_pollset_set_add_pollset(grpc_exec_ctx* exec_ctx,
-                                  grpc_pollset_set* pollset_set,
-                                  grpc_pollset* pollset);
-void grpc_pollset_set_del_pollset(grpc_exec_ctx* exec_ctx,
-                                  grpc_pollset_set* pollset_set,
-                                  grpc_pollset* pollset);
+void grpc_pollset_set_init(grpc_pollset_set *pollset_set);
+void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set);
+void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
+                                  grpc_pollset_set *pollset_set,
+                                  grpc_pollset *pollset);
+void grpc_pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
+                                  grpc_pollset_set *pollset_set,
+                                  grpc_pollset *pollset);
+void grpc_pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
+                                      grpc_pollset_set *bag,
+                                      grpc_pollset_set *item);
+void grpc_pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
+                                      grpc_pollset_set *bag,
+                                      grpc_pollset_set *item);
 
 #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */
diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c
index c86ed3d5da..4ec92202e3 100644
--- a/src/core/iomgr/pollset_set_posix.c
+++ b/src/core/iomgr/pollset_set_posix.c
@@ -52,9 +52,10 @@ void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) {
   size_t i;
   gpr_mu_destroy(&pollset_set->mu);
   for (i = 0; i < pollset_set->fd_count; i++) {
-    GRPC_FD_UNREF(pollset_set->fds[i], "pollset");
+    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
   }
   gpr_free(pollset_set->pollsets);
+  gpr_free(pollset_set->pollset_sets);
   gpr_free(pollset_set->fds);
 }
 
@@ -73,7 +74,7 @@ void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
   pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
   for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
     if (grpc_fd_is_orphaned(pollset_set->fds[i])) {
-      GRPC_FD_UNREF(pollset_set->fds[i], "pollset");
+      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
     } else {
       grpc_pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
       pollset_set->fds[j++] = pollset_set->fds[i];
@@ -99,6 +100,46 @@ void grpc_pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
   gpr_mu_unlock(&pollset_set->mu);
 }
 
+void grpc_pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
+                                      grpc_pollset_set *bag,
+                                      grpc_pollset_set *item) {
+  size_t i, j;
+  gpr_mu_lock(&bag->mu);
+  if (bag->pollset_set_count == bag->pollset_set_capacity) {
+    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
+    bag->pollset_sets =
+        gpr_realloc(bag->pollset_sets,
+                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
+  }
+  bag->pollset_sets[bag->pollset_set_count++] = item;
+  for (i = 0, j = 0; i < bag->fd_count; i++) {
+    if (grpc_fd_is_orphaned(bag->fds[i])) {
+      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
+    } else {
+      grpc_pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
+      bag->fds[j++] = bag->fds[i];
+    }
+  }
+  bag->fd_count = j;
+  gpr_mu_unlock(&bag->mu);
+}
+
+void grpc_pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
+                                      grpc_pollset_set *bag,
+                                      grpc_pollset_set *item) {
+  size_t i;
+  gpr_mu_lock(&bag->mu);
+  for (i = 0; i < bag->pollset_set_count; i++) {
+    if (bag->pollset_sets[i] == item) {
+      bag->pollset_set_count--;
+      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
+               bag->pollset_sets[bag->pollset_set_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&bag->mu);
+}
+
 void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
                              grpc_pollset_set *pollset_set, grpc_fd *fd) {
   size_t i;
@@ -113,6 +154,9 @@ void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
   for (i = 0; i < pollset_set->pollset_count; i++) {
     grpc_pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
   }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    grpc_pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
   gpr_mu_unlock(&pollset_set->mu);
 }
 
@@ -129,6 +173,9 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
       break;
     }
   }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    grpc_pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
   gpr_mu_unlock(&pollset_set->mu);
 }
 
diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h
index 05234fb642..4820a61e4b 100644
--- a/src/core/iomgr/pollset_set_posix.h
+++ b/src/core/iomgr/pollset_set_posix.h
@@ -44,6 +44,10 @@ typedef struct grpc_pollset_set {
   size_t pollset_capacity;
   grpc_pollset **pollsets;
 
+  size_t pollset_set_count;
+  size_t pollset_set_capacity;
+  struct grpc_pollset_set **pollset_sets;
+
   size_t fd_count;
   size_t fd_capacity;
   grpc_fd **fds;
diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c
index 53d5d3dcd4..157b46ec32 100644
--- a/src/core/iomgr/pollset_set_windows.c
+++ b/src/core/iomgr/pollset_set_windows.c
@@ -49,4 +49,12 @@ void grpc_pollset_set_del_pollset(grpc_exec_ctx* exec_ctx,
                                   grpc_pollset_set* pollset_set,
                                   grpc_pollset* pollset) {}
 
+void grpc_pollset_set_add_pollset_set(grpc_exec_ctx* exec_ctx,
+                                      grpc_pollset_set* bag,
+                                      grpc_pollset_set* item) {}
+
+void grpc_pollset_set_del_pollset_set(grpc_exec_ctx* exec_ctx,
+                                      grpc_pollset_set* bag,
+                                      grpc_pollset_set* item) {}
+
 #endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c
index b758702da8..835675c390 100644
--- a/src/core/iomgr/tcp_server_posix.c
+++ b/src/core/iomgr/tcp_server_posix.c
@@ -532,8 +532,12 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
 }
 
 int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) {
-  grpc_tcp_listener *sp = listener;
-  return sp->port;
+  if (listener != NULL) {
+    grpc_tcp_listener *sp = listener;
+    return sp->port;
+  } else {
+    return 0;
+  }
 }
 
 void grpc_tcp_listener_ref(grpc_tcp_listener *listener) {
diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c
index a2425cd4d2..583cab4890 100644
--- a/src/core/iomgr/tcp_server_windows.c
+++ b/src/core/iomgr/tcp_server_windows.c
@@ -486,8 +486,12 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
 }
 
 int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) {
-  grpc_tcp_listener *sp = listener;
-  return sp->port;
+  if (listener != NULL) {
+    grpc_tcp_listener *sp = listener;
+    return sp->port;
+  } else {
+    return 0;
+  }
 }
 
 void grpc_tcp_listener_ref(grpc_tcp_listener *listener) {
diff --git a/src/core/iomgr/tcp_windows.c b/src/core/iomgr/tcp_windows.c
index 5ff78231bd..cc7f7ff8d2 100644
--- a/src/core/iomgr/tcp_windows.c
+++ b/src/core/iomgr/tcp_windows.c
@@ -197,7 +197,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
   tcp->read_slice = gpr_slice_malloc(8192);
 
-  buffer.len = GPR_SLICE_LENGTH(tcp->read_slice);
+  buffer.len = (ULONG)GPR_SLICE_LENGTH(
+      tcp->read_slice);  // we know slice size fits in 32bit.
   buffer.buf = (char *)GPR_SLICE_START_PTR(tcp->read_slice);
 
   TCP_REF(tcp, "read");
@@ -273,6 +274,7 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   WSABUF local_buffers[16];
   WSABUF *allocated = NULL;
   WSABUF *buffers = local_buffers;
+  size_t len;
 
   if (tcp->shutting_down) {
     grpc_exec_ctx_enqueue(exec_ctx, cb, 0);
@@ -281,19 +283,21 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
   tcp->write_cb = cb;
   tcp->write_slices = slices;
-
+  GPR_ASSERT(tcp->write_slices->count <= UINT_MAX);
   if (tcp->write_slices->count > GPR_ARRAY_SIZE(local_buffers)) {
     buffers = (WSABUF *)gpr_malloc(sizeof(WSABUF) * tcp->write_slices->count);
     allocated = buffers;
   }
 
   for (i = 0; i < tcp->write_slices->count; i++) {
-    buffers[i].len = GPR_SLICE_LENGTH(tcp->write_slices->slices[i]);
+    len = GPR_SLICE_LENGTH(tcp->write_slices->slices[i]);
+    GPR_ASSERT(len <= ULONG_MAX);
+    buffers[i].len = (ULONG)len;
     buffers[i].buf = (char *)GPR_SLICE_START_PTR(tcp->write_slices->slices[i]);
   }
 
   /* First, let's try a synchronous, non-blocking write. */
-  status = WSASend(socket->socket, buffers, tcp->write_slices->count,
+  status = WSASend(socket->socket, buffers, (DWORD)tcp->write_slices->count,
                    &bytes_sent, 0, NULL, NULL);
   info->wsa_error = status == 0 ? 0 : WSAGetLastError();
 
@@ -322,7 +326,7 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   /* If we got a WSAEWOULDBLOCK earlier, then we need to re-do the same
      operation, this time asynchronously. */
   memset(&socket->write_info.overlapped, 0, sizeof(OVERLAPPED));
-  status = WSASend(socket->socket, buffers, tcp->write_slices->count,
+  status = WSASend(socket->socket, buffers, (DWORD)tcp->write_slices->count,
                    &bytes_sent, 0, &socket->write_info.overlapped, NULL);
   if (allocated) gpr_free(allocated);
 
diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c
index 66fafe75ad..bbf9800049 100644
--- a/src/core/iomgr/timer.c
+++ b/src/core/iomgr/timer.c
@@ -126,8 +126,8 @@ static double ts_to_dbl(gpr_timespec ts) {
 
 static gpr_timespec dbl_to_ts(double d) {
   gpr_timespec ts;
-  ts.tv_sec = (time_t)d;
-  ts.tv_nsec = (int)(1e9 * (d - (double)ts.tv_sec));
+  ts.tv_sec = (gpr_int64)d;
+  ts.tv_nsec = (gpr_int32)(1e9 * (d - (double)ts.tv_sec));
   ts.clock_type = GPR_TIMESPAN;
   return ts;
 }
@@ -343,11 +343,3 @@ int grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
       exec_ctx, now, next,
       gpr_time_cmp(now, gpr_inf_future(now.clock_type)) != 0);
 }
-
-gpr_timespec grpc_timer_list_next_timeout(void) {
-  gpr_timespec out;
-  gpr_mu_lock(&g_mu);
-  out = g_shard_queue[0]->min_deadline;
-  gpr_mu_unlock(&g_mu);
-  return out;
-}
diff --git a/src/core/iomgr/timer_internal.h b/src/core/iomgr/timer_internal.h
index f180eca36e..f182e73764 100644
--- a/src/core/iomgr/timer_internal.h
+++ b/src/core/iomgr/timer_internal.h
@@ -54,8 +54,6 @@ int grpc_timer_check(grpc_exec_ctx* exec_ctx, gpr_timespec now,
 void grpc_timer_list_init(gpr_timespec now);
 void grpc_timer_list_shutdown(grpc_exec_ctx* exec_ctx);
 
-gpr_timespec grpc_timer_list_next_timeout(void);
-
 /* the following must be implemented by each iomgr implementation */
 
 void grpc_kick_poller(void);
diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c
index 782fbd9f46..28f1bfae26 100644
--- a/src/core/iomgr/udp_server.c
+++ b/src/core/iomgr/udp_server.c
@@ -38,6 +38,7 @@
 
 #include <grpc/support/port_platform.h>
 
+#ifdef GRPC_NEED_UDP
 #ifdef GPR_POSIX_SOCKET
 
 #include "src/core/iomgr/udp_server.h"
@@ -435,3 +436,4 @@ void grpc_udp_server_write(server_port *sp, const char *buffer, size_t buf_len,
 }
 
 #endif
+#endif
diff --git a/src/core/iomgr/wakeup_fd_posix.c b/src/core/iomgr/wakeup_fd_posix.c
index d09fb78d12..f40be081b0 100644
--- a/src/core/iomgr/wakeup_fd_posix.c
+++ b/src/core/iomgr/wakeup_fd_posix.c
@@ -40,19 +40,17 @@
 #include <stddef.h>
 
 static const grpc_wakeup_fd_vtable *wakeup_fd_vtable = NULL;
+int grpc_allow_specialized_wakeup_fd = 1;
 
 void grpc_wakeup_fd_global_init(void) {
-  if (grpc_specialized_wakeup_fd_vtable.check_availability()) {
+  if (grpc_allow_specialized_wakeup_fd &&
+      grpc_specialized_wakeup_fd_vtable.check_availability()) {
     wakeup_fd_vtable = &grpc_specialized_wakeup_fd_vtable;
   } else {
     wakeup_fd_vtable = &grpc_pipe_wakeup_fd_vtable;
   }
 }
 
-void grpc_wakeup_fd_global_init_force_fallback(void) {
-  wakeup_fd_vtable = &grpc_pipe_wakeup_fd_vtable;
-}
-
 void grpc_wakeup_fd_global_destroy(void) { wakeup_fd_vtable = NULL; }
 
 void grpc_wakeup_fd_init(grpc_wakeup_fd *fd_info) {
diff --git a/src/core/iomgr/wakeup_fd_posix.h b/src/core/iomgr/wakeup_fd_posix.h
index fe71b5abe9..ffd60d1d4e 100644
--- a/src/core/iomgr/wakeup_fd_posix.h
+++ b/src/core/iomgr/wakeup_fd_posix.h
@@ -85,6 +85,8 @@ struct grpc_wakeup_fd {
   int write_fd;
 };
 
+extern int grpc_allow_specialized_wakeup_fd;
+
 #define GRPC_WAKEUP_FD_GET_READ_FD(fd_info) ((fd_info)->read_fd)
 
 void grpc_wakeup_fd_init(grpc_wakeup_fd* fd_info);
diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c
index 2e30178131..d2a1c34612 100644
--- a/src/core/iomgr/workqueue_posix.c
+++ b/src/core/iomgr/workqueue_posix.c
@@ -103,6 +103,9 @@ void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx,
 
 void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
   gpr_mu_lock(&workqueue->mu);
+  if (grpc_closure_list_empty(workqueue->closure_list)) {
+    grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
+  }
   grpc_closure_list_move(&exec_ctx->closure_list, &workqueue->closure_list);
   gpr_mu_unlock(&workqueue->mu);
 }
diff --git a/src/core/json/json_reader.c b/src/core/json/json_reader.c
index 8abad01252..256995240a 100644
--- a/src/core/json/json_reader.c
+++ b/src/core/json/json_reader.c
@@ -35,6 +35,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <grpc/support/log.h>
+
 #include "src/core/json/json_reader.h"
 
 static void json_reader_string_clear(grpc_json_reader *reader) {
@@ -224,13 +226,13 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
                   reader->in_array = 1;
                   break;
                 case GRPC_JSON_TOP_LEVEL:
-                  if (reader->depth != 0) return GRPC_JSON_INTERNAL_ERROR;
+                  GPR_ASSERT(reader->depth == 0);
                   reader->in_object = 0;
                   reader->in_array = 0;
                   reader->state = GRPC_JSON_STATE_END;
                   break;
                 default:
-                  return GRPC_JSON_INTERNAL_ERROR;
+                  GPR_UNREACHABLE_CODE(return GRPC_JSON_INTERNAL_ERROR);
               }
             }
             break;
@@ -279,8 +281,7 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
             break;
 
           case GRPC_JSON_STATE_OBJECT_KEY_STRING:
-            if (reader->unicode_high_surrogate != 0)
-              return GRPC_JSON_PARSE_ERROR;
+            GPR_ASSERT(reader->unicode_high_surrogate == 0);
             if (c == '"') {
               reader->state = GRPC_JSON_STATE_OBJECT_KEY_END;
               json_reader_set_key(reader);
@@ -461,7 +462,7 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
                 }
                 break;
               default:
-                return GRPC_JSON_INTERNAL_ERROR;
+                GPR_UNREACHABLE_CODE(return GRPC_JSON_INTERNAL_ERROR);
             }
             break;
 
@@ -641,7 +642,7 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
               case ',':
               case '}':
               case ']':
-                return GRPC_JSON_INTERNAL_ERROR;
+                GPR_UNREACHABLE_CODE(return GRPC_JSON_INTERNAL_ERROR);
                 break;
 
               default:
@@ -655,5 +656,5 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
     }
   }
 
-  return GRPC_JSON_INTERNAL_ERROR;
+  GPR_UNREACHABLE_CODE(return GRPC_JSON_INTERNAL_ERROR);
 }
diff --git a/src/core/json/json_string.c b/src/core/json/json_string.c
index 0461c2703f..06c157dc98 100644
--- a/src/core/json/json_string.c
+++ b/src/core/json/json_string.c
@@ -353,7 +353,7 @@ static void json_dump_recursive(grpc_json_writer *writer, grpc_json *json,
         grpc_json_writer_value_raw_with_len(writer, "null", 4);
         break;
       default:
-        abort();
+        GPR_UNREACHABLE_CODE(abort());
     }
     json = json->next;
   }
diff --git a/src/core/profiling/basic_timers.c b/src/core/profiling/basic_timers.c
index f0fce7858d..eedd387ebc 100644
--- a/src/core/profiling/basic_timers.c
+++ b/src/core/profiling/basic_timers.c
@@ -141,10 +141,11 @@ static void write_log(gpr_timer_log *log) {
       entry->tm = gpr_time_0(entry->tm.clock_type);
     }
     fprintf(output_file,
-            "{\"t\": %ld.%09d, \"thd\": \"%d\", \"type\": \"%c\", \"tag\": "
+            "{\"t\": %lld.%09d, \"thd\": \"%d\", \"type\": \"%c\", \"tag\": "
             "\"%s\", \"file\": \"%s\", \"line\": %d, \"imp\": %d}\n",
-            entry->tm.tv_sec, entry->tm.tv_nsec, entry->thd, entry->type,
-            entry->tagstr, entry->file, entry->line, entry->important);
+            (long long)entry->tm.tv_sec, (int)entry->tm.tv_nsec, entry->thd,
+            entry->type, entry->tagstr, entry->file, entry->line,
+            entry->important);
   }
 }
 
diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c
index 543c75044b..a0054741ad 100644
--- a/src/core/security/credentials.c
+++ b/src/core/security/credentials.c
@@ -39,7 +39,7 @@
 #include "src/core/channel/channel_args.h"
 #include "src/core/channel/http_client_filter.h"
 #include "src/core/httpcli/httpcli.h"
-#include "src/core/iomgr/iomgr.h"
+#include "src/core/iomgr/executor.h"
 #include "src/core/json/json.h"
 #include "src/core/support/string.h"
 #include "src/core/surface/api_trace.h"
@@ -48,7 +48,6 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
-#include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 
 /* -- Common. -- */
@@ -511,10 +510,11 @@ grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
       "grpc_service_account_jwt_access_credentials_create("
       "json_key=%s, "
       "token_lifetime="
-      "gpr_timespec { tv_sec: %ld, tv_nsec: %d, clock_type: %d }, "
+      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
-      5, (json_key, (long)token_lifetime.tv_sec, token_lifetime.tv_nsec,
-          (int)token_lifetime.clock_type, reserved));
+      5,
+      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
+       (int)token_lifetime.clock_type, reserved));
   GPR_ASSERT(reserved == NULL);
   return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
       grpc_auth_json_key_create_from_string(json_key), token_lifetime);
@@ -792,15 +792,14 @@ static void md_only_test_destruct(grpc_call_credentials *creds) {
   grpc_credentials_md_store_unref(c->md_store);
 }
 
-static void on_simulated_token_fetch_done(void *user_data) {
+static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
+                                          void *user_data, int success) {
   grpc_credentials_metadata_request *r =
       (grpc_credentials_metadata_request *)user_data;
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  r->cb(&exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
+  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
         GRPC_CREDENTIALS_OK);
   grpc_credentials_metadata_request_destroy(r);
-  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void md_only_test_get_request_metadata(
@@ -810,10 +809,10 @@ static void md_only_test_get_request_metadata(
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
 
   if (c->is_async) {
-    gpr_thd_id thd_id;
     grpc_credentials_metadata_request *cb_arg =
         grpc_credentials_metadata_request_create(creds, cb, user_data);
-    gpr_thd_new(&thd_id, on_simulated_token_fetch_done, cb_arg, NULL);
+    grpc_executor_enqueue(
+        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), 1);
   } else {
     cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
   }
diff --git a/src/core/security/credentials.h b/src/core/security/credentials.h
index 6d45895e77..3cd652cd57 100644
--- a/src/core/security/credentials.h
+++ b/src/core/security/credentials.h
@@ -93,6 +93,14 @@ typedef enum {
 /* It is the caller's responsibility to gpr_free the result if not NULL. */
 char *grpc_get_well_known_google_credentials_file_path(void);
 
+/* Implementation function for the different platforms. */
+char *grpc_get_well_known_google_credentials_file_path_impl(void);
+
+/* Override for testing only. Not thread-safe */
+typedef char *(*grpc_well_known_credentials_path_getter)(void);
+void grpc_override_well_known_credentials_path_getter(
+    grpc_well_known_credentials_path_getter getter);
+
 /* --- grpc_channel_credentials. --- */
 
 typedef struct {
@@ -201,6 +209,7 @@ grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     const struct grpc_httpcli_response *response,
     grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+
 void grpc_flush_cached_google_default_credentials(void);
 
 /* Metadata-only credentials with the specified key and value where
diff --git a/src/core/security/credentials_posix.c b/src/core/security/credentials_posix.c
index 20f67a7f14..0c92bd4a96 100644
--- a/src/core/security/credentials_posix.c
+++ b/src/core/security/credentials_posix.c
@@ -44,7 +44,7 @@
 #include "src/core/support/env.h"
 #include "src/core/support/string.h"
 
-char *grpc_get_well_known_google_credentials_file_path(void) {
+char *grpc_get_well_known_google_credentials_file_path_impl(void) {
   char *result = NULL;
   char *home = gpr_getenv("HOME");
   if (home == NULL) {
diff --git a/src/core/security/credentials_win32.c b/src/core/security/credentials_win32.c
index 92dfd9bdfe..8ee9f706a1 100644
--- a/src/core/security/credentials_win32.c
+++ b/src/core/security/credentials_win32.c
@@ -44,7 +44,7 @@
 #include "src/core/support/env.h"
 #include "src/core/support/string.h"
 
-char *grpc_get_well_known_google_credentials_file_path(void) {
+char *grpc_get_well_known_google_credentials_file_path_impl(void) {
   char *result = NULL;
   char *appdata_path = gpr_getenv("APPDATA");
   if (appdata_path == NULL) {
diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c
index 6a54fe4e47..5385e41130 100644
--- a/src/core/security/google_default_credentials.c
+++ b/src/core/security/google_default_credentials.c
@@ -241,5 +241,20 @@ void grpc_flush_cached_google_default_credentials(void) {
     grpc_channel_credentials_unref(default_credentials);
     default_credentials = NULL;
   }
+  compute_engine_detection_done = 0;
   gpr_mu_unlock(&g_mu);
 }
+
+/* -- Well known credentials path. -- */
+
+static grpc_well_known_credentials_path_getter creds_path_getter = NULL;
+
+char *grpc_get_well_known_google_credentials_file_path(void) {
+  if (creds_path_getter != NULL) return creds_path_getter();
+  return grpc_get_well_known_google_credentials_file_path_impl();
+}
+
+void grpc_override_well_known_credentials_path_getter(
+    grpc_well_known_credentials_path_getter getter) {
+  creds_path_getter = getter;
+}
diff --git a/src/core/security/json_token.c b/src/core/security/json_token.c
index 021912f333..92775d885d 100644
--- a/src/core/security/json_token.c
+++ b/src/core/security/json_token.c
@@ -215,8 +215,8 @@ static char *encoded_jwt_claim(const grpc_auth_json_key *json_key,
     gpr_log(GPR_INFO, "Cropping token lifetime to maximum allowed value.");
     expiration = gpr_time_add(now, grpc_max_auth_token_lifetime);
   }
-  gpr_ltoa(now.tv_sec, now_str);
-  gpr_ltoa(expiration.tv_sec, expiration_str);
+  gpr_int64toa(now.tv_sec, now_str);
+  gpr_int64toa(expiration.tv_sec, expiration_str);
 
   child =
       create_child(NULL, json, "iss", json_key->client_email, GRPC_JSON_STRING);
diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c
index 6bda8a360c..d1468e40e0 100644
--- a/src/core/security/server_secure_chttp2.c
+++ b/src/core/security/server_secure_chttp2.c
@@ -209,7 +209,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
         tcp, (struct sockaddr *)&resolved->addrs[i].addr,
         resolved->addrs[i].len);
     port_temp = grpc_tcp_listener_get_port(listener);
-    if (port_temp >= 0) {
+    if (port_temp > 0) {
       if (port_num == -1) {
         port_num = port_temp;
       } else {
diff --git a/src/core/statistics/window_stats.c b/src/core/statistics/window_stats.c
index 4d0d3cca4a..e744006bb5 100644
--- a/src/core/statistics/window_stats.c
+++ b/src/core/statistics/window_stats.c
@@ -94,7 +94,7 @@ static gpr_int64 timespec_to_ns(const gpr_timespec ts) {
   if (ts.tv_sec > max_seconds) {
     return GPR_INT64_MAX - 1;
   }
-  return (gpr_int64)ts.tv_sec * GPR_NS_PER_SEC + ts.tv_nsec;
+  return ts.tv_sec * GPR_NS_PER_SEC + ts.tv_nsec;
 }
 
 static void cws_initialize_statistic(void *statistic,
diff --git a/src/core/support/cmdline.c b/src/core/support/cmdline.c
index 87f60bca2e..b517f30b2d 100644
--- a/src/core/support/cmdline.c
+++ b/src/core/support/cmdline.c
@@ -62,11 +62,13 @@ struct gpr_cmdline {
   void (*extra_arg)(void *user_data, const char *arg);
   void *extra_arg_user_data;
 
-  void (*state)(gpr_cmdline *cl, char *arg);
+  int (*state)(gpr_cmdline *cl, char *arg);
   arg *cur_arg;
+
+  int survive_failure;
 };
 
-static void normal_state(gpr_cmdline *cl, char *arg);
+static int normal_state(gpr_cmdline *cl, char *arg);
 
 gpr_cmdline *gpr_cmdline_create(const char *description) {
   gpr_cmdline *cl = gpr_malloc(sizeof(gpr_cmdline));
@@ -78,6 +80,10 @@ gpr_cmdline *gpr_cmdline_create(const char *description) {
   return cl;
 }
 
+void gpr_cmdline_set_survive_failure(gpr_cmdline *cl) {
+  cl->survive_failure = 1;
+}
+
 void gpr_cmdline_destroy(gpr_cmdline *cl) {
   while (cl->args) {
     arg *a = cl->args;
@@ -185,16 +191,22 @@ char *gpr_cmdline_usage_string(gpr_cmdline *cl, const char *argv0) {
   return tmp;
 }
 
-static void print_usage_and_die(gpr_cmdline *cl) {
+static int print_usage_and_die(gpr_cmdline *cl) {
   char *usage = gpr_cmdline_usage_string(cl, cl->argv0);
   fprintf(stderr, "%s", usage);
   gpr_free(usage);
-  exit(1);
+  if (!cl->survive_failure) {
+    exit(1);
+  }
+  return 0;
 }
 
-static void extra_state(gpr_cmdline *cl, char *str) {
-  if (!cl->extra_arg) print_usage_and_die(cl);
+static int extra_state(gpr_cmdline *cl, char *str) {
+  if (!cl->extra_arg) {
+    return print_usage_and_die(cl);
+  }
   cl->extra_arg(cl->extra_arg_user_data, str);
+  return 1;
 }
 
 static arg *find_arg(gpr_cmdline *cl, char *name) {
@@ -208,13 +220,13 @@ static arg *find_arg(gpr_cmdline *cl, char *name) {
 
   if (!a) {
     fprintf(stderr, "Unknown argument: %s\n", name);
-    print_usage_and_die(cl);
+    return NULL;
   }
 
   return a;
 }
 
-static void value_state(gpr_cmdline *cl, char *str) {
+static int value_state(gpr_cmdline *cl, char *str) {
   long intval;
   char *end;
 
@@ -226,7 +238,7 @@ static void value_state(gpr_cmdline *cl, char *str) {
       if (*end || intval < INT_MIN || intval > INT_MAX) {
         fprintf(stderr, "expected integer, got '%s' for %s\n", str,
                 cl->cur_arg->name);
-        print_usage_and_die(cl);
+        return print_usage_and_die(cl);
       }
       *(int *)cl->cur_arg->value = (int)intval;
       break;
@@ -238,7 +250,7 @@ static void value_state(gpr_cmdline *cl, char *str) {
       } else {
         fprintf(stderr, "expected boolean, got '%s' for %s\n", str,
                 cl->cur_arg->name);
-        print_usage_and_die(cl);
+        return print_usage_and_die(cl);
       }
       break;
     case ARGTYPE_STRING:
@@ -247,16 +259,18 @@ static void value_state(gpr_cmdline *cl, char *str) {
   }
 
   cl->state = normal_state;
+  return 1;
 }
 
-static void normal_state(gpr_cmdline *cl, char *str) {
+static int normal_state(gpr_cmdline *cl, char *str) {
   char *eq = NULL;
   char *tmp = NULL;
   char *arg_name = NULL;
+  int r = 1;
 
   if (0 == strcmp(str, "-help") || 0 == strcmp(str, "--help") ||
       0 == strcmp(str, "-h")) {
-    print_usage_and_die(cl);
+    return print_usage_and_die(cl);
   }
 
   cl->cur_arg = NULL;
@@ -266,7 +280,7 @@ static void normal_state(gpr_cmdline *cl, char *str) {
       if (str[2] == 0) {
         /* handle '--' to move to just extra args */
         cl->state = extra_state;
-        return;
+        return 1;
       }
       str += 2;
     } else {
@@ -277,12 +291,15 @@ static void normal_state(gpr_cmdline *cl, char *str) {
       /* str is of the form '--no-foo' - it's a flag disable */
       str += 3;
       cl->cur_arg = find_arg(cl, str);
+      if (cl->cur_arg == NULL) {
+        return print_usage_and_die(cl);
+      }
       if (cl->cur_arg->type != ARGTYPE_BOOL) {
         fprintf(stderr, "%s is not a flag argument\n", str);
-        print_usage_and_die(cl);
+        return print_usage_and_die(cl);
       }
       *(int *)cl->cur_arg->value = 0;
-      return; /* early out */
+      return 1; /* early out */
     }
     eq = strchr(str, '=');
     if (eq != NULL) {
@@ -294,9 +311,12 @@ static void normal_state(gpr_cmdline *cl, char *str) {
       arg_name = str;
     }
     cl->cur_arg = find_arg(cl, arg_name);
+    if (cl->cur_arg == NULL) {
+      return print_usage_and_die(cl);
+    }
     if (eq != NULL) {
       /* str was of the type --foo=value, parse the value */
-      value_state(cl, eq + 1);
+      r = value_state(cl, eq + 1);
     } else if (cl->cur_arg->type != ARGTYPE_BOOL) {
       /* flag types don't have a '--foo value' variant, other types do */
       cl->state = value_state;
@@ -305,19 +325,23 @@ static void normal_state(gpr_cmdline *cl, char *str) {
       *(int *)cl->cur_arg->value = 1;
     }
   } else {
-    extra_state(cl, str);
+    r = extra_state(cl, str);
   }
 
   gpr_free(tmp);
+  return r;
 }
 
-void gpr_cmdline_parse(gpr_cmdline *cl, int argc, char **argv) {
+int gpr_cmdline_parse(gpr_cmdline *cl, int argc, char **argv) {
   int i;
 
   GPR_ASSERT(argc >= 1);
   cl->argv0 = argv[0];
 
   for (i = 1; i < argc; i++) {
-    cl->state(cl, argv[i]);
+    if (!cl->state(cl, argv[i])) {
+      return 0;
+    }
   }
+  return 1;
 }
diff --git a/src/core/support/log_linux.c b/src/core/support/log_linux.c
index 02f64d8b7e..d66b7a3cc0 100644
--- a/src/core/support/log_linux.c
+++ b/src/core/support/log_linux.c
@@ -76,16 +76,18 @@ void gpr_default_log(gpr_log_func_args *args) {
   char *prefix;
   const char *display_file;
   char time_buffer[64];
+  time_t timer;
   gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
   struct tm tm;
 
+  timer = (time_t)now.tv_sec;
   final_slash = strrchr(args->file, '/');
   if (final_slash == NULL)
     display_file = args->file;
   else
     display_file = final_slash + 1;
 
-  if (!localtime_r(&now.tv_sec, &tm)) {
+  if (!localtime_r(&timer, &tm)) {
     strcpy(time_buffer, "error:localtime");
   } else if (0 ==
              strftime(time_buffer, sizeof(time_buffer), "%m%d %H:%M:%S", &tm)) {
diff --git a/src/core/support/log_posix.c b/src/core/support/log_posix.c
index 8b050dbee7..8986254e4e 100644
--- a/src/core/support/log_posix.c
+++ b/src/core/support/log_posix.c
@@ -75,16 +75,18 @@ void gpr_default_log(gpr_log_func_args *args) {
   char *final_slash;
   const char *display_file;
   char time_buffer[64];
+  time_t timer;
   gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
   struct tm tm;
 
+  timer = (time_t)now.tv_sec;
   final_slash = strrchr(args->file, '/');
   if (final_slash == NULL)
     display_file = args->file;
   else
     display_file = final_slash + 1;
 
-  if (!localtime_r(&now.tv_sec, &tm)) {
+  if (!localtime_r(&timer, &tm)) {
     strcpy(time_buffer, "error:localtime");
   } else if (0 ==
              strftime(time_buffer, sizeof(time_buffer), "%m%d %H:%M:%S", &tm)) {
diff --git a/src/core/support/log_win32.c b/src/core/support/log_win32.c
index b68239f8f5..40adcd1b50 100644
--- a/src/core/support/log_win32.c
+++ b/src/core/support/log_win32.c
@@ -84,16 +84,18 @@ void gpr_default_log(gpr_log_func_args *args) {
   char *final_slash;
   const char *display_file;
   char time_buffer[64];
+  time_t timer;
   gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
   struct tm tm;
 
+  timer = (time_t)now.tv_sec;
   final_slash = strrchr(args->file, '\\');
   if (final_slash == NULL)
     display_file = args->file;
   else
     display_file = final_slash + 1;
 
-  if (localtime_s(&tm, &now.tv_sec)) {
+  if (localtime_s(&tm, &timer)) {
     strcpy(time_buffer, "error:localtime");
   } else if (0 ==
              strftime(time_buffer, sizeof(time_buffer), "%m%d %H:%M:%S", &tm)) {
@@ -104,6 +106,7 @@ void gpr_default_log(gpr_log_func_args *args) {
           gpr_log_severity_string(args->severity), time_buffer,
           (int)(now.tv_nsec), GetCurrentThreadId(), display_file, args->line,
           args->message);
+  fflush(stderr);
 }
 
 char *gpr_format_message(DWORD messageid) {
diff --git a/src/core/support/stack_lockfree.c b/src/core/support/stack_lockfree.c
index df9a09894c..fc934d404c 100644
--- a/src/core/support/stack_lockfree.c
+++ b/src/core/support/stack_lockfree.c
@@ -128,8 +128,8 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) {
     gpr_atm old_val;
 
     old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index],
-                                           (gpr_atm)(1UL << pushed_bit));
-    GPR_ASSERT((old_val & (gpr_atm)(1UL << pushed_bit)) == 0);
+                                           ((gpr_atm)1 << pushed_bit));
+    GPR_ASSERT((old_val & (((gpr_atm)1) << pushed_bit)) == 0);
   }
 #endif
 
@@ -166,8 +166,8 @@ int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) {
     gpr_atm old_val;
 
     old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index],
-                                           -(gpr_atm)(1UL << pushed_bit));
-    GPR_ASSERT((old_val & (gpr_atm)(1UL << pushed_bit)) != 0);
+                                           -((gpr_atm)1 << pushed_bit));
+    GPR_ASSERT((old_val & (((gpr_atm)1) << pushed_bit)) != 0);
   }
 #endif
 
diff --git a/src/core/support/string.c b/src/core/support/string.c
index e0ffeb8a4a..46a7ca3d46 100644
--- a/src/core/support/string.c
+++ b/src/core/support/string.c
@@ -153,8 +153,8 @@ void gpr_reverse_bytes(char *str, int len) {
 }
 
 int gpr_ltoa(long value, char *string) {
+  long sign;
   int i = 0;
-  int neg = value < 0;
 
   if (value == 0) {
     string[0] = '0';
@@ -162,12 +162,33 @@ int gpr_ltoa(long value, char *string) {
     return 1;
   }
 
-  if (neg) value = -value;
+  sign = value < 0 ? -1 : 1;
   while (value) {
-    string[i++] = (char)('0' + value % 10);
+    string[i++] = (char)('0' + sign * (value % 10));
     value /= 10;
   }
-  if (neg) string[i++] = '-';
+  if (sign < 0) string[i++] = '-';
+  gpr_reverse_bytes(string, i);
+  string[i] = 0;
+  return i;
+}
+
+int gpr_int64toa(gpr_int64 value, char *string) {
+  gpr_int64 sign;
+  int i = 0;
+
+  if (value == 0) {
+    string[0] = '0';
+    string[1] = 0;
+    return 1;
+  }
+
+  sign = value < 0 ? -1 : 1;
+  while (value) {
+    string[i++] = (char)('0' + sign * (value % 10));
+    value /= 10;
+  }
+  if (sign < 0) string[i++] = '-';
   gpr_reverse_bytes(string, i);
   string[i] = 0;
   return i;
diff --git a/src/core/support/string.h b/src/core/support/string.h
index a28e00fd3e..9b604ac5bf 100644
--- a/src/core/support/string.h
+++ b/src/core/support/string.h
@@ -70,6 +70,16 @@ int gpr_parse_bytes_to_uint32(const char *data, size_t length,
    output must be at least GPR_LTOA_MIN_BUFSIZE bytes long. */
 int gpr_ltoa(long value, char *output);
 
+/* Minimum buffer size for calling int64toa */
+#define GPR_INT64TOA_MIN_BUFSIZE (3 * sizeof(gpr_int64))
+
+/* Convert an int64 to a string in base 10; returns the length of the
+output string (or 0 on failure).
+output must be at least GPR_INT64TOA_MIN_BUFSIZE bytes long.
+NOTE: This function ensures sufficient bit width even on Win x64,
+where long is 32bit is size.*/
+int gpr_int64toa(gpr_int64 value, char *output);
+
 /* Reverse a run of bytes */
 void gpr_reverse_bytes(char *str, int len);
 
diff --git a/src/core/support/time.c b/src/core/support/time.c
index 929adac918..197fa9ad44 100644
--- a/src/core/support/time.c
+++ b/src/core/support/time.c
@@ -56,22 +56,6 @@ gpr_timespec gpr_time_max(gpr_timespec a, gpr_timespec b) {
   return gpr_time_cmp(a, b) > 0 ? a : b;
 }
 
-/* There's no standard TIME_T_MIN and TIME_T_MAX, so we construct them.  The
-   following assumes that signed types are two's-complement and that bytes are
-   8 bits.  */
-
-/* The top bit of integral type t. */
-#define TOP_BIT_OF_TYPE(t) (((gpr_uintmax)1) << ((8 * sizeof(t)) - 1))
-
-/* Return whether integral type t is signed. */
-#define TYPE_IS_SIGNED(t) (((t)1) > (t) ~(t)0)
-
-/* The minimum and maximum value of integral type t. */
-#define TYPE_MIN(t) ((t)(TYPE_IS_SIGNED(t) ? TOP_BIT_OF_TYPE(t) : 0))
-#define TYPE_MAX(t)                                 \
-  ((t)(TYPE_IS_SIGNED(t) ? (TOP_BIT_OF_TYPE(t) - 1) \
-                         : ((TOP_BIT_OF_TYPE(t) - 1) << 1) + 1))
-
 gpr_timespec gpr_time_0(gpr_clock_type type) {
   gpr_timespec out;
   out.tv_sec = 0;
@@ -82,7 +66,7 @@ gpr_timespec gpr_time_0(gpr_clock_type type) {
 
 gpr_timespec gpr_inf_future(gpr_clock_type type) {
   gpr_timespec out;
-  out.tv_sec = TYPE_MAX(time_t);
+  out.tv_sec = INT64_MAX;
   out.tv_nsec = 0;
   out.clock_type = type;
   return out;
@@ -90,7 +74,7 @@ gpr_timespec gpr_inf_future(gpr_clock_type type) {
 
 gpr_timespec gpr_inf_past(gpr_clock_type type) {
   gpr_timespec out;
-  out.tv_sec = TYPE_MIN(time_t);
+  out.tv_sec = INT64_MIN;
   out.tv_nsec = 0;
   out.clock_type = type;
   return out;
@@ -108,11 +92,11 @@ gpr_timespec gpr_time_from_nanos(long ns, gpr_clock_type type) {
     result = gpr_inf_past(type);
   } else if (ns >= 0) {
     result.tv_sec = ns / GPR_NS_PER_SEC;
-    result.tv_nsec = (int)(ns - result.tv_sec * GPR_NS_PER_SEC);
+    result.tv_nsec = (gpr_int32)(ns - result.tv_sec * GPR_NS_PER_SEC);
   } else {
     /* Calculation carefully formulated to avoid any possible under/overflow. */
     result.tv_sec = (-(999999999 - (ns + GPR_NS_PER_SEC)) / GPR_NS_PER_SEC) - 1;
-    result.tv_nsec = (int)(ns - result.tv_sec * GPR_NS_PER_SEC);
+    result.tv_nsec = (gpr_int32)(ns - result.tv_sec * GPR_NS_PER_SEC);
   }
   return result;
 }
@@ -126,11 +110,11 @@ gpr_timespec gpr_time_from_micros(long us, gpr_clock_type type) {
     result = gpr_inf_past(type);
   } else if (us >= 0) {
     result.tv_sec = us / 1000000;
-    result.tv_nsec = (int)((us - result.tv_sec * 1000000) * 1000);
+    result.tv_nsec = (gpr_int32)((us - result.tv_sec * 1000000) * 1000);
   } else {
     /* Calculation carefully formulated to avoid any possible under/overflow. */
     result.tv_sec = (-(999999 - (us + 1000000)) / 1000000) - 1;
-    result.tv_nsec = (int)((us - result.tv_sec * 1000000) * 1000);
+    result.tv_nsec = (gpr_int32)((us - result.tv_sec * 1000000) * 1000);
   }
   return result;
 }
@@ -144,11 +128,11 @@ gpr_timespec gpr_time_from_millis(long ms, gpr_clock_type type) {
     result = gpr_inf_past(type);
   } else if (ms >= 0) {
     result.tv_sec = ms / 1000;
-    result.tv_nsec = (int)((ms - result.tv_sec * 1000) * 1000000);
+    result.tv_nsec = (gpr_int32)((ms - result.tv_sec * 1000) * 1000000);
   } else {
     /* Calculation carefully formulated to avoid any possible under/overflow. */
     result.tv_sec = (-(999 - (ms + 1000)) / 1000) - 1;
-    result.tv_nsec = (int)((ms - result.tv_sec * 1000) * 1000000);
+    result.tv_nsec = (gpr_int32)((ms - result.tv_sec * 1000) * 1000000);
   }
   return result;
 }
@@ -197,7 +181,7 @@ gpr_timespec gpr_time_from_hours(long h, gpr_clock_type type) {
 
 gpr_timespec gpr_time_add(gpr_timespec a, gpr_timespec b) {
   gpr_timespec sum;
-  int inc = 0;
+  gpr_int64 inc = 0;
   GPR_ASSERT(b.clock_type == GPR_TIMESPAN);
   sum.clock_type = a.clock_type;
   sum.tv_nsec = a.tv_nsec + b.tv_nsec;
@@ -205,17 +189,17 @@ gpr_timespec gpr_time_add(gpr_timespec a, gpr_timespec b) {
     sum.tv_nsec -= GPR_NS_PER_SEC;
     inc++;
   }
-  if (a.tv_sec == TYPE_MAX(time_t) || a.tv_sec == TYPE_MIN(time_t)) {
+  if (a.tv_sec == INT64_MAX || a.tv_sec == INT64_MIN) {
     sum = a;
-  } else if (b.tv_sec == TYPE_MAX(time_t) ||
-             (b.tv_sec >= 0 && a.tv_sec >= TYPE_MAX(time_t) - b.tv_sec)) {
+  } else if (b.tv_sec == INT64_MAX ||
+             (b.tv_sec >= 0 && a.tv_sec >= INT64_MAX - b.tv_sec)) {
     sum = gpr_inf_future(sum.clock_type);
-  } else if (b.tv_sec == TYPE_MIN(time_t) ||
-             (b.tv_sec <= 0 && a.tv_sec <= TYPE_MIN(time_t) - b.tv_sec)) {
+  } else if (b.tv_sec == INT64_MIN ||
+             (b.tv_sec <= 0 && a.tv_sec <= INT64_MIN - b.tv_sec)) {
     sum = gpr_inf_past(sum.clock_type);
   } else {
     sum.tv_sec = a.tv_sec + b.tv_sec;
-    if (inc != 0 && sum.tv_sec == TYPE_MAX(time_t) - 1) {
+    if (inc != 0 && sum.tv_sec == INT64_MAX - 1) {
       sum = gpr_inf_future(sum.clock_type);
     } else {
       sum.tv_sec += inc;
@@ -226,7 +210,7 @@ gpr_timespec gpr_time_add(gpr_timespec a, gpr_timespec b) {
 
 gpr_timespec gpr_time_sub(gpr_timespec a, gpr_timespec b) {
   gpr_timespec diff;
-  int dec = 0;
+  gpr_int64 dec = 0;
   if (b.clock_type == GPR_TIMESPAN) {
     diff.clock_type = a.clock_type;
   } else {
@@ -238,17 +222,17 @@ gpr_timespec gpr_time_sub(gpr_timespec a, gpr_timespec b) {
     diff.tv_nsec += GPR_NS_PER_SEC;
     dec++;
   }
-  if (a.tv_sec == TYPE_MAX(time_t) || a.tv_sec == TYPE_MIN(time_t)) {
+  if (a.tv_sec == INT64_MAX || a.tv_sec == INT64_MIN) {
     diff = a;
-  } else if (b.tv_sec == TYPE_MIN(time_t) ||
-             (b.tv_sec <= 0 && a.tv_sec >= TYPE_MAX(time_t) + b.tv_sec)) {
+  } else if (b.tv_sec == INT64_MIN ||
+             (b.tv_sec <= 0 && a.tv_sec >= INT64_MAX + b.tv_sec)) {
     diff = gpr_inf_future(GPR_CLOCK_REALTIME);
-  } else if (b.tv_sec == TYPE_MAX(time_t) ||
-             (b.tv_sec >= 0 && a.tv_sec <= TYPE_MIN(time_t) + b.tv_sec)) {
+  } else if (b.tv_sec == INT64_MAX ||
+             (b.tv_sec >= 0 && a.tv_sec <= INT64_MIN + b.tv_sec)) {
     diff = gpr_inf_past(GPR_CLOCK_REALTIME);
   } else {
     diff.tv_sec = a.tv_sec - b.tv_sec;
-    if (dec != 0 && diff.tv_sec == TYPE_MIN(time_t) + 1) {
+    if (dec != 0 && diff.tv_sec == INT64_MIN + 1) {
       diff = gpr_inf_past(GPR_CLOCK_REALTIME);
     } else {
       diff.tv_sec -= dec;
@@ -297,11 +281,11 @@ gpr_timespec gpr_convert_clock_type(gpr_timespec t, gpr_clock_type clock_type) {
   }
 
   if (t.tv_nsec == 0) {
-    if (t.tv_sec == TYPE_MAX(time_t)) {
+    if (t.tv_sec == INT64_MAX) {
       t.clock_type = clock_type;
       return t;
     }
-    if (t.tv_sec == TYPE_MIN(time_t)) {
+    if (t.tv_sec == INT64_MIN) {
       t.clock_type = clock_type;
       return t;
     }
diff --git a/src/core/support/time_posix.c b/src/core/support/time_posix.c
index 02cfca8555..ba72572e05 100644
--- a/src/core/support/time_posix.c
+++ b/src/core/support/time_posix.c
@@ -45,7 +45,11 @@
 
 static struct timespec timespec_from_gpr(gpr_timespec gts) {
   struct timespec rv;
-  rv.tv_sec = gts.tv_sec;
+  if (sizeof(time_t) < sizeof(gpr_int64)) {
+    /* fine to assert, as this is only used in gpr_sleep_until */
+    GPR_ASSERT(gts.tv_sec <= INT32_MAX && gts.tv_sec >= INT32_MIN);
+  }
+  rv.tv_sec = (time_t)gts.tv_sec;
   rv.tv_nsec = gts.tv_nsec;
   return rv;
 }
@@ -53,9 +57,14 @@ static struct timespec timespec_from_gpr(gpr_timespec gts) {
 #if _POSIX_TIMERS > 0
 static gpr_timespec gpr_from_timespec(struct timespec ts,
                                       gpr_clock_type clock_type) {
+  /*
+   * timespec.tv_sec can have smaller size than gpr_timespec.tv_sec,
+   * but we are only using this function to implement gpr_now
+   * so there's no need to handle "infinity" values.
+   */
   gpr_timespec rv;
   rv.tv_sec = ts.tv_sec;
-  rv.tv_nsec = (int)ts.tv_nsec;
+  rv.tv_nsec = (gpr_int32)ts.tv_nsec;
   rv.clock_type = clock_type;
   return rv;
 }
@@ -110,8 +119,8 @@ gpr_timespec gpr_now(gpr_clock_type clock) {
       break;
     case GPR_CLOCK_MONOTONIC:
       now_dbl = (mach_absolute_time() - g_time_start) * g_time_scale;
-      now.tv_sec = (time_t)(now_dbl * 1e-9);
-      now.tv_nsec = (int)(now_dbl - ((double)now.tv_sec) * 1e9);
+      now.tv_sec = (gpr_int64)(now_dbl * 1e-9);
+      now.tv_nsec = (gpr_int32)(now_dbl - ((double)now.tv_sec) * 1e9);
       break;
     case GPR_CLOCK_PRECISE:
       gpr_precise_clock_now(&now);
diff --git a/src/core/support/time_precise.c b/src/core/support/time_precise.c
index b37517e639..4de1d9b071 100644
--- a/src/core/support/time_precise.c
+++ b/src/core/support/time_precise.c
@@ -75,8 +75,8 @@ void gpr_precise_clock_now(gpr_timespec *clk) {
   gpr_get_cycle_counter(&counter);
   secs = (double)(counter - start_cycle) / cycles_per_second;
   clk->clock_type = GPR_CLOCK_PRECISE;
-  clk->tv_sec = (time_t)secs;
-  clk->tv_nsec = (int)(1e9 * (secs - (double)clk->tv_sec));
+  clk->tv_sec = (gpr_int64)secs;
+  clk->tv_nsec = (gpr_int32)(1e9 * (secs - (double)clk->tv_sec));
 }
 
 #else  /* GRPC_TIMERS_RDTSC */
diff --git a/src/core/support/time_win32.c b/src/core/support/time_win32.c
index 623a8d9233..7ccaaa248d 100644
--- a/src/core/support/time_win32.c
+++ b/src/core/support/time_win32.c
@@ -62,15 +62,15 @@ gpr_timespec gpr_now(gpr_clock_type clock) {
   switch (clock) {
     case GPR_CLOCK_REALTIME:
       _ftime_s(&now_tb);
-      now_tv.tv_sec = now_tb.time;
+      now_tv.tv_sec = (gpr_int64)now_tb.time;
       now_tv.tv_nsec = now_tb.millitm * 1000000;
       break;
     case GPR_CLOCK_MONOTONIC:
     case GPR_CLOCK_PRECISE:
       QueryPerformanceCounter(&timestamp);
       now_dbl = (timestamp.QuadPart - g_start_time.QuadPart) * g_time_scale;
-      now_tv.tv_sec = (time_t)now_dbl;
-      now_tv.tv_nsec = (int)((now_dbl - (double)now_tv.tv_sec) * 1e9);
+      now_tv.tv_sec = (gpr_int64)now_dbl;
+      now_tv.tv_nsec = (gpr_int32)((now_dbl - (double)now_tv.tv_sec) * 1e9);
       break;
   }
   return now_tv;
diff --git a/src/core/surface/call.c b/src/core/surface/call.c
index 79295ae0ff..b999ecd37c 100644
--- a/src/core/surface/call.c
+++ b/src/core/surface/call.c
@@ -336,26 +336,19 @@ void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call,
                               grpc_cq_pollset(cq));
 }
 
-grpc_completion_queue *grpc_call_get_completion_queue(grpc_call *call) {
-  return call->cq;
-}
-
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
-void grpc_call_internal_ref(grpc_call *c, const char *reason) {
-  grpc_call_stack_ref(CALL_STACK_FROM_CALL(c), reason);
-}
-void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c,
-                              const char *reason) {
-  grpc_call_stack_unref(exec_ctx, CALL_STACK_FROM_CALL(c), reason);
-}
+#define REF_REASON reason
+#define REF_ARG , const char *reason
 #else
-void grpc_call_internal_ref(grpc_call *c) {
-  grpc_call_stack_ref(CALL_STACK_FROM_CALL(c));
+#define REF_REASON ""
+#define REF_ARG
+#endif
+void grpc_call_internal_ref(grpc_call *c REF_ARG) {
+  GRPC_CALL_STACK_REF(CALL_STACK_FROM_CALL(c), REF_REASON);
 }
-void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c) {
-  grpc_call_stack_unref(exec_ctx, CALL_STACK_FROM_CALL(c));
+void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) {
+  GRPC_CALL_STACK_UNREF(exec_ctx, CALL_STACK_FROM_CALL(c), REF_REASON);
 }
-#endif
 
 static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, int success) {
   size_t i;
@@ -742,8 +735,15 @@ static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
 char *grpc_call_get_peer(grpc_call *call) {
   grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  char *result = elem->filter->get_peer(&exec_ctx, elem);
+  char *result;
   GRPC_API_TRACE("grpc_call_get_peer(%p)", 1, (call));
+  result = elem->filter->get_peer(&exec_ctx, elem);
+  if (result == NULL) {
+    result = grpc_channel_get_target(call->channel);
+  }
+  if (result == NULL) {
+    result = gpr_strdup("unknown");
+  }
   grpc_exec_ctx_finish(&exec_ctx);
   return result;
 }
@@ -974,11 +974,19 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
 
-  GPR_ASSERT(success);
-  gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer,
-                       call->receiving_slice);
-
-  continue_receiving_slices(exec_ctx, bctl);
+  if (success) {
+    gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer,
+                         call->receiving_slice);
+    continue_receiving_slices(exec_ctx, bctl);
+  } else {
+    grpc_byte_stream_destroy(call->receiving_stream);
+    call->receiving_stream = NULL;
+    grpc_byte_buffer_destroy(*call->receiving_buffer);
+    *call->receiving_buffer = NULL;
+    if (gpr_unref(&bctl->steps_to_complete)) {
+      post_batch_completion(exec_ctx, bctl);
+    }
+  }
 }
 
 static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, int success) {
@@ -1060,6 +1068,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
 
   if (call->receiving_stream == NULL) {
     *call->receiving_buffer = NULL;
+    call->receiving_message = 0; 
     if (gpr_unref(&bctl->steps_to_complete)) {
       post_batch_completion(exec_ctx, bctl);
     }
@@ -1070,6 +1079,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
     grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
     call->receiving_stream = NULL;
     *call->receiving_buffer = NULL;
+    call->receiving_message = 0; 
     if (gpr_unref(&bctl->steps_to_complete)) {
       post_batch_completion(exec_ctx, bctl);
     }
@@ -1119,11 +1129,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
     GRPC_CALL_INTERNAL_REF(call, "completion");
     bctl->success = 1;
     if (!is_notify_tag_closure) {
-      grpc_cq_begin_op(call->cq);
+      grpc_cq_begin_op(call->cq, notify_tag);
     }
     gpr_mu_unlock(&call->mu);
     post_batch_completion(exec_ctx, bctl);
-    return GRPC_CALL_OK;
+    error = GRPC_CALL_OK;
+    goto done;
   }
 
   /* rewrite batch ops into a transport op */
@@ -1333,7 +1344,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
 
   GRPC_CALL_INTERNAL_REF(call, "completion");
   if (!is_notify_tag_closure) {
-    grpc_cq_begin_op(call->cq);
+    grpc_cq_begin_op(call->cq, notify_tag);
   }
   gpr_ref_init(&bctl->steps_to_complete, num_completion_callbacks_needed);
 
diff --git a/src/core/surface/call.h b/src/core/surface/call.h
index 20907ac6d6..b53340df8e 100644
--- a/src/core/surface/call.h
+++ b/src/core/surface/call.h
@@ -58,7 +58,6 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
 
 void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call,
                                     grpc_completion_queue *cq);
-grpc_completion_queue *grpc_call_get_completion_queue(grpc_call *call);
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 void grpc_call_internal_ref(grpc_call *call, const char *reason);
diff --git a/src/core/surface/call_log_batch.c b/src/core/surface/call_log_batch.c
index f4cd8375c2..46756f418b 100644
--- a/src/core/surface/call_log_batch.c
+++ b/src/core/surface/call_log_batch.c
@@ -116,4 +116,3 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity,
     gpr_free(tmp);
   }
 }
-
diff --git a/src/core/surface/channel.c b/src/core/surface/channel.c
index 14fe97c30d..d0a8b0be09 100644
--- a/src/core/surface/channel.c
+++ b/src/core/surface/channel.c
@@ -63,7 +63,6 @@ typedef struct registered_call {
 
 struct grpc_channel {
   int is_client;
-  gpr_refcount refs;
   gpr_uint32 max_message_length;
   grpc_mdelem *default_authority;
 
@@ -81,6 +80,8 @@ struct grpc_channel {
 /* the protobuf library will (by default) start warning at 100megs */
 #define DEFAULT_MAX_MESSAGE_LENGTH (100 * 1024 * 1024)
 
+static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg, int success);
+
 grpc_channel *grpc_channel_create_from_filters(
     grpc_exec_ctx *exec_ctx, const char *target,
     const grpc_channel_filter **filters, size_t num_filters,
@@ -93,8 +94,6 @@ grpc_channel *grpc_channel_create_from_filters(
   channel->target = gpr_strdup(target);
   GPR_ASSERT(grpc_is_initialized() && "call grpc_init()");
   channel->is_client = is_client;
-  /* decremented by grpc_channel_destroy */
-  gpr_ref_init(&channel->refs, 1);
   gpr_mu_init(&channel->registered_call_mu);
   channel->registered_calls = NULL;
 
@@ -153,7 +152,9 @@ grpc_channel *grpc_channel_create_from_filters(
     gpr_free(default_authority);
   }
 
-  grpc_channel_stack_init(exec_ctx, filters, num_filters, channel, args,
+  grpc_channel_stack_init(exec_ctx, 1, destroy_channel, channel, filters,
+                          num_filters, args,
+                          is_client ? "CLIENT_CHANNEL" : "SERVER_CHANNEL",
                           CHANNEL_STACK_FROM_CHANNEL(channel));
 
   return channel;
@@ -194,11 +195,11 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel,
       "grpc_channel_create_call("
       "channel=%p, parent_call=%p, propagation_mask=%x, cq=%p, method=%s, "
       "host=%s, "
-      "deadline=gpr_timespec { tv_sec: %ld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
       10, (channel, parent_call, (unsigned)propagation_mask, cq, method, host,
-           (long)deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
-           reserved));
+           (long long)deadline.tv_sec, (int)deadline.tv_nsec,
+           (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, cq,
@@ -238,11 +239,11 @@ grpc_call *grpc_channel_create_registered_call(
       "grpc_channel_create_registered_call("
       "channel=%p, parent_call=%p, propagation_mask=%x, completion_queue=%p, "
       "registered_call_handle=%p, "
-      "deadline=gpr_timespec { tv_sec: %ld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
       9, (channel, parent_call, (unsigned)propagation_mask, completion_queue,
-          registered_call_handle, (long)deadline.tv_sec, deadline.tv_nsec,
-          (int)deadline.clock_type, reserved));
+          registered_call_handle, (long long)deadline.tv_sec,
+          (int)deadline.tv_nsec, (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, completion_queue,
@@ -250,17 +251,25 @@ grpc_call *grpc_channel_create_registered_call(
       rc->authority ? GRPC_MDELEM_REF(rc->authority) : NULL, deadline);
 }
 
-#ifdef GRPC_CHANNEL_REF_COUNT_DEBUG
-void grpc_channel_internal_ref(grpc_channel *c, const char *reason) {
-  gpr_log(GPR_DEBUG, "CHANNEL:   ref %p %d -> %d [%s]", c, c->refs.count,
-          c->refs.count + 1, reason);
+#ifdef GRPC_STREAM_REFCOUNT_DEBUG
+#define REF_REASON reason
+#define REF_ARG , const char *reason
 #else
-void grpc_channel_internal_ref(grpc_channel *c) {
+#define REF_REASON ""
+#define REF_ARG
 #endif
-  gpr_ref(&c->refs);
+void grpc_channel_internal_ref(grpc_channel *c REF_ARG) {
+  GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CHANNEL(c), REF_REASON);
 }
 
-static void destroy_channel(grpc_exec_ctx *exec_ctx, grpc_channel *channel) {
+void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel *c REF_ARG) {
+  GRPC_CHANNEL_STACK_UNREF(exec_ctx, CHANNEL_STACK_FROM_CHANNEL(c), REF_REASON);
+}
+
+static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg,
+                            int iomgr_success) {
+  grpc_channel *channel = arg;
   grpc_channel_stack_destroy(exec_ctx, CHANNEL_STACK_FROM_CHANNEL(channel));
   while (channel->registered_calls) {
     registered_call *rc = channel->registered_calls;
@@ -279,20 +288,6 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, grpc_channel *channel) {
   gpr_free(channel);
 }
 
-#ifdef GRPC_CHANNEL_REF_COUNT_DEBUG
-void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
-                                 const char *reason) {
-  gpr_log(GPR_DEBUG, "CHANNEL: unref %p %d -> %d [%s]", channel,
-          channel->refs.count, channel->refs.count - 1, reason);
-#else
-void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
-                                 grpc_channel *channel) {
-#endif
-  if (gpr_unref(&channel->refs)) {
-    destroy_channel(exec_ctx, channel);
-  }
-}
-
 void grpc_channel_destroy(grpc_channel *channel) {
   grpc_transport_op op;
   grpc_channel_element *elem;
diff --git a/src/core/surface/channel.h b/src/core/surface/channel.h
index 7dea609ebc..3d2ff23542 100644
--- a/src/core/surface/channel.h
+++ b/src/core/surface/channel.h
@@ -53,7 +53,7 @@ grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel,
                                                  int status_code);
 gpr_uint32 grpc_channel_get_max_message_length(grpc_channel *channel);
 
-#ifdef GRPC_CHANNEL_REF_COUNT_DEBUG
+#ifdef GRPC_STREAM_REFCOUNT_DEBUG
 void grpc_channel_internal_ref(grpc_channel *channel, const char *reason);
 void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
                                  const char *reason);
diff --git a/src/core/surface/channel_connectivity.c b/src/core/surface/channel_connectivity.c
index df2774b527..10f5c4da4d 100644
--- a/src/core/surface/channel_connectivity.c
+++ b/src/core/surface/channel_connectivity.c
@@ -83,7 +83,6 @@ typedef struct {
   gpr_mu mu;
   callback_phase phase;
   int success;
-  int removed;
   grpc_closure on_complete;
   grpc_timer alarm;
   grpc_connectivity_state state;
@@ -135,30 +134,15 @@ static void finished_completion(grpc_exec_ctx *exec_ctx, void *pw,
 static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w,
                         int due_to_completion) {
   int delete = 0;
-  grpc_channel_element *client_channel_elem = NULL;
 
-  gpr_mu_lock(&w->mu);
-  if (w->removed == 0) {
-    w->removed = 1;
-    client_channel_elem = grpc_channel_stack_last_element(
-        grpc_channel_get_channel_stack(w->channel));
-    if (client_channel_elem->filter == &grpc_client_channel_filter) {
-      grpc_client_channel_del_interested_party(exec_ctx, client_channel_elem,
-                                               grpc_cq_pollset(w->cq));
-    } else {
-      grpc_client_uchannel_del_interested_party(exec_ctx, client_channel_elem,
-                                                grpc_cq_pollset(w->cq));
-    }
-  }
-  gpr_mu_unlock(&w->mu);
   if (due_to_completion) {
-    gpr_mu_lock(&w->mu);
-    w->success = 1;
-    gpr_mu_unlock(&w->mu);
     grpc_timer_cancel(exec_ctx, &w->alarm);
   }
 
   gpr_mu_lock(&w->mu);
+  if (due_to_completion) {
+    w->success = 1;
+  }
   switch (w->phase) {
     case WAITING:
       w->phase = CALLING_BACK;
@@ -200,19 +184,18 @@ void grpc_channel_watch_connectivity_state(
   GRPC_API_TRACE(
       "grpc_channel_watch_connectivity_state("
       "channel=%p, last_observed_state=%d, "
-      "deadline=gpr_timespec { tv_sec: %ld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
       "cq=%p, tag=%p)",
-      7, (channel, (int)last_observed_state, (long)deadline.tv_sec,
-          deadline.tv_nsec, (int)deadline.clock_type, cq, tag));
+      7, (channel, (int)last_observed_state, (long long)deadline.tv_sec,
+          (int)deadline.tv_nsec, (int)deadline.clock_type, cq, tag));
 
-  grpc_cq_begin_op(cq);
+  grpc_cq_begin_op(cq, tag);
 
   gpr_mu_init(&w->mu);
   grpc_closure_init(&w->on_complete, watch_complete, w);
   w->phase = WAITING;
   w->state = last_observed_state;
   w->success = 0;
-  w->removed = 0;
   w->cq = cq;
   w->tag = tag;
   w->channel = channel;
@@ -223,16 +206,14 @@ void grpc_channel_watch_connectivity_state(
 
   if (client_channel_elem->filter == &grpc_client_channel_filter) {
     GRPC_CHANNEL_INTERNAL_REF(channel, "watch_channel_connectivity");
-    grpc_client_channel_add_interested_party(&exec_ctx, client_channel_elem,
-                                             grpc_cq_pollset(cq));
     grpc_client_channel_watch_connectivity_state(&exec_ctx, client_channel_elem,
-                                                 &w->state, &w->on_complete);
+                                                 grpc_cq_pollset(cq), &w->state,
+                                                 &w->on_complete);
   } else if (client_channel_elem->filter == &grpc_client_uchannel_filter) {
     GRPC_CHANNEL_INTERNAL_REF(channel, "watch_uchannel_connectivity");
-    grpc_client_uchannel_add_interested_party(&exec_ctx, client_channel_elem,
-                                              grpc_cq_pollset(cq));
     grpc_client_uchannel_watch_connectivity_state(
-        &exec_ctx, client_channel_elem, &w->state, &w->on_complete);
+        &exec_ctx, client_channel_elem, grpc_cq_pollset(cq), &w->state,
+        &w->on_complete);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c
index fe7e1072ac..97ec23408f 100644
--- a/src/core/surface/channel_create.c
+++ b/src/core/surface/channel_create.c
@@ -171,7 +171,6 @@ static grpc_subchannel *subchannel_factory_create_subchannel(
   c->base.vtable = &connector_vtable;
   gpr_ref_init(&c->refs, 1);
   args->args = final_args;
-  args->master = f->master;
   s = grpc_subchannel_create(&c->base, args);
   grpc_connector_unref(exec_ctx, &c->base);
   grpc_channel_args_destroy(final_args);
diff --git a/src/core/surface/channel_ping.c b/src/core/surface/channel_ping.c
new file mode 100644
index 0000000000..b4ce282787
--- /dev/null
+++ b/src/core/surface/channel_ping.c
@@ -0,0 +1,79 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/surface/channel.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/surface/api_trace.h"
+#include "src/core/surface/completion_queue.h"
+
+typedef struct {
+  grpc_closure closure;
+  void *tag;
+  grpc_completion_queue *cq;
+  grpc_cq_completion completion_storage;
+} ping_result;
+
+static void ping_destroy(grpc_exec_ctx *exec_ctx, void *arg,
+                         grpc_cq_completion *storage) {
+  gpr_free(arg);
+}
+
+static void ping_done(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+  ping_result *pr = arg;
+  grpc_cq_end_op(exec_ctx, pr->cq, pr->tag, success, ping_destroy, pr,
+                 &pr->completion_storage);
+}
+
+void grpc_channel_ping(grpc_channel *channel, grpc_completion_queue *cq,
+                       void *tag, void *reserved) {
+  grpc_transport_op op;
+  ping_result *pr = gpr_malloc(sizeof(*pr));
+  grpc_channel_element *top_elem =
+      grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GPR_ASSERT(reserved == NULL);
+  memset(&op, 0, sizeof(op));
+  pr->tag = tag;
+  pr->cq = cq;
+  grpc_closure_init(&pr->closure, ping_done, pr);
+  op.send_ping = &pr->closure;
+  op.bind_pollset = grpc_cq_pollset(cq);
+  grpc_cq_begin_op(cq, tag);
+  top_elem->filter->start_transport_op(&exec_ctx, top_elem, &op);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c
index d56e5cbe84..848a33adc3 100644
--- a/src/core/surface/completion_queue.c
+++ b/src/core/surface/completion_queue.c
@@ -73,6 +73,12 @@ struct grpc_completion_queue {
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
   grpc_closure pollset_shutdown_done;
 
+#ifndef NDEBUG
+  void **outstanding_tags;
+  size_t outstanding_tag_count;
+  size_t outstanding_tag_capacity;
+#endif
+
   grpc_completion_queue *next_free;
 };
 
@@ -89,6 +95,9 @@ void grpc_cq_global_shutdown(void) {
   while (g_freelist) {
     grpc_completion_queue *next = g_freelist->next_free;
     grpc_pollset_destroy(&g_freelist->pollset);
+#ifndef NDEBUG
+    gpr_free(g_freelist->outstanding_tags);
+#endif
     gpr_free(g_freelist);
     g_freelist = next;
   }
@@ -117,6 +126,10 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
 
     cc = gpr_malloc(sizeof(grpc_completion_queue));
     grpc_pollset_init(&cc->pollset);
+#ifndef NDEBUG
+    cc->outstanding_tags = NULL;
+    cc->outstanding_tag_capacity = 0;
+#endif
   } else {
     cc = g_freelist;
     g_freelist = g_freelist->next_free;
@@ -134,6 +147,9 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
   cc->shutdown_called = 0;
   cc->is_server_cq = 0;
   cc->num_pluckers = 0;
+#ifndef NDEBUG
+  cc->outstanding_tag_count = 0;
+#endif
   grpc_closure_init(&cc->pollset_shutdown_done, on_pollset_shutdown_done, cc);
 
   GPR_TIMER_END("grpc_completion_queue_create", 0);
@@ -176,10 +192,17 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) {
   }
 }
 
-void grpc_cq_begin_op(grpc_completion_queue *cc) {
+void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) {
 #ifndef NDEBUG
   gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
   GPR_ASSERT(!cc->shutdown_called);
+  if (cc->outstanding_tag_count == cc->outstanding_tag_capacity) {
+    cc->outstanding_tag_capacity = GPR_MAX(4, 2 * cc->outstanding_tag_capacity);
+    cc->outstanding_tags =
+        gpr_realloc(cc->outstanding_tags, sizeof(*cc->outstanding_tags) *
+                                              cc->outstanding_tag_capacity);
+  }
+  cc->outstanding_tags[cc->outstanding_tag_count++] = tag;
   gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
 #endif
   gpr_ref(&cc->pending_events);
@@ -196,6 +219,9 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
   int shutdown;
   int i;
   grpc_pollset_worker *pluck_worker;
+#ifndef NDEBUG
+  int found = 0;
+#endif
 
   GPR_TIMER_BEGIN("grpc_cq_end_op", 0);
 
@@ -206,6 +232,18 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
       ((gpr_uintptr)&cc->completed_head) | ((gpr_uintptr)(success != 0));
 
   gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
+#ifndef NDEBUG
+  for (i = 0; i < (int)cc->outstanding_tag_count; i++) {
+    if (cc->outstanding_tags[i] == tag) {
+      cc->outstanding_tag_count--;
+      GPR_SWAP(void *, cc->outstanding_tags[i],
+               cc->outstanding_tags[cc->outstanding_tag_count]);
+      found = 1;
+      break;
+    }
+  }
+  GPR_ASSERT(found);
+#endif
   shutdown = gpr_unref(&cc->pending_events);
   if (!shutdown) {
     cc->completed_tail->next =
@@ -247,10 +285,10 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
   GRPC_API_TRACE(
       "grpc_completion_queue_next("
       "cc=%p, "
-      "deadline=gpr_timespec { tv_sec: %ld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
-      5, (cc, (long)deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
-          reserved));
+      5, (cc, (long long)deadline.tv_sec, (int)deadline.tv_nsec,
+          (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
@@ -335,9 +373,9 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
   GRPC_API_TRACE(
       "grpc_completion_queue_pluck("
       "cc=%p, tag=%p, "
-      "deadline=gpr_timespec { tv_sec: %ld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
-      6, (cc, tag, (long)deadline.tv_sec, deadline.tv_nsec,
+      6, (cc, tag, (long long)deadline.tv_sec, (int)deadline.tv_nsec,
           (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
 
diff --git a/src/core/surface/completion_queue.h b/src/core/surface/completion_queue.h
index a40bb048ac..1e40c48bea 100644
--- a/src/core/surface/completion_queue.h
+++ b/src/core/surface/completion_queue.h
@@ -68,10 +68,12 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc);
 #endif
 
 /* Flag that an operation is beginning: the completion channel will not finish
-   shutdown until a corrensponding grpc_cq_end_* call is made */
-void grpc_cq_begin_op(grpc_completion_queue *cc);
+   shutdown until a corrensponding grpc_cq_end_* call is made.
+   \a tag is currently used only in debug builds. */
+void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag);
 
-/* Queue a GRPC_OP_COMPLETED operation */
+/* Queue a GRPC_OP_COMPLETED operation; tag must correspond to the tag passed to
+   grpc_cq_begin_op */
 void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
                     void *tag, int success,
                     void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
diff --git a/src/core/surface/lame_client.c b/src/core/surface/lame_client.c
index 4a55544ac1..a60e9d20da 100644
--- a/src/core/surface/lame_client.c
+++ b/src/core/surface/lame_client.c
@@ -49,7 +49,6 @@ typedef struct {
 } call_data;
 
 typedef struct {
-  grpc_channel *master;
   grpc_status_code error_code;
   const char *error_message;
 } channel_data;
@@ -84,8 +83,7 @@ static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
 }
 
 static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  channel_data *chand = elem->channel_data;
-  return grpc_channel_get_target(chand->master);
+  return NULL;
 }
 
 static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
@@ -111,10 +109,8 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx,
 static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               grpc_channel_element *elem,
                               grpc_channel_element_args *args) {
-  channel_data *chand = elem->channel_data;
   GPR_ASSERT(args->is_first);
   GPR_ASSERT(args->is_last);
-  chand->master = args->master;
 }
 
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c
index c9a54d9237..92bd53411d 100644
--- a/src/core/surface/secure_channel_create.c
+++ b/src/core/surface/secure_channel_create.c
@@ -228,7 +228,6 @@ static grpc_subchannel *subchannel_factory_create_subchannel(
   gpr_mu_init(&c->mu);
   gpr_ref_init(&c->refs, 1);
   args->args = final_args;
-  args->master = f->master;
   s = grpc_subchannel_create(&c->base, args);
   grpc_connector_unref(exec_ctx, &c->base);
   grpc_channel_args_destroy(final_args);
@@ -305,22 +304,22 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds,
   f->master = channel;
   GRPC_CHANNEL_INTERNAL_REF(channel, "subchannel_factory");
   resolver = grpc_resolver_create(target, &f->base);
-  if (!resolver) {
-    grpc_exec_ctx_finish(&exec_ctx);
-    return NULL;
+  if (resolver) {
+    grpc_client_channel_set_resolver(
+        &exec_ctx, grpc_channel_get_channel_stack(channel), resolver);
+    GRPC_RESOLVER_UNREF(&exec_ctx, resolver, "create");
   }
-
-  grpc_client_channel_set_resolver(
-      &exec_ctx, grpc_channel_get_channel_stack(channel), resolver);
-  GRPC_RESOLVER_UNREF(&exec_ctx, resolver, "create");
   grpc_subchannel_factory_unref(&exec_ctx, &f->base);
   GRPC_SECURITY_CONNECTOR_UNREF(&security_connector->base, "channel_create");
-
   grpc_channel_args_destroy(args_copy);
   if (new_args_from_connector != NULL) {
     grpc_channel_args_destroy(new_args_from_connector);
   }
 
+  if (!resolver) {
+    GRPC_CHANNEL_INTERNAL_UNREF(&exec_ctx, channel, "subchannel_factory");
+    channel = NULL;
+  }
   grpc_exec_ctx_finish(&exec_ctx);
 
   return channel;
diff --git a/src/core/surface/server.c b/src/core/surface/server.c
index cdbd542d9a..1e1cde3648 100644
--- a/src/core/surface/server.c
+++ b/src/core/surface/server.c
@@ -1007,7 +1007,7 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
 
   /* lock, and gather up some stuff to do */
   gpr_mu_lock(&server->mu_global);
-  grpc_cq_begin_op(cq);
+  grpc_cq_begin_op(cq, tag);
   if (server->shutdown_published) {
     grpc_cq_end_op(&exec_ctx, cq, tag, 1, done_published_shutdown, NULL,
                    gpr_malloc(sizeof(grpc_cq_completion)));
@@ -1176,7 +1176,7 @@ grpc_call_error grpc_server_request_call(
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
   }
-  grpc_cq_begin_op(cq_for_notification);
+  grpc_cq_begin_op(cq_for_notification, tag);
   details->reserved = NULL;
   rc->type = BATCH_CALL;
   rc->server = server;
@@ -1213,7 +1213,7 @@ grpc_call_error grpc_server_request_registered_call(
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
   }
-  grpc_cq_begin_op(cq_for_notification);
+  grpc_cq_begin_op(cq_for_notification, tag);
   rc->type = REGISTERED_CALL;
   rc->server = server;
   rc->tag = tag;
diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c
index 990bc4aa23..5ce7c1955b 100644
--- a/src/core/surface/server_chttp2.c
+++ b/src/core/surface/server_chttp2.c
@@ -101,9 +101,7 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
   }
 
   tcp = grpc_tcp_server_create();
-  if (!tcp) {
-    goto error;
-  }
+  GPR_ASSERT(tcp);
 
   for (i = 0; i < resolved->naddrs; i++) {
     grpc_tcp_listener *listener;
@@ -111,7 +109,7 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
         tcp, (struct sockaddr *)&resolved->addrs[i].addr,
         resolved->addrs[i].len);
     port_temp = grpc_tcp_listener_get_port(listener);
-    if (port_temp >= 0) {
+    if (port_temp > 0) {
       if (port_num == -1) {
         port_num = port_temp;
       } else {
diff --git a/src/core/surface/server_create.c b/src/core/surface/server_create.c
index c7811a6d88..f30093e06b 100644
--- a/src/core/surface/server_create.c
+++ b/src/core/surface/server_create.c
@@ -32,14 +32,20 @@
  */
 
 #include <grpc/grpc.h>
+#include "src/core/census/grpc_filter.h"
+#include "src/core/channel/channel_args.h"
+#include "src/core/channel/compress_filter.h"
 #include "src/core/surface/api_trace.h"
 #include "src/core/surface/completion_queue.h"
 #include "src/core/surface/server.h"
-#include "src/core/channel/compress_filter.h"
 
 grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
-  const grpc_channel_filter *filters[] = {&grpc_compress_filter};
+  const grpc_channel_filter *filters[3];
+  size_t num_filters = 0;
+  filters[num_filters++] = &grpc_compress_filter;
+  if (grpc_channel_args_is_census_enabled(args)) {
+    filters[num_filters++] = &grpc_server_census_filter;
+  }
   GRPC_API_TRACE("grpc_server_create(%p, %p)", 2, (args, reserved));
-  return grpc_server_create_from_filters(filters, GPR_ARRAY_SIZE(filters),
-                                         args);
+  return grpc_server_create_from_filters(filters, num_filters, args);
 }
diff --git a/src/core/transport/chttp2/frame_data.c b/src/core/transport/chttp2/frame_data.c
index 08cc760aba..5b16ce6334 100644
--- a/src/core/transport/chttp2/frame_data.c
+++ b/src/core/transport/chttp2/frame_data.c
@@ -53,7 +53,8 @@ void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_data_parser *parser) {
   grpc_byte_stream *bs;
   if (parser->parsing_frame) {
-    grpc_chttp2_incoming_byte_stream_finished(exec_ctx, parser->parsing_frame);
+    grpc_chttp2_incoming_byte_stream_finished(exec_ctx, parser->parsing_frame,
+                                              0, 1);
   }
   while (
       (bs = grpc_chttp2_incoming_frame_queue_pop(&parser->incoming_frames))) {
@@ -118,7 +119,7 @@ void grpc_chttp2_encode_data(gpr_uint32 id, gpr_slice_buffer *inbuf,
 
   hdr = gpr_slice_malloc(9);
   p = GPR_SLICE_START_PTR(hdr);
-  GPR_ASSERT(write_bytes < (1<<24));
+  GPR_ASSERT(write_bytes < (1 << 24));
   *p++ = (gpr_uint8)(write_bytes >> 16);
   *p++ = (gpr_uint8)(write_bytes >> 8);
   *p++ = (gpr_uint8)(write_bytes);
@@ -218,7 +219,8 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
         grpc_chttp2_incoming_byte_stream_push(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg)));
-        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame);
+        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, 1,
+                                                  1);
         p->parsing_frame = NULL;
         p->state = GRPC_CHTTP2_DATA_FH_0;
         return GRPC_CHTTP2_PARSE_OK;
@@ -227,7 +229,8 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg),
                           (size_t)(cur + p->frame_size - beg)));
-        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame);
+        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, 1,
+                                                  1);
         p->parsing_frame = NULL;
         cur += p->frame_size;
         goto fh_0; /* loop */
diff --git a/src/core/transport/chttp2/frame_ping.c b/src/core/transport/chttp2/frame_ping.c
index 4d2c54269d..8e763278ff 100644
--- a/src/core/transport/chttp2/frame_ping.c
+++ b/src/core/transport/chttp2/frame_ping.c
@@ -76,7 +76,6 @@ grpc_chttp2_parse_error grpc_chttp2_ping_parser_parse(
   gpr_uint8 *const end = GPR_SLICE_END_PTR(slice);
   gpr_uint8 *cur = beg;
   grpc_chttp2_ping_parser *p = parser;
-  grpc_chttp2_outstanding_ping *ping;
 
   while (p->byte != 8 && cur != end) {
     p->opaque_8bytes[p->byte] = *cur;
@@ -87,15 +86,7 @@ grpc_chttp2_parse_error grpc_chttp2_ping_parser_parse(
   if (p->byte == 8) {
     GPR_ASSERT(is_last);
     if (p->is_ack) {
-      for (ping = transport_parsing->pings.next;
-           ping != &transport_parsing->pings; ping = ping->next) {
-        if (0 == memcmp(p->opaque_8bytes, ping->id, 8)) {
-          grpc_exec_ctx_enqueue(exec_ctx, ping->on_recv, 1);
-        }
-        ping->next->prev = ping->prev;
-        ping->prev->next = ping->next;
-        gpr_free(ping);
-      }
+      grpc_chttp2_ack_ping(exec_ctx, transport_parsing, p->opaque_8bytes);
     } else {
       gpr_slice_buffer_add(&transport_parsing->qbuf,
                            grpc_chttp2_ping_create(1, p->opaque_8bytes));
diff --git a/src/core/transport/chttp2/frame_settings.c b/src/core/transport/chttp2/frame_settings.c
index f03fb45908..383b6e7f93 100644
--- a/src/core/transport/chttp2/frame_settings.c
+++ b/src/core/transport/chttp2/frame_settings.c
@@ -44,7 +44,7 @@
 #include "src/core/transport/chttp2/http2_errors.h"
 #include "src/core/transport/chttp2_transport.h"
 
-#define MAX_MAX_HEADER_LIST_SIZE (1024*1024*1024)
+#define MAX_MAX_HEADER_LIST_SIZE (1024 * 1024 * 1024)
 
 /* HTTP/2 mandated initial connection settings */
 const grpc_chttp2_setting_parameters
@@ -62,8 +62,9 @@ const grpc_chttp2_setting_parameters
          GRPC_CHTTP2_FLOW_CONTROL_ERROR},
         {"MAX_FRAME_SIZE", 16384, 16384, 16777215,
          GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_CHTTP2_PROTOCOL_ERROR},
-        {"MAX_HEADER_LIST_SIZE", MAX_MAX_HEADER_LIST_SIZE, 0, MAX_MAX_HEADER_LIST_SIZE,
-         GRPC_CHTTP2_CLAMP_INVALID_VALUE, GRPC_CHTTP2_PROTOCOL_ERROR},
+        {"MAX_HEADER_LIST_SIZE", MAX_MAX_HEADER_LIST_SIZE, 0,
+         MAX_MAX_HEADER_LIST_SIZE, GRPC_CHTTP2_CLAMP_INVALID_VALUE,
+         GRPC_CHTTP2_PROTOCOL_ERROR},
 };
 
 static gpr_uint8 *fill_header(gpr_uint8 *out, gpr_uint32 length,
diff --git a/src/core/transport/chttp2/hpack_encoder.c b/src/core/transport/chttp2/hpack_encoder.c
index 7575031e58..6c558bc1cb 100644
--- a/src/core/transport/chttp2/hpack_encoder.c
+++ b/src/core/transport/chttp2/hpack_encoder.c
@@ -365,10 +365,10 @@ static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem,
   GPR_ASSERT(GPR_SLICE_LENGTH(elem->key->slice) > 0);
   if (GPR_SLICE_START_PTR(elem->key->slice)[0] != ':') { /* regular header */
     st->seen_regular_header = 1;
-  } else if (st->seen_regular_header != 0) { /* reserved header */
-    gpr_log(GPR_ERROR,
-            "Reserved header (colon-prefixed) happening after regular ones.");
-    abort();
+  } else {
+    GPR_ASSERT(
+        st->seen_regular_header == 0 &&
+        "Reserved header (colon-prefixed) happening after regular ones.");
   }
 
   inc_filter(HASH_FRAGMENT_1(elem_hash), &c->filter_elems_sum, c->filter_elems);
diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h
index 45d2599cdc..4ad900378b 100644
--- a/src/core/transport/chttp2/internal.h
+++ b/src/core/transport/chttp2/internal.h
@@ -65,6 +65,7 @@ typedef enum {
   GRPC_CHTTP2_LIST_WRITTEN,
   GRPC_CHTTP2_LIST_PARSING_SEEN,
   GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING,
+  GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING,
   GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT,
   /** streams that are waiting to start because there are too many concurrent
       streams on the connection */
@@ -151,6 +152,7 @@ struct grpc_chttp2_incoming_byte_stream {
   grpc_byte_stream base;
   gpr_refcount refs;
   struct grpc_chttp2_incoming_byte_stream *next_message;
+  int failed;
 
   grpc_chttp2_transport *transport;
   grpc_chttp2_stream *stream;
@@ -283,9 +285,6 @@ struct grpc_chttp2_transport_parsing {
   gpr_slice goaway_text;
 
   gpr_int64 outgoing_window;
-
-  /** pings awaiting responses */
-  grpc_chttp2_outstanding_ping pings;
 };
 
 struct grpc_chttp2_transport {
@@ -394,8 +393,6 @@ typedef struct {
   gpr_uint8 write_closed;
   /** is this stream reading half-closed (boolean) */
   gpr_uint8 read_closed;
-  /** is this stream finished closing (and reportably closed) */
-  gpr_uint8 finished_close;
   /** is this stream in the stream map? (boolean) */
   gpr_uint8 in_stream_map;
   /** has this stream seen an error? if 1, then pending incoming frames
@@ -589,6 +586,13 @@ int grpc_chttp2_list_pop_closed_waiting_for_parsing(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global **stream_global);
 
+void grpc_chttp2_list_add_closed_waiting_for_writing(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global);
+int grpc_chttp2_list_pop_closed_waiting_for_writing(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global **stream_global);
+
 grpc_chttp2_stream_parsing *grpc_chttp2_parsing_lookup_stream(
     grpc_chttp2_transport_parsing *transport_parsing, gpr_uint32 id);
 grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream(
@@ -745,6 +749,11 @@ void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx,
                                            grpc_chttp2_incoming_byte_stream *bs,
                                            gpr_slice slice);
 void grpc_chttp2_incoming_byte_stream_finished(
-    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs);
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, int success,
+    int from_parsing_thread);
+
+void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx,
+                          grpc_chttp2_transport_parsing *parsing,
+                          const gpr_uint8 *opaque_8bytes);
 
 #endif
diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c
index a4c85b4e57..49f951d08b 100644
--- a/src/core/transport/chttp2/stream_lists.c
+++ b/src/core/transport/chttp2/stream_lists.c
@@ -353,6 +353,26 @@ int grpc_chttp2_list_pop_closed_waiting_for_parsing(
   return r;
 }
 
+void grpc_chttp2_list_add_closed_waiting_for_writing(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global) {
+  stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global),
+                  STREAM_FROM_GLOBAL(stream_global),
+                  GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING);
+}
+
+int grpc_chttp2_list_pop_closed_waiting_for_writing(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global **stream_global) {
+  grpc_chttp2_stream *stream;
+  int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream,
+                          GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING);
+  if (r != 0) {
+    *stream_global = &stream->global;
+  }
+  return r;
+}
+
 void grpc_chttp2_register_stream(grpc_chttp2_transport *t,
                                  grpc_chttp2_stream *s) {
   stream_list_add_tail(t, s, GRPC_CHTTP2_LIST_ALL_STREAMS);
diff --git a/src/core/transport/chttp2/timeout_encoding.c b/src/core/transport/chttp2/timeout_encoding.c
index 8a9b290ecb..7ec8b4e8bf 100644
--- a/src/core/transport/chttp2/timeout_encoding.c
+++ b/src/core/transport/chttp2/timeout_encoding.c
@@ -36,14 +36,15 @@
 #include <stdio.h>
 #include <string.h>
 
+#include <grpc/support/port_platform.h>
 #include "src/core/support/string.h"
 
-static int round_up(int x, int divisor) {
+static gpr_int64 round_up(gpr_int64 x, gpr_int64 divisor) {
   return (x / divisor + (x % divisor != 0)) * divisor;
 }
 
 /* round an integer up to the next value with three significant figures */
-static int round_up_to_three_sig_figs(int x) {
+static gpr_int64 round_up_to_three_sig_figs(gpr_int64 x) {
   if (x < 1000) return x;
   if (x < 10000) return round_up(x, 10);
   if (x < 100000) return round_up(x, 100);
@@ -57,13 +58,13 @@ static int round_up_to_three_sig_figs(int x) {
 /* encode our minimum viable timeout value */
 static void enc_tiny(char *buffer) { memcpy(buffer, "1n", 3); }
 
-static void enc_ext(char *buffer, long value, char ext) {
-  int n = gpr_ltoa(value, buffer);
+static void enc_ext(char *buffer, gpr_int64 value, char ext) {
+  int n = gpr_int64toa(value, buffer);
   buffer[n] = ext;
   buffer[n + 1] = 0;
 }
 
-static void enc_seconds(char *buffer, long sec) {
+static void enc_seconds(char *buffer, gpr_int64 sec) {
   if (sec % 3600 == 0) {
     enc_ext(buffer, sec / 3600, 'H');
   } else if (sec % 60 == 0) {
@@ -73,7 +74,7 @@ static void enc_seconds(char *buffer, long sec) {
   }
 }
 
-static void enc_nanos(char *buffer, int x) {
+static void enc_nanos(char *buffer, gpr_int64 x) {
   x = round_up_to_three_sig_figs(x);
   if (x < 100000) {
     if (x % 1000 == 0) {
@@ -97,7 +98,7 @@ static void enc_nanos(char *buffer, int x) {
   }
 }
 
-static void enc_micros(char *buffer, int x) {
+static void enc_micros(char *buffer, gpr_int64 x) {
   x = round_up_to_three_sig_figs(x);
   if (x < 100000) {
     if (x % 1000 == 0) {
@@ -123,7 +124,7 @@ void grpc_chttp2_encode_timeout(gpr_timespec timeout, char *buffer) {
     enc_nanos(buffer, timeout.tv_nsec);
   } else if (timeout.tv_sec < 1000 && timeout.tv_nsec != 0) {
     enc_micros(buffer,
-               (int)(timeout.tv_sec * 1000000) +
+               (gpr_int64)(timeout.tv_sec * 1000000) +
                    (timeout.tv_nsec / 1000 + (timeout.tv_nsec % 1000 != 0)));
   } else {
     enc_seconds(buffer, timeout.tv_sec + (timeout.tv_nsec != 0));
diff --git a/src/core/transport/chttp2/varint.h b/src/core/transport/chttp2/varint.h
index 970d1e17e2..5acb15d032 100644
--- a/src/core/transport/chttp2/varint.h
+++ b/src/core/transport/chttp2/varint.h
@@ -66,7 +66,8 @@ void grpc_chttp2_hpack_write_varint_tail(gpr_uint32 tail_value,
     if ((length) == 1u) {                                                     \
       (tgt)[0] = (gpr_uint8)((prefix_or) | (n));                              \
     } else {                                                                  \
-      (tgt)[0] = (prefix_or) | (gpr_uint8)GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits);        \
+      (tgt)[0] =                                                              \
+          (prefix_or) | (gpr_uint8)GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits);    \
       grpc_chttp2_hpack_write_varint_tail(                                    \
           (n)-GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits), (tgt) + 1, (length)-1); \
     }                                                                         \
diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c
index 805d05222d..b5ca42d69c 100644
--- a/src/core/transport/chttp2/writing.c
+++ b/src/core/transport/chttp2/writing.c
@@ -332,17 +332,12 @@ void grpc_chttp2_cleanup_writing(
 
   while (grpc_chttp2_list_pop_written_stream(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
-    if (stream_writing->sent_trailing_metadata) {
-      grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
-                                     !transport_global->is_client, 1);
-    }
     if (stream_writing->sent_initial_metadata) {
       grpc_chttp2_complete_closure_step(
           exec_ctx, &stream_global->send_initial_metadata_finished, 1);
     }
     if (stream_writing->sent_message) {
       GPR_ASSERT(stream_writing->send_message == NULL);
-      GPR_ASSERT(stream_global->send_message_finished);
       grpc_chttp2_complete_closure_step(
           exec_ctx, &stream_global->send_message_finished, 1);
       stream_writing->sent_message = 0;
@@ -351,6 +346,10 @@ void grpc_chttp2_cleanup_writing(
       grpc_chttp2_complete_closure_step(
           exec_ctx, &stream_global->send_trailing_metadata_finished, 1);
     }
+    if (stream_writing->sent_trailing_metadata) {
+      grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
+                                     !transport_global->is_client, 1);
+    }
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing");
   }
   gpr_slice_buffer_reset_and_unref(&transport_writing->outbuf);
diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c
index deb903324e..3f21d84aa7 100644
--- a/src/core/transport/chttp2_transport.c
+++ b/src/core/transport/chttp2_transport.c
@@ -139,6 +139,9 @@ static void incoming_byte_stream_update_flow_control(
     grpc_chttp2_stream_global *stream_global, size_t max_size_hint,
     size_t have_already);
 
+static void fail_pending_writes(grpc_exec_ctx *exec_ctx, 
+                                grpc_chttp2_stream_global *stream_global);
+
 /*
  * CONSTRUCTION/DESTRUCTION/REFCOUNTING
  */
@@ -630,6 +633,7 @@ void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx,
                                    void *transport_writing_ptr, int success) {
   grpc_chttp2_transport_writing *transport_writing = transport_writing_ptr;
   grpc_chttp2_transport *t = TRANSPORT_FROM_WRITING(transport_writing);
+  grpc_chttp2_stream_global *stream_global;
 
   GPR_TIMER_BEGIN("grpc_chttp2_terminate_writing", 0);
 
@@ -643,6 +647,11 @@ void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx,
 
   grpc_chttp2_cleanup_writing(exec_ctx, &t->global, &t->writing);
 
+  while (grpc_chttp2_list_pop_closed_waiting_for_writing(&t->global, &stream_global)) {
+    fail_pending_writes(exec_ctx, stream_global);
+    GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "finish_writes");
+  }
+
   /* leave the writing flag up on shutdown to prevent further writes in unlock()
      from starting */
   t->writing_active = 0;
@@ -909,6 +918,26 @@ static void send_ping_locked(grpc_chttp2_transport *t, grpc_closure *on_recv) {
   gpr_slice_buffer_add(&t->global.qbuf, grpc_chttp2_ping_create(0, p->id));
 }
 
+void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx,
+                          grpc_chttp2_transport_parsing *transport_parsing,
+                          const gpr_uint8 *opaque_8bytes) {
+  grpc_chttp2_outstanding_ping *ping;
+  grpc_chttp2_transport *t = TRANSPORT_FROM_PARSING(transport_parsing);
+  grpc_chttp2_transport_global *transport_global = &t->global;
+  lock(t);
+  for (ping = transport_global->pings.next; ping != &transport_global->pings;
+       ping = ping->next) {
+    if (0 == memcmp(opaque_8bytes, ping->id, 8)) {
+      grpc_exec_ctx_enqueue(exec_ctx, ping->on_recv, 1);
+      ping->next->prev = ping->prev;
+      ping->prev->next = ping->next;
+      gpr_free(ping);
+      break;
+    }
+  }
+  unlock(exec_ctx, t);
+}
+
 static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                                  grpc_transport_op *op) {
   grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt;
@@ -918,7 +947,7 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
 
   grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, 1);
 
-  if (op->on_connectivity_state_change) {
+  if (op->on_connectivity_state_change != NULL) {
     grpc_connectivity_state_notify_on_state_change(
         exec_ctx, &t->channel_callback.state_tracker, op->connectivity_state,
         op->on_connectivity_state_change);
@@ -1028,6 +1057,12 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     t->parsing.incoming_stream = NULL;
     grpc_chttp2_parsing_become_skip_parser(exec_ctx, &t->parsing);
   }
+  if (s->parsing.data_parser.parsing_frame != NULL) {
+    grpc_chttp2_incoming_byte_stream_finished(
+        exec_ctx, s->parsing.data_parser.parsing_frame, 0, 0);
+    s->parsing.data_parser.parsing_frame = NULL;
+  }
+
   if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) {
     close_transport_locked(exec_ctx, t);
   }
@@ -1095,6 +1130,16 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
   }
 }
 
+static void fail_pending_writes(grpc_exec_ctx *exec_ctx, 
+                                grpc_chttp2_stream_global *stream_global) {
+  grpc_chttp2_complete_closure_step(
+      exec_ctx, &stream_global->send_initial_metadata_finished, 0);
+  grpc_chttp2_complete_closure_step(
+      exec_ctx, &stream_global->send_trailing_metadata_finished, 0);
+  grpc_chttp2_complete_closure_step(exec_ctx,
+                                    &stream_global->send_message_finished, 0);
+}
+
 void grpc_chttp2_mark_stream_closed(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global, int close_reads,
@@ -1111,6 +1156,13 @@ void grpc_chttp2_mark_stream_closed(
   }
   if (close_writes && !stream_global->write_closed) {
     stream_global->write_closed = 1;
+    if (TRANSPORT_FROM_GLOBAL(transport_global)->writing_active) {
+      GRPC_CHTTP2_STREAM_REF(stream_global, "finish_writes");
+      grpc_chttp2_list_add_closed_waiting_for_writing(transport_global,
+                                                      stream_global);
+    } else {
+      fail_pending_writes(exec_ctx, stream_global);
+    }
   }
   if (stream_global->read_closed && stream_global->write_closed) {
     if (stream_global->id != 0 &&
@@ -1122,7 +1174,6 @@ void grpc_chttp2_mark_stream_closed(
         remove_stream(exec_ctx, TRANSPORT_FROM_GLOBAL(transport_global),
                       stream_global->id);
       }
-      stream_global->finished_close = 1;
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2");
     }
   }
@@ -1336,7 +1387,6 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, int success) {
       GPR_ASSERT(stream_global->write_closed);
       GPR_ASSERT(stream_global->read_closed);
       remove_stream(exec_ctx, t, stream_global->id);
-      stream_global->finished_close = 1;
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2");
     }
   }
@@ -1467,6 +1517,10 @@ static int incoming_byte_stream_next(grpc_exec_ctx *exec_ctx,
     *slice = gpr_slice_buffer_take_first(&bs->slices);
     unlock(exec_ctx, bs->transport);
     return 1;
+  } else if (bs->failed) {
+    grpc_exec_ctx_enqueue(exec_ctx, on_complete, 0);
+    unlock(exec_ctx, bs->transport);
+    return 0;
   } else {
     bs->on_next = on_complete;
     bs->next = slice;
@@ -1501,7 +1555,29 @@ void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx,
 }
 
 void grpc_chttp2_incoming_byte_stream_finished(
-    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs) {
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, int success,
+    int from_parsing_thread) {
+  if (!success) {
+    if (from_parsing_thread) {
+      gpr_mu_lock(&bs->transport->mu);
+    }
+    grpc_exec_ctx_enqueue(exec_ctx, bs->on_next, 0);
+    bs->on_next = NULL;
+    bs->failed = 1;
+    if (from_parsing_thread) {
+      gpr_mu_unlock(&bs->transport->mu);
+    }
+  } else {
+#ifndef NDEBUG
+    if (from_parsing_thread) {
+      gpr_mu_lock(&bs->transport->mu);
+    }
+    GPR_ASSERT(bs->on_next == NULL);
+    if (from_parsing_thread) {
+      gpr_mu_unlock(&bs->transport->mu);
+    }
+#endif
+  }
   incoming_byte_stream_unref(bs);
 }
 
@@ -1522,6 +1598,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create(
   gpr_slice_buffer_init(&incoming_byte_stream->slices);
   incoming_byte_stream->on_next = NULL;
   incoming_byte_stream->is_tail = 1;
+  incoming_byte_stream->failed = 0;
   if (add_to_queue->head == NULL) {
     add_to_queue->head = incoming_byte_stream;
   } else {
diff --git a/src/core/transport/connectivity_state.c b/src/core/transport/connectivity_state.c
index 09b298c131..3c3fd4671d 100644
--- a/src/core/transport/connectivity_state.c
+++ b/src/core/transport/connectivity_state.c
@@ -54,8 +54,7 @@ const char *grpc_connectivity_state_name(grpc_connectivity_state state) {
     case GRPC_CHANNEL_FATAL_FAILURE:
       return "FATAL_FAILURE";
   }
-  abort();
-  return "UNKNOWN";
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
 }
 
 void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker,
@@ -88,7 +87,7 @@ void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx,
 grpc_connectivity_state grpc_connectivity_state_check(
     grpc_connectivity_state_tracker *tracker) {
   if (grpc_connectivity_state_trace) {
-    gpr_log(GPR_DEBUG, "CONWATCH: %s: get %s", tracker->name,
+    gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
             grpc_connectivity_state_name(tracker->current_state));
   }
   return tracker->current_state;
@@ -98,42 +97,47 @@ int grpc_connectivity_state_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
     grpc_connectivity_state *current, grpc_closure *notify) {
   if (grpc_connectivity_state_trace) {
-    gpr_log(GPR_DEBUG, "CONWATCH: %s: from %s [cur=%s] notify=%p",
-            tracker->name, grpc_connectivity_state_name(*current),
-            grpc_connectivity_state_name(tracker->current_state), notify);
+    if (current == NULL) {
+      gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker,
+              tracker->name, notify);
+    } else {
+      gpr_log(GPR_DEBUG, "CONWATCH: %p %s: from %s [cur=%s] notify=%p", tracker,
+              tracker->name, grpc_connectivity_state_name(*current),
+              grpc_connectivity_state_name(tracker->current_state), notify);
+    }
   }
-  if (tracker->current_state != *current) {
-    *current = tracker->current_state;
-    grpc_exec_ctx_enqueue(exec_ctx, notify, 1);
+  if (current == NULL) {
+    grpc_connectivity_state_watcher *w = tracker->watchers;
+    if (w != NULL && w->notify == notify) {
+      grpc_exec_ctx_enqueue(exec_ctx, notify, 0);
+      tracker->watchers = w->next;
+      gpr_free(w);
+      return 0;
+    }
+    while (w != NULL) {
+      grpc_connectivity_state_watcher *rm_candidate = w->next;
+      if (rm_candidate != NULL && rm_candidate->notify == notify) {
+        grpc_exec_ctx_enqueue(exec_ctx, notify, 0);
+        w->next = w->next->next;
+        gpr_free(rm_candidate);
+        return 0;
+      }
+      w = w->next;
+    }
+    return 0;
   } else {
-    grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w));
-    w->current = current;
-    w->notify = notify;
-    w->next = tracker->watchers;
-    tracker->watchers = w;
-  }
-  return tracker->current_state == GRPC_CHANNEL_IDLE;
-}
-
-int grpc_connectivity_state_change_unsubscribe(
-    grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
-    grpc_closure *subscribed_notify) {
-  grpc_connectivity_state_watcher *w = tracker->watchers;
-  if (w != NULL && w->notify == subscribed_notify) {
-    tracker->watchers = w->next;
-    gpr_free(w);
-    return 1;
-  }
-  while (w != NULL) {
-    grpc_connectivity_state_watcher *rm_candidate = w->next;
-    if (rm_candidate != NULL && rm_candidate->notify == subscribed_notify) {
-      w->next = w->next->next;
-      gpr_free(rm_candidate);
-      return 1;
+    if (tracker->current_state != *current) {
+      *current = tracker->current_state;
+      grpc_exec_ctx_enqueue(exec_ctx, notify, 1);
+    } else {
+      grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w));
+      w->current = current;
+      w->notify = notify;
+      w->next = tracker->watchers;
+      tracker->watchers = w;
     }
-    w = w->next;
+    return tracker->current_state == GRPC_CHANNEL_IDLE;
   }
-  return 0;
 }
 
 void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
@@ -142,7 +146,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
                                  const char *reason) {
   grpc_connectivity_state_watcher *w;
   if (grpc_connectivity_state_trace) {
-    gpr_log(GPR_DEBUG, "SET: %s: %s --> %s [%s]", tracker->name,
+    gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s]", tracker, tracker->name,
             grpc_connectivity_state_name(tracker->current_state),
             grpc_connectivity_state_name(state), reason);
   }
diff --git a/src/core/transport/connectivity_state.h b/src/core/transport/connectivity_state.h
index 119b1c1554..a4eb6652e5 100644
--- a/src/core/transport/connectivity_state.h
+++ b/src/core/transport/connectivity_state.h
@@ -57,6 +57,8 @@ typedef struct {
 
 extern int grpc_connectivity_state_trace;
 
+const char *grpc_connectivity_state_name(grpc_connectivity_state state);
+
 void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker,
                                   grpc_connectivity_state init_state,
                                   const char *name);
@@ -73,16 +75,11 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
 grpc_connectivity_state grpc_connectivity_state_check(
     grpc_connectivity_state_tracker *tracker);
 
-/** Return 1 if the channel should start connecting, 0 otherwise */
+/** Return 1 if the channel should start connecting, 0 otherwise.
+    If current==NULL cancel notify if it is already queued (success==0 in that
+    case) */
 int grpc_connectivity_state_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
     grpc_connectivity_state *current, grpc_closure *notify);
 
-/** Remove \a subscribed_notify from the list of closures to be called on a
- * state change if present, returning 1. Otherwise, nothing is done and return
- * 0. */
-int grpc_connectivity_state_change_unsubscribe(
-    grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
-    grpc_closure *subscribed_notify);
-
 #endif /* GRPC_INTERNAL_CORE_TRANSPORT_CONNECTIVITY_STATE_H */
diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c
index 4328bdd684..02b2402820 100644
--- a/src/core/transport/metadata.c
+++ b/src/core/transport/metadata.c
@@ -703,7 +703,7 @@ static int conforms_to(grpc_mdstr *s, const gpr_uint8 *legal_bits) {
 
 int grpc_mdstr_is_legal_header(grpc_mdstr *s) {
   static const gpr_uint8 legal_header_bits[256 / 8] = {
-      0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xff, 0x03, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0xff, 0x03, 0x00, 0x00, 0x00,
       0x80, 0xfe, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
   return conforms_to(s, legal_header_bits);
@@ -711,7 +711,7 @@ int grpc_mdstr_is_legal_header(grpc_mdstr *s) {
 
 int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s) {
   static const gpr_uint8 legal_header_bits[256 / 8] = {
-      0x00, 0x00, 0x00, 0x00, 0xff, 0xef, 0xff, 0xff, 0xff, 0xff, 0xff,
+      0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
   return conforms_to(s, legal_header_bits);
diff --git a/src/core/transport/metadata_batch.c b/src/core/transport/metadata_batch.c
index c5d39e0c9f..1266862f82 100644
--- a/src/core/transport/metadata_batch.c
+++ b/src/core/transport/metadata_batch.c
@@ -133,16 +133,6 @@ void grpc_metadata_batch_link_tail(grpc_metadata_batch *batch,
   link_tail(&batch->list, storage);
 }
 
-void grpc_metadata_batch_merge(grpc_metadata_batch *target,
-                               grpc_metadata_batch *to_add) {
-  grpc_linked_mdelem *l;
-  grpc_linked_mdelem *next;
-  for (l = to_add->list.head; l; l = next) {
-    next = l->next;
-    link_tail(&target->list, l);
-  }
-}
-
 void grpc_metadata_batch_move(grpc_metadata_batch *dst,
                               grpc_metadata_batch *src) {
   *dst = *src;
diff --git a/src/core/transport/metadata_batch.h b/src/core/transport/metadata_batch.h
index fc3a46004f..1b0d1fda3e 100644
--- a/src/core/transport/metadata_batch.h
+++ b/src/core/transport/metadata_batch.h
@@ -63,8 +63,6 @@ typedef struct grpc_metadata_batch {
 
 void grpc_metadata_batch_init(grpc_metadata_batch *batch);
 void grpc_metadata_batch_destroy(grpc_metadata_batch *batch);
-void grpc_metadata_batch_merge(grpc_metadata_batch *target,
-                               grpc_metadata_batch *add);
 void grpc_metadata_batch_clear(grpc_metadata_batch *batch);
 int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch);
 
diff --git a/src/core/transport/static_metadata.c b/src/core/transport/static_metadata.c
index e7aff325c2..6e42379eee 100644
--- a/src/core/transport/static_metadata.c
+++ b/src/core/transport/static_metadata.c
@@ -33,11 +33,11 @@
  * WARNING: Auto-generated code.
  *
  * To make changes to this file, change
- * tools/codegen/core/gen_static_metadata.py,
+ *tools/codegen/core/gen_static_metadata.py,
  * and then re-run it.
  *
  * See metadata.h for an explanation of the interface here, and metadata.c for
- * an
+ *an
  * explanation of what's going on.
  */
 
@@ -54,103 +54,34 @@ gpr_uintptr grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = {
 
 const gpr_uint8
     grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] = {
-        11, 33, 10, 33, 12, 33, 12, 47, 13, 33, 14, 33, 15, 33, 16, 33, 17, 33,
-        19, 33, 20, 33, 21, 33, 22, 33, 23, 33, 24, 33, 25, 33, 26, 33, 27, 33,
-        28, 18, 28, 33, 29, 33, 30, 33, 34, 33, 35, 33, 36, 33, 37, 33, 40, 31,
-        40, 32, 40, 46, 40, 51, 40, 52, 40, 53, 40, 54, 41, 31, 41, 46, 41, 51,
-        44, 0,  44, 1,  44, 2,  48, 33, 55, 33, 56, 33, 57, 33, 58, 33, 59, 33,
-        60, 33, 61, 33, 62, 33, 63, 33, 64, 38, 64, 66, 65, 76, 65, 77, 67, 33,
-        68, 33, 69, 33, 70, 33, 71, 33, 72, 33, 73, 39, 73, 49, 73, 50, 74, 33,
-        75, 33, 78, 3,  78, 4,  78, 5,  78, 6,  78, 7,  78, 8,  78, 9,  79, 33,
-        80, 81, 82, 33, 83, 33, 84, 33, 85, 33, 86, 33};
+        11, 35, 10, 35, 12, 35, 12, 49, 13, 35, 14, 35, 15, 35, 16, 35, 17, 35,
+        19, 35, 20, 35, 21, 35, 24, 35, 25, 35, 26, 35, 27, 35, 28, 35, 29, 35,
+        30, 18, 30, 35, 31, 35, 32, 35, 36, 35, 37, 35, 38, 35, 39, 35, 42, 33,
+        42, 34, 42, 48, 42, 53, 42, 54, 42, 55, 42, 56, 43, 33, 43, 48, 43, 53,
+        46, 0,  46, 1,  46, 2,  50, 35, 57, 35, 58, 35, 59, 35, 60, 35, 61, 35,
+        62, 35, 63, 35, 64, 35, 65, 35, 66, 40, 66, 68, 67, 78, 67, 79, 69, 35,
+        70, 35, 71, 35, 72, 35, 73, 35, 74, 35, 75, 41, 75, 51, 75, 52, 76, 35,
+        77, 35, 80, 3,  80, 4,  80, 5,  80, 6,  80, 7,  80, 8,  80, 9,  81, 35,
+        82, 83, 84, 35, 85, 35, 86, 35, 87, 35, 88, 35};
 
 const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
-    "0",
-    "1",
-    "2",
-    "200",
-    "204",
-    "206",
-    "304",
-    "400",
-    "404",
-    "500",
-    "accept",
-    "accept-charset",
-    "accept-encoding",
-    "accept-language",
-    "accept-ranges",
-    "access-control-allow-origin",
-    "age",
-    "allow",
-    "application/grpc",
-    ":authority",
-    "authorization",
-    "cache-control",
-    "content-disposition",
-    "content-encoding",
-    "content-language",
-    "content-length",
-    "content-location",
-    "content-range",
-    "content-type",
-    "cookie",
-    "date",
-    "deflate",
-    "deflate,gzip",
-    "",
-    "etag",
-    "expect",
-    "expires",
-    "from",
-    "GET",
-    "grpc",
-    "grpc-accept-encoding",
-    "grpc-encoding",
-    "grpc-internal-encoding-request",
-    "grpc-message",
-    "grpc-status",
-    "grpc-timeout",
-    "gzip",
-    "gzip, deflate",
-    "host",
-    "http",
-    "https",
-    "identity",
-    "identity,deflate",
-    "identity,deflate,gzip",
-    "identity,gzip",
-    "if-match",
-    "if-modified-since",
-    "if-none-match",
-    "if-range",
-    "if-unmodified-since",
-    "last-modified",
-    "link",
-    "location",
-    "max-forwards",
-    ":method",
-    ":path",
-    "POST",
-    "proxy-authenticate",
-    "proxy-authorization",
-    "range",
-    "referer",
-    "refresh",
-    "retry-after",
-    ":scheme",
-    "server",
-    "set-cookie",
-    "/",
-    "/index.html",
-    ":status",
-    "strict-transport-security",
-    "te",
-    "trailers",
-    "transfer-encoding",
-    "user-agent",
-    "vary",
-    "via",
+    "0", "1", "2", "200", "204", "206", "304", "400", "404", "500", "accept",
+    "accept-charset", "accept-encoding", "accept-language", "accept-ranges",
+    "access-control-allow-origin", "age", "allow", "application/grpc",
+    ":authority", "authorization", "cache-control", "census", "census-bin",
+    "content-disposition", "content-encoding", "content-language",
+    "content-length", "content-location", "content-range", "content-type",
+    "cookie", "date", "deflate", "deflate,gzip", "", "etag", "expect",
+    "expires", "from", "GET", "grpc", "grpc-accept-encoding", "grpc-encoding",
+    "grpc-internal-encoding-request", "grpc-message", "grpc-status",
+    "grpc-timeout", "gzip", "gzip, deflate", "host", "http", "https",
+    "identity", "identity,deflate", "identity,deflate,gzip", "identity,gzip",
+    "if-match", "if-modified-since", "if-none-match", "if-range",
+    "if-unmodified-since", "last-modified", "link", "location", "max-forwards",
+    ":method", ":path", "POST", "proxy-authenticate", "proxy-authorization",
+    "range", "referer", "refresh", "retry-after", ":scheme", "server",
+    "set-cookie", "/", "/index.html", ":status", "strict-transport-security",
+    "te", "trailers", "transfer-encoding", "user-agent", "vary", "via",
     "www-authenticate"};
 
 const gpr_uint8 grpc_static_accept_encoding_metadata[8] = {0,  29, 26, 30,
diff --git a/src/core/transport/static_metadata.h b/src/core/transport/static_metadata.h
index e9055fb45c..0e630b1b03 100644
--- a/src/core/transport/static_metadata.h
+++ b/src/core/transport/static_metadata.h
@@ -46,7 +46,7 @@
 
 #include "src/core/transport/metadata.h"
 
-#define GRPC_STATIC_MDSTR_COUNT 87
+#define GRPC_STATIC_MDSTR_COUNT 89
 extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 /* "0" */
 #define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0])
@@ -92,137 +92,141 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_AUTHORIZATION (&grpc_static_mdstr_table[20])
 /* "cache-control" */
 #define GRPC_MDSTR_CACHE_CONTROL (&grpc_static_mdstr_table[21])
+/* "census" */
+#define GRPC_MDSTR_CENSUS (&grpc_static_mdstr_table[22])
+/* "census-bin" */
+#define GRPC_MDSTR_CENSUS_BIN (&grpc_static_mdstr_table[23])
 /* "content-disposition" */
-#define GRPC_MDSTR_CONTENT_DISPOSITION (&grpc_static_mdstr_table[22])
+#define GRPC_MDSTR_CONTENT_DISPOSITION (&grpc_static_mdstr_table[24])
 /* "content-encoding" */
-#define GRPC_MDSTR_CONTENT_ENCODING (&grpc_static_mdstr_table[23])
+#define GRPC_MDSTR_CONTENT_ENCODING (&grpc_static_mdstr_table[25])
 /* "content-language" */
-#define GRPC_MDSTR_CONTENT_LANGUAGE (&grpc_static_mdstr_table[24])
+#define GRPC_MDSTR_CONTENT_LANGUAGE (&grpc_static_mdstr_table[26])
 /* "content-length" */
-#define GRPC_MDSTR_CONTENT_LENGTH (&grpc_static_mdstr_table[25])
+#define GRPC_MDSTR_CONTENT_LENGTH (&grpc_static_mdstr_table[27])
 /* "content-location" */
-#define GRPC_MDSTR_CONTENT_LOCATION (&grpc_static_mdstr_table[26])
+#define GRPC_MDSTR_CONTENT_LOCATION (&grpc_static_mdstr_table[28])
 /* "content-range" */
-#define GRPC_MDSTR_CONTENT_RANGE (&grpc_static_mdstr_table[27])
+#define GRPC_MDSTR_CONTENT_RANGE (&grpc_static_mdstr_table[29])
 /* "content-type" */
-#define GRPC_MDSTR_CONTENT_TYPE (&grpc_static_mdstr_table[28])
+#define GRPC_MDSTR_CONTENT_TYPE (&grpc_static_mdstr_table[30])
 /* "cookie" */
-#define GRPC_MDSTR_COOKIE (&grpc_static_mdstr_table[29])
+#define GRPC_MDSTR_COOKIE (&grpc_static_mdstr_table[31])
 /* "date" */
-#define GRPC_MDSTR_DATE (&grpc_static_mdstr_table[30])
+#define GRPC_MDSTR_DATE (&grpc_static_mdstr_table[32])
 /* "deflate" */
-#define GRPC_MDSTR_DEFLATE (&grpc_static_mdstr_table[31])
+#define GRPC_MDSTR_DEFLATE (&grpc_static_mdstr_table[33])
 /* "deflate,gzip" */
-#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (&grpc_static_mdstr_table[32])
+#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (&grpc_static_mdstr_table[34])
 /* "" */
-#define GRPC_MDSTR_EMPTY (&grpc_static_mdstr_table[33])
+#define GRPC_MDSTR_EMPTY (&grpc_static_mdstr_table[35])
 /* "etag" */
-#define GRPC_MDSTR_ETAG (&grpc_static_mdstr_table[34])
+#define GRPC_MDSTR_ETAG (&grpc_static_mdstr_table[36])
 /* "expect" */
-#define GRPC_MDSTR_EXPECT (&grpc_static_mdstr_table[35])
+#define GRPC_MDSTR_EXPECT (&grpc_static_mdstr_table[37])
 /* "expires" */
-#define GRPC_MDSTR_EXPIRES (&grpc_static_mdstr_table[36])
+#define GRPC_MDSTR_EXPIRES (&grpc_static_mdstr_table[38])
 /* "from" */
-#define GRPC_MDSTR_FROM (&grpc_static_mdstr_table[37])
+#define GRPC_MDSTR_FROM (&grpc_static_mdstr_table[39])
 /* "GET" */
-#define GRPC_MDSTR_GET (&grpc_static_mdstr_table[38])
+#define GRPC_MDSTR_GET (&grpc_static_mdstr_table[40])
 /* "grpc" */
-#define GRPC_MDSTR_GRPC (&grpc_static_mdstr_table[39])
+#define GRPC_MDSTR_GRPC (&grpc_static_mdstr_table[41])
 /* "grpc-accept-encoding" */
-#define GRPC_MDSTR_GRPC_ACCEPT_ENCODING (&grpc_static_mdstr_table[40])
+#define GRPC_MDSTR_GRPC_ACCEPT_ENCODING (&grpc_static_mdstr_table[42])
 /* "grpc-encoding" */
-#define GRPC_MDSTR_GRPC_ENCODING (&grpc_static_mdstr_table[41])
+#define GRPC_MDSTR_GRPC_ENCODING (&grpc_static_mdstr_table[43])
 /* "grpc-internal-encoding-request" */
-#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (&grpc_static_mdstr_table[42])
+#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (&grpc_static_mdstr_table[44])
 /* "grpc-message" */
-#define GRPC_MDSTR_GRPC_MESSAGE (&grpc_static_mdstr_table[43])
+#define GRPC_MDSTR_GRPC_MESSAGE (&grpc_static_mdstr_table[45])
 /* "grpc-status" */
-#define GRPC_MDSTR_GRPC_STATUS (&grpc_static_mdstr_table[44])
+#define GRPC_MDSTR_GRPC_STATUS (&grpc_static_mdstr_table[46])
 /* "grpc-timeout" */
-#define GRPC_MDSTR_GRPC_TIMEOUT (&grpc_static_mdstr_table[45])
+#define GRPC_MDSTR_GRPC_TIMEOUT (&grpc_static_mdstr_table[47])
 /* "gzip" */
-#define GRPC_MDSTR_GZIP (&grpc_static_mdstr_table[46])
+#define GRPC_MDSTR_GZIP (&grpc_static_mdstr_table[48])
 /* "gzip, deflate" */
-#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (&grpc_static_mdstr_table[47])
+#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (&grpc_static_mdstr_table[49])
 /* "host" */
-#define GRPC_MDSTR_HOST (&grpc_static_mdstr_table[48])
+#define GRPC_MDSTR_HOST (&grpc_static_mdstr_table[50])
 /* "http" */
-#define GRPC_MDSTR_HTTP (&grpc_static_mdstr_table[49])
+#define GRPC_MDSTR_HTTP (&grpc_static_mdstr_table[51])
 /* "https" */
-#define GRPC_MDSTR_HTTPS (&grpc_static_mdstr_table[50])
+#define GRPC_MDSTR_HTTPS (&grpc_static_mdstr_table[52])
 /* "identity" */
-#define GRPC_MDSTR_IDENTITY (&grpc_static_mdstr_table[51])
+#define GRPC_MDSTR_IDENTITY (&grpc_static_mdstr_table[53])
 /* "identity,deflate" */
-#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (&grpc_static_mdstr_table[52])
+#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (&grpc_static_mdstr_table[54])
 /* "identity,deflate,gzip" */
 #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \
-  (&grpc_static_mdstr_table[53])
+  (&grpc_static_mdstr_table[55])
 /* "identity,gzip" */
-#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (&grpc_static_mdstr_table[54])
+#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (&grpc_static_mdstr_table[56])
 /* "if-match" */
-#define GRPC_MDSTR_IF_MATCH (&grpc_static_mdstr_table[55])
+#define GRPC_MDSTR_IF_MATCH (&grpc_static_mdstr_table[57])
 /* "if-modified-since" */
-#define GRPC_MDSTR_IF_MODIFIED_SINCE (&grpc_static_mdstr_table[56])
+#define GRPC_MDSTR_IF_MODIFIED_SINCE (&grpc_static_mdstr_table[58])
 /* "if-none-match" */
-#define GRPC_MDSTR_IF_NONE_MATCH (&grpc_static_mdstr_table[57])
+#define GRPC_MDSTR_IF_NONE_MATCH (&grpc_static_mdstr_table[59])
 /* "if-range" */
-#define GRPC_MDSTR_IF_RANGE (&grpc_static_mdstr_table[58])
+#define GRPC_MDSTR_IF_RANGE (&grpc_static_mdstr_table[60])
 /* "if-unmodified-since" */
-#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (&grpc_static_mdstr_table[59])
+#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (&grpc_static_mdstr_table[61])
 /* "last-modified" */
-#define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[60])
+#define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[62])
 /* "link" */
-#define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[61])
+#define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[63])
 /* "location" */
-#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[62])
+#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[64])
 /* "max-forwards" */
-#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[63])
+#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[65])
 /* ":method" */
-#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[64])
+#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[66])
 /* ":path" */
-#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[65])
+#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[67])
 /* "POST" */
-#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[66])
+#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[68])
 /* "proxy-authenticate" */
-#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[67])
+#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[69])
 /* "proxy-authorization" */
-#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[68])
+#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[70])
 /* "range" */
-#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[69])
+#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[71])
 /* "referer" */
-#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[70])
+#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[72])
 /* "refresh" */
-#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[71])
+#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[73])
 /* "retry-after" */
-#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[72])
+#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[74])
 /* ":scheme" */
-#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[73])
+#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[75])
 /* "server" */
-#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[74])
+#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[76])
 /* "set-cookie" */
-#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[75])
+#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[77])
 /* "/" */
-#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[76])
+#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[78])
 /* "/index.html" */
-#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[77])
+#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[79])
 /* ":status" */
-#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[78])
+#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[80])
 /* "strict-transport-security" */
-#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[79])
+#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[81])
 /* "te" */
-#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[80])
+#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[82])
 /* "trailers" */
-#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[81])
+#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[83])
 /* "transfer-encoding" */
-#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[82])
+#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[84])
 /* "user-agent" */
-#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[83])
+#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[85])
 /* "vary" */
-#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[84])
+#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[86])
 /* "via" */
-#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[85])
+#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[87])
 /* "www-authenticate" */
-#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[86])
+#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[88])
 
 #define GRPC_STATIC_MDELEM_COUNT 78
 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c
index f2bebc62f3..2ab978be46 100644
--- a/src/core/transport/transport.c
+++ b/src/core/transport/transport.c
@@ -40,8 +40,8 @@
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason) {
   gpr_atm val = gpr_atm_no_barrier_load(&refcount->refs.count);
-  gpr_log(GPR_DEBUG, "STREAM %p:%p   REF %d->%d %s", refcount,
-          refcount->destroy.cb_arg, val, val + 1, reason);
+  gpr_log(GPR_DEBUG, "%s %p:%p   REF %d->%d %s", refcount->object_type,
+          refcount, refcount->destroy.cb_arg, val, val + 1, reason);
 #else
 void grpc_stream_ref(grpc_stream_refcount *refcount) {
 #endif
@@ -52,8 +52,8 @@ void grpc_stream_ref(grpc_stream_refcount *refcount) {
 void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount,
                        const char *reason) {
   gpr_atm val = gpr_atm_no_barrier_load(&refcount->refs.count);
-  gpr_log(GPR_DEBUG, "STREAM %p:%p UNREF %d->%d %s", refcount,
-          refcount->destroy.cb_arg, val, val - 1, reason);
+  gpr_log(GPR_DEBUG, "%s %p:%p UNREF %d->%d %s", refcount->object_type,
+          refcount, refcount->destroy.cb_arg, val, val - 1, reason);
 #else
 void grpc_stream_unref(grpc_exec_ctx *exec_ctx,
                        grpc_stream_refcount *refcount) {
@@ -63,6 +63,19 @@ void grpc_stream_unref(grpc_exec_ctx *exec_ctx,
   }
 }
 
+#ifdef GRPC_STREAM_REFCOUNT_DEBUG
+void grpc_stream_ref_init(grpc_stream_refcount *refcount, int initial_refs,
+                          grpc_iomgr_cb_func cb, void *cb_arg,
+                          const char *object_type) {
+  refcount->object_type = object_type;
+#else
+void grpc_stream_ref_init(grpc_stream_refcount *refcount, int initial_refs,
+                          grpc_iomgr_cb_func cb, void *cb_arg) {
+#endif
+  gpr_ref_init(&refcount->refs, initial_refs);
+  grpc_closure_init(&refcount->destroy, cb, cb_arg);
+}
+
 size_t grpc_transport_stream_size(grpc_transport *transport) {
   return transport->vtable->sizeof_stream;
 }
diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h
index f296ce8251..f94f0ae76e 100644
--- a/src/core/transport/transport.h
+++ b/src/core/transport/transport.h
@@ -50,19 +50,32 @@ typedef struct grpc_transport grpc_transport;
    for a stream. */
 typedef struct grpc_stream grpc_stream;
 
+/*#define GRPC_STREAM_REFCOUNT_DEBUG*/
+
 typedef struct grpc_stream_refcount {
   gpr_refcount refs;
   grpc_closure destroy;
+#ifdef GRPC_STREAM_REFCOUNT_DEBUG
+  const char *object_type;
+#endif
 } grpc_stream_refcount;
 
-/*#define GRPC_STREAM_REFCOUNT_DEBUG*/
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
+void grpc_stream_ref_init(grpc_stream_refcount *refcount, int initial_refs,
+                          grpc_iomgr_cb_func cb, void *cb_arg,
+                          const char *object_type);
 void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason);
 void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount,
                        const char *reason);
+#define GRPC_STREAM_REF_INIT(rc, ir, cb, cb_arg, objtype) \
+  grpc_stream_ref_init(rc, ir, cb, cb_arg, objtype)
 #else
+void grpc_stream_ref_init(grpc_stream_refcount *refcount, int initial_refs,
+                          grpc_iomgr_cb_func cb, void *cb_arg);
 void grpc_stream_ref(grpc_stream_refcount *refcount);
 void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount);
+#define GRPC_STREAM_REF_INIT(rc, ir, cb, cb_arg, objtype) \
+  grpc_stream_ref_init(rc, ir, cb, cb_arg)
 #endif
 
 /* Transport stream op: a set of operations to perform on a transport
@@ -96,7 +109,7 @@ typedef struct grpc_transport_stream_op {
 typedef struct grpc_transport_op {
   /** called when processing of this op is done */
   grpc_closure *on_consumed;
-  /** connectivity monitoring */
+  /** connectivity monitoring - set connectivity_state to NULL to unsubscribe */
   grpc_closure *on_connectivity_state_change;
   grpc_connectivity_state *connectivity_state;
   /** should the transport be disconnected */
diff --git a/src/core/transport/transport_op_string.c b/src/core/transport/transport_op_string.c
index f3b6db29d6..98b51afc88 100644
--- a/src/core/transport/transport_op_string.c
+++ b/src/core/transport/transport_op_string.c
@@ -63,8 +63,8 @@ static void put_metadata_list(gpr_strvec *b, grpc_metadata_batch md) {
   }
   if (gpr_time_cmp(md.deadline, gpr_inf_future(md.deadline.clock_type)) != 0) {
     char *tmp;
-    gpr_asprintf(&tmp, " deadline=%d.%09d", md.deadline.tv_sec,
-                 md.deadline.tv_nsec);
+    gpr_asprintf(&tmp, " deadline=%lld.%09d", (long long)md.deadline.tv_sec,
+                 (int)md.deadline.tv_nsec);
     gpr_strvec_add(b, tmp);
   }
 }
diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc
index 9bb358b233..2aa532808c 100644
--- a/src/cpp/client/client_context.cc
+++ b/src/cpp/client/client_context.cc
@@ -45,17 +45,31 @@
 
 namespace grpc {
 
+class DefaultGlobalClientCallbacks GRPC_FINAL
+    : public ClientContext::GlobalCallbacks {
+ public:
+  void DefaultConstructor(ClientContext* context) GRPC_OVERRIDE {}
+  void Destructor(ClientContext* context) GRPC_OVERRIDE {}
+};
+
+static DefaultGlobalClientCallbacks g_default_client_callbacks;
+static ClientContext::GlobalCallbacks* g_client_callbacks =
+    &g_default_client_callbacks;
+
 ClientContext::ClientContext()
     : initial_metadata_received_(false),
       call_(nullptr),
       call_canceled_(false),
       deadline_(gpr_inf_future(GPR_CLOCK_REALTIME)),
-      propagate_from_call_(nullptr) {}
+      propagate_from_call_(nullptr) {
+  g_client_callbacks->DefaultConstructor(this);
+}
 
 ClientContext::~ClientContext() {
   if (call_) {
     grpc_call_destroy(call_);
   }
+  g_client_callbacks->Destructor(this);
 }
 
 std::unique_ptr<ClientContext> ClientContext::FromServerContext(
@@ -124,4 +138,11 @@ grpc::string ClientContext::peer() const {
   return peer;
 }
 
+void ClientContext::SetGlobalCallbacks(GlobalCallbacks* client_callbacks) {
+  GPR_ASSERT(g_client_callbacks == &g_default_client_callbacks);
+  GPR_ASSERT(client_callbacks != NULL);
+  GPR_ASSERT(client_callbacks != &g_default_client_callbacks);
+  g_client_callbacks = client_callbacks;
+}
+
 }  // namespace grpc
diff --git a/src/cpp/proto/proto_utils.cc b/src/cpp/proto/proto_utils.cc
index b1330fde7f..898a1d4f58 100644
--- a/src/cpp/proto/proto_utils.cc
+++ b/src/cpp/proto/proto_utils.cc
@@ -33,6 +33,8 @@
 
 #include <grpc++/impl/proto_utils.h>
 
+#include <climits>
+
 #include <grpc/grpc.h>
 #include <grpc/byte_buffer.h>
 #include <grpc/byte_buffer_reader.h>
@@ -70,7 +72,9 @@ class GrpcBufferWriter GRPC_FINAL
       slice_ = gpr_slice_malloc(block_size_);
     }
     *data = GPR_SLICE_START_PTR(slice_);
-    byte_count_ += * size = GPR_SLICE_LENGTH(slice_);
+    // On win x64, int is only 32bit
+    GPR_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX);
+    byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_);
     gpr_slice_buffer_add(slice_buffer_, slice_);
     return true;
   }
@@ -124,7 +128,9 @@ class GrpcBufferReader GRPC_FINAL
     }
     gpr_slice_unref(slice_);
     *data = GPR_SLICE_START_PTR(slice_);
-    byte_count_ += * size = GPR_SLICE_LENGTH(slice_);
+    // On win x64, int is only 32bit
+    GPR_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX);
+    byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_);
     return true;
   }
 
diff --git a/src/cpp/util/time.cc b/src/cpp/util/time.cc
index 6157a37745..9c38b34879 100644
--- a/src/cpp/util/time.cc
+++ b/src/cpp/util/time.cc
@@ -57,8 +57,8 @@ void Timepoint2Timespec(const system_clock::time_point& from,
     return;
   }
   nanoseconds nsecs = duration_cast<nanoseconds>(deadline - secs);
-  to->tv_sec = (time_t)secs.count();
-  to->tv_nsec = (int)nsecs.count();
+  to->tv_sec = (gpr_int64)secs.count();
+  to->tv_nsec = (gpr_int32)nsecs.count();
   to->clock_type = GPR_CLOCK_REALTIME;
 }
 
@@ -73,8 +73,8 @@ void TimepointHR2Timespec(const high_resolution_clock::time_point& from,
     return;
   }
   nanoseconds nsecs = duration_cast<nanoseconds>(deadline - secs);
-  to->tv_sec = (time_t)secs.count();
-  to->tv_nsec = (int)nsecs.count();
+  to->tv_sec = (gpr_int64)secs.count();
+  to->tv_nsec = (gpr_int32)nsecs.count();
   to->clock_type = GPR_CLOCK_REALTIME;
 }
 
diff --git a/src/csharp/Grpc.Core.Tests/CallOptionsTest.cs b/src/csharp/Grpc.Core.Tests/CallOptionsTest.cs
new file mode 100644
index 0000000000..a3a613be74
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/CallOptionsTest.cs
@@ -0,0 +1,88 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class CallOptionsTest
+    {
+        [Test]
+        public void WithMethods()
+        {
+            var options = new CallOptions();
+            
+            var metadata = new Metadata();
+            Assert.AreSame(metadata, options.WithHeaders(metadata).Headers);
+
+            var deadline = DateTime.UtcNow;
+            Assert.AreEqual(deadline, options.WithDeadline(deadline).Deadline.Value);
+
+            var token = new CancellationTokenSource().Token;
+            Assert.AreEqual(token, options.WithCancellationToken(token).CancellationToken);
+
+            // Change original instance is unchanged.
+            Assert.IsNull(options.Headers);
+            Assert.IsNull(options.Deadline);
+            Assert.AreEqual(CancellationToken.None, options.CancellationToken);
+            Assert.IsNull(options.WriteOptions);
+            Assert.IsNull(options.PropagationToken);
+            Assert.IsNull(options.Credentials);
+        }
+
+        [Test]
+        public void Normalize()
+        {
+            Assert.AreSame(Metadata.Empty, new CallOptions().Normalize().Headers);
+            Assert.AreEqual(DateTime.MaxValue, new CallOptions().Normalize().Deadline.Value);
+
+            var deadline = DateTime.UtcNow;
+            var propagationToken1 = new ContextPropagationToken(CallSafeHandle.NullInstance, deadline, CancellationToken.None,
+                new ContextPropagationOptions(propagateDeadline: true, propagateCancellation: false));
+            Assert.AreEqual(deadline, new CallOptions(propagationToken: propagationToken1).Normalize().Deadline.Value);
+            Assert.Throws(typeof(ArgumentException), () => new CallOptions(deadline: deadline, propagationToken: propagationToken1).Normalize());
+
+            var token = new CancellationTokenSource().Token;
+            var propagationToken2 = new ContextPropagationToken(CallSafeHandle.NullInstance, deadline, token,
+                new ContextPropagationOptions(propagateDeadline: false, propagateCancellation: true));
+            Assert.AreEqual(token, new CallOptions(propagationToken: propagationToken2).Normalize().CancellationToken);
+            Assert.Throws(typeof(ArgumentException), () => new CallOptions(cancellationToken: token, propagationToken: propagationToken2).Normalize());
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core.Tests/ChannelOptionsTest.cs b/src/csharp/Grpc.Core.Tests/ChannelOptionsTest.cs
index 52be77c846..d2b5a436fd 100644
--- a/src/csharp/Grpc.Core.Tests/ChannelOptionsTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ChannelOptionsTest.cs
@@ -38,7 +38,7 @@ using Grpc.Core.Internal;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 
-namespace Grpc.Core.Internal.Tests
+namespace Grpc.Core.Tests
 {
     public class ChannelOptionsTest
     {
diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
index b683751bc0..77f6a63156 100644
--- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
@@ -32,6 +32,7 @@
 #endregion
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using System.Threading;
@@ -144,6 +145,45 @@ namespace Grpc.Core.Tests
             var call = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall());
             await call.RequestStream.WriteAllAsync(new string[] { "A", "B", "C" });
             Assert.AreEqual("ABC", await call.ResponseAsync);
+
+            Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
+            Assert.IsNotNull(call.GetTrailers());
+        }
+
+        [Test]
+        public async Task ServerStreamingCall()
+        {
+            helper.ServerStreamingHandler = new ServerStreamingServerMethod<string, string>(async (request, responseStream, context) =>
+            {
+                await responseStream.WriteAllAsync(request.Split(new []{' '}));
+                context.ResponseTrailers.Add("xyz", "");
+            });
+
+            var call = Calls.AsyncServerStreamingCall(helper.CreateServerStreamingCall(), "A B C");
+            CollectionAssert.AreEqual(new string[] { "A", "B", "C" }, await call.ResponseStream.ToListAsync());
+
+            Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
+            Assert.IsNotNull("xyz", call.GetTrailers()[0].Key);
+        }
+
+        [Test]
+        public async Task DuplexStreamingCall()
+        {
+            helper.DuplexStreamingHandler = new DuplexStreamingServerMethod<string, string>(async (requestStream, responseStream, context) =>
+            {
+                while (await requestStream.MoveNext())
+                {
+                    await responseStream.WriteAsync(requestStream.Current);
+                }
+                context.ResponseTrailers.Add("xyz", "xyz-value");
+            });
+
+            var call = Calls.AsyncDuplexStreamingCall(helper.CreateDuplexStreamingCall());
+            await call.RequestStream.WriteAllAsync(new string[] { "A", "B", "C" });
+            CollectionAssert.AreEqual(new string[] { "A", "B", "C" }, await call.ResponseStream.ToListAsync());
+
+            Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
+            Assert.IsNotNull("xyz-value", call.GetTrailers()[0].Value);
         }
 
         [Test]
@@ -219,7 +259,7 @@ namespace Grpc.Core.Tests
         }
 
         [Test]
-        public void PeerInfoPresent()
+        public void ServerCallContext_PeerInfoPresent()
         {
             helper.UnaryHandler = new UnaryServerMethod<string, string>(async (request, context) =>
             {
@@ -231,6 +271,18 @@ namespace Grpc.Core.Tests
         }
 
         [Test]
+        public void ServerCallContext_HostAndMethodPresent()
+        {
+            helper.UnaryHandler = new UnaryServerMethod<string, string>(async (request, context) =>
+            {
+                Assert.IsTrue(context.Host.Contains(Host));
+                Assert.AreEqual("/tests.Test/Unary", context.Method);
+                return "PASS";
+            });
+            Assert.AreEqual("PASS", Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "abc"));
+        }
+
+        [Test]
         public async Task Channel_WaitForStateChangedAsync()
         {
             helper.UnaryHandler = new UnaryServerMethod<string, string>(async (request, context) =>
diff --git a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
index 2db3f286f7..90c510ec61 100644
--- a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
@@ -69,11 +69,19 @@ namespace Grpc.Core.Tests
         [Test]
         public async Task PropagateCancellation()
         {
+            var readyToCancelTcs = new TaskCompletionSource<object>();
+            var successTcs = new TaskCompletionSource<string>();
+
             helper.UnaryHandler = new UnaryServerMethod<string, string>(async (request, context) =>
             {
-                // check that we didn't obtain the default cancellation token.
-                Assert.IsTrue(context.CancellationToken.CanBeCanceled);
-                return "PASS";
+                readyToCancelTcs.SetResult(null);  // child call running, ready to parent call
+
+                while (!context.CancellationToken.IsCancellationRequested)
+                {
+                    await Task.Delay(10);
+                }
+                successTcs.SetResult("CHILD_CALL_CANCELLED");
+                return "";
             });
 
             helper.ClientStreamingHandler = new ClientStreamingServerMethod<string, string>(async (requestStream, context) =>
@@ -82,13 +90,23 @@ namespace Grpc.Core.Tests
                 Assert.IsNotNull(propagationToken.ParentCall);
 
                 var callOptions = new CallOptions(propagationToken: propagationToken);
-                return await Calls.AsyncUnaryCall(helper.CreateUnaryCall(callOptions), "xyz");
+                try
+                {
+                    await Calls.AsyncUnaryCall(helper.CreateUnaryCall(callOptions), "xyz");
+                }
+                catch(RpcException)
+                {
+                    // Child call will get cancelled, eat the exception.
+                }
+                return "";
             });
                 
             var cts = new CancellationTokenSource();
-            var call = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall(new CallOptions(cancellationToken: cts.Token)));
-            await call.RequestStream.CompleteAsync();
-            Assert.AreEqual("PASS", await call);
+            var parentCall = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall(new CallOptions(cancellationToken: cts.Token)));
+            await readyToCancelTcs.Task;
+            cts.Cancel();
+            Assert.Throws(typeof(RpcException), async () => await parentCall);
+            Assert.AreEqual("CHILD_CALL_CANCELLED", await successTcs.Task);
         }
 
         [Test]
diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
index 70b83f7fb1..475c792347 100644
--- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
+++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
@@ -64,6 +64,7 @@
       <Link>Version.cs</Link>
     </Compile>
     <Compile Include="CallCredentialsTest.cs" />
+    <Compile Include="CallOptionsTest.cs" />
     <Compile Include="UserAgentStringTest.cs" />
     <Compile Include="FakeCredentials.cs" />
     <Compile Include="MarshallingErrorsTest.cs" />
@@ -90,6 +91,7 @@
     <Compile Include="ContextPropagationTest.cs" />
     <Compile Include="MetadataTest.cs" />
     <Compile Include="PerformanceTest.cs" />
+    <Compile Include="SanityTest.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>
diff --git a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
index 9be5450d81..74f7f2497a 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
@@ -82,17 +82,17 @@ namespace Grpc.Core.Internal.Tests
         public void ToDateTime()
         {
             Assert.AreEqual(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc),
-                new Timespec(IntPtr.Zero, 0).ToDateTime());
+                new Timespec(0, 0).ToDateTime());
 
             Assert.AreEqual(new DateTime(1970, 1, 1, 0, 0, 10, DateTimeKind.Utc).AddTicks(50),
-                new Timespec(new IntPtr(10), 5000).ToDateTime());
+                new Timespec(10, 5000).ToDateTime());
 
             Assert.AreEqual(new DateTime(2015, 7, 21, 4, 21, 48, DateTimeKind.Utc),
-                new Timespec(new IntPtr(1437452508), 0).ToDateTime());
+                new Timespec(1437452508, 0).ToDateTime());
 
             // before epoch
             Assert.AreEqual(new DateTime(1969, 12, 31, 23, 59, 55, DateTimeKind.Utc).AddTicks(10),
-                new Timespec(new IntPtr(-5), 1000).ToDateTime());
+                new Timespec(-5, 1000).ToDateTime());
 
             // infinity
             Assert.AreEqual(DateTime.MaxValue, Timespec.InfFuture.ToDateTime());
@@ -100,80 +100,62 @@ namespace Grpc.Core.Internal.Tests
 
             // nanos are rounded to ticks are rounded up
             Assert.AreEqual(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddTicks(1),
-                new Timespec(IntPtr.Zero, 99).ToDateTime());
+                new Timespec(0, 99).ToDateTime());
 
             // Illegal inputs
             Assert.Throws(typeof(InvalidOperationException),
-                () => new Timespec(new IntPtr(0), -2).ToDateTime());
+                () => new Timespec(0, -2).ToDateTime());
             Assert.Throws(typeof(InvalidOperationException),
-                () => new Timespec(new IntPtr(0), 1000 * 1000 * 1000).ToDateTime());
+                () => new Timespec(0, 1000 * 1000 * 1000).ToDateTime());
             Assert.Throws(typeof(InvalidOperationException),
-                () => new Timespec(new IntPtr(0), 0, GPRClockType.Monotonic).ToDateTime());
+                () => new Timespec(0, 0, GPRClockType.Monotonic).ToDateTime());
         }
 
         [Test]
         public void ToDateTime_ReturnsUtc()
         {
-            Assert.AreEqual(DateTimeKind.Utc, new Timespec(new IntPtr(1437452508), 0).ToDateTime().Kind);
-            Assert.AreNotEqual(DateTimeKind.Unspecified, new Timespec(new IntPtr(1437452508), 0).ToDateTime().Kind);
+            Assert.AreEqual(DateTimeKind.Utc, new Timespec(1437452508, 0).ToDateTime().Kind);
+            Assert.AreNotEqual(DateTimeKind.Unspecified, new Timespec(1437452508, 0).ToDateTime().Kind);
         }
 
         [Test]
         public void ToDateTime_Overflow()
-        {
-            // we can only get overflow in ticks arithmetic on 64-bit
-            if (IntPtr.Size == 8)
-            {
-                var timespec = new Timespec(new IntPtr(long.MaxValue - 100), 0);
-                Assert.AreNotEqual(Timespec.InfFuture, timespec);
-                Assert.AreEqual(DateTime.MaxValue, timespec.ToDateTime());
-
-                Assert.AreEqual(DateTime.MinValue, new Timespec(new IntPtr(long.MinValue + 100), 0).ToDateTime());
-            }
-            else
-            {
-                Console.WriteLine("Test cannot be run on this platform, skipping the test.");
-            }
+        {     
+            var timespec = new Timespec(long.MaxValue - 100, 0);
+            Assert.AreNotEqual(Timespec.InfFuture, timespec);
+            Assert.AreEqual(DateTime.MaxValue, timespec.ToDateTime());
+
+            Assert.AreEqual(DateTime.MinValue, new Timespec(long.MinValue + 100, 0).ToDateTime());
         }
 
         [Test]
         public void ToDateTime_OutOfDateTimeRange()
         {
-            // we can only get out of range on 64-bit, on 32 bit the max 
-            // timestamp is ~ Jan 19 2038, which is far within range of DateTime
-            // same case for min value.
-            if (IntPtr.Size == 8)
-            {
-                // DateTime range goes up to year 9999, 20000 years from now should
-                // be out of range.
-                long seconds = 20000L * 365L * 24L * 3600L; 
-
-                var timespec = new Timespec(new IntPtr(seconds), 0);
-                Assert.AreNotEqual(Timespec.InfFuture, timespec);
-                Assert.AreEqual(DateTime.MaxValue, timespec.ToDateTime());
-
-                Assert.AreEqual(DateTime.MinValue, new Timespec(new IntPtr(-seconds), 0).ToDateTime());
-            }
-            else
-            {
-                Console.WriteLine("Test cannot be run on this platform, skipping the test");
-            }
+            // DateTime range goes up to year 9999, 20000 years from now should
+            // be out of range.
+            long seconds = 20000L * 365L * 24L * 3600L; 
+
+            var timespec = new Timespec(seconds, 0);
+            Assert.AreNotEqual(Timespec.InfFuture, timespec);
+            Assert.AreEqual(DateTime.MaxValue, timespec.ToDateTime());
+
+            Assert.AreEqual(DateTime.MinValue, new Timespec(-seconds, 0).ToDateTime());
         }
 
         [Test]
         public void FromDateTime()
         {
-            Assert.AreEqual(new Timespec(IntPtr.Zero, 0),
+            Assert.AreEqual(new Timespec(0, 0),
                 Timespec.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)));
 
-            Assert.AreEqual(new Timespec(new IntPtr(10), 5000),
+            Assert.AreEqual(new Timespec(10, 5000),
                 Timespec.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 10, DateTimeKind.Utc).AddTicks(50)));
 
-            Assert.AreEqual(new Timespec(new IntPtr(1437452508), 0),
+            Assert.AreEqual(new Timespec(1437452508, 0),
                 Timespec.FromDateTime(new DateTime(2015, 7, 21, 4, 21, 48, DateTimeKind.Utc)));
 
             // before epoch
-            Assert.AreEqual(new Timespec(new IntPtr(-5), 1000),
+            Assert.AreEqual(new Timespec(-5, 1000),
                 Timespec.FromDateTime(new DateTime(1969, 12, 31, 23, 59, 55, DateTimeKind.Utc).AddTicks(10)));
 
             // infinity
@@ -184,34 +166,19 @@ namespace Grpc.Core.Internal.Tests
             Assert.Throws(typeof(ArgumentException),
                 () => Timespec.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified)));
         }
-
-        [Test]
-        public void FromDateTime_OutOfTimespecRange()
-        {
-            // we can only get overflow in Timespec on 32-bit
-            if (IntPtr.Size == 4)
-            {
-                Assert.AreEqual(Timespec.InfFuture, Timespec.FromDateTime(new DateTime(2040, 1, 1, 0, 0, 0, DateTimeKind.Utc)));
-                Assert.AreEqual(Timespec.InfPast, Timespec.FromDateTime(new DateTime(1800, 1, 1, 0, 0, 0, DateTimeKind.Utc)));
-            }
-            else
-            {
-                Console.WriteLine("Test cannot be run on this platform, skipping the test.");
-            }
-        }
             
-        // Test attribute commented out to prevent running as part of the default test suite.
-        // [Test]
-        // [Category("Performance")]
+        [Test]
+        [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public void NowBenchmark() 
         {
             // approx Timespec.Now latency <33ns
             BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.Now; });
         }
-
-        // Test attribute commented out to prevent running as part of the default test suite.
-        // [Test]
-        // [Category("Performance")]
+            
+        [Test]
+        [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public void PreciseNowBenchmark()
         {
             // approx Timespec.PreciseNow latency <18ns (when compiled with GRPC_TIMERS_RDTSC)
diff --git a/src/csharp/Grpc.Core.Tests/MetadataTest.cs b/src/csharp/Grpc.Core.Tests/MetadataTest.cs
index ddeb7d0926..49e9de1174 100644
--- a/src/csharp/Grpc.Core.Tests/MetadataTest.cs
+++ b/src/csharp/Grpc.Core.Tests/MetadataTest.cs
@@ -127,5 +127,118 @@ namespace Grpc.Core.Tests
             Assert.Throws(typeof(InvalidOperationException), () => { var v = entry.Value; });
             CollectionAssert.AreEqual(bytes, entry.ValueBytes);
         }
+
+        [Test]
+        public void IndexOf()
+        {
+            var metadata = CreateMetadata();
+            Assert.AreEqual(0, metadata.IndexOf(metadata[0]));
+            Assert.AreEqual(1, metadata.IndexOf(metadata[1]));
+        }
+
+        [Test]
+        public void Insert()
+        {
+            var metadata = CreateMetadata();
+            metadata.Insert(0, new Metadata.Entry("new-key", "new-value"));
+            Assert.AreEqual(3, metadata.Count);
+            Assert.AreEqual("new-key", metadata[0].Key);
+            Assert.AreEqual("abc", metadata[1].Key);
+        }
+
+        [Test]
+        public void RemoveAt()
+        {
+            var metadata = CreateMetadata();
+            metadata.RemoveAt(0);
+            Assert.AreEqual(1, metadata.Count);
+            Assert.AreEqual("xyz", metadata[0].Key);
+        }
+
+        [Test]
+        public void Remove()
+        {
+            var metadata = CreateMetadata();
+            metadata.Remove(metadata[0]);
+            Assert.AreEqual(1, metadata.Count);
+            Assert.AreEqual("xyz", metadata[0].Key);
+        }
+
+        [Test]
+        public void Indexer_Set()
+        {
+            var metadata = CreateMetadata();
+            var entry = new Metadata.Entry("new-key", "new-value");
+
+            metadata[1] = entry;
+            Assert.AreEqual(entry, metadata[1]);
+        }
+
+        [Test]
+        public void Clear()
+        {
+            var metadata = CreateMetadata();
+            metadata.Clear();
+            Assert.AreEqual(0, metadata.Count);
+        }
+
+        [Test]
+        public void Contains()
+        {
+            var metadata = CreateMetadata();
+            Assert.IsTrue(metadata.Contains(metadata[0]));
+            Assert.IsFalse(metadata.Contains(new Metadata.Entry("new-key", "new-value")));
+        }
+
+        [Test]
+        public void CopyTo()
+        {
+            var metadata = CreateMetadata();
+            var array = new Metadata.Entry[metadata.Count + 1];
+
+            metadata.CopyTo(array, 1);
+            Assert.AreEqual(default(Metadata.Entry), array[0]);
+            Assert.AreEqual(metadata[0], array[1]);
+        }
+
+        [Test]
+        public void IEnumerableGetEnumerator()
+        {
+            var metadata = CreateMetadata();
+            var enumerator = (metadata as System.Collections.IEnumerable).GetEnumerator();
+            
+            int i = 0;
+            while (enumerator.MoveNext())
+            {
+                Assert.AreEqual(metadata[i], enumerator.Current);
+                i++;
+            }
+        }
+
+        [Test]
+        public void FreezeMakesReadOnly()
+        {
+            var entry = new Metadata.Entry("new-key", "new-value");
+            var metadata = CreateMetadata().Freeze();
+
+            Assert.IsTrue(metadata.IsReadOnly);
+            Assert.Throws<InvalidOperationException>(() => metadata.Insert(0, entry));
+            Assert.Throws<InvalidOperationException>(() => metadata.RemoveAt(0));
+            Assert.Throws<InvalidOperationException>(() => metadata[0] = entry);
+            Assert.Throws<InvalidOperationException>(() => metadata.Add(entry));
+            Assert.Throws<InvalidOperationException>(() => metadata.Add("new-key", "new-value"));
+            Assert.Throws<InvalidOperationException>(() => metadata.Add("new-key-bin", new byte[] { 0xaa }));
+            Assert.Throws<InvalidOperationException>(() => metadata.Clear());
+            Assert.Throws<InvalidOperationException>(() => metadata.Remove(metadata[0]));
+        }
+
+        private Metadata CreateMetadata()
+        {
+            return new Metadata
+            {
+                { "abc", "abc-value" },
+                { "xyz", "xyz-value" },
+            };
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core.Tests/PInvokeTest.cs b/src/csharp/Grpc.Core.Tests/PInvokeTest.cs
index 073c502daf..af55cb0566 100644
--- a/src/csharp/Grpc.Core.Tests/PInvokeTest.cs
+++ b/src/csharp/Grpc.Core.Tests/PInvokeTest.cs
@@ -59,6 +59,8 @@ namespace Grpc.Core.Tests
         [Test]
         public void CompletionQueueCreateDestroyBenchmark()
         {
+            GrpcEnvironment.AddRef();  // completion queue requires gRPC environment being initialized.
+
             BenchmarkUtil.RunBenchmark(
                 10, 10,
                 () =>
@@ -66,6 +68,8 @@ namespace Grpc.Core.Tests
                     CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.Create();
                     cq.Dispose();
                 });
+
+            GrpcEnvironment.Release();
         }
 
         /// <summary>
diff --git a/src/csharp/Grpc.Core.Tests/PerformanceTest.cs b/src/csharp/Grpc.Core.Tests/PerformanceTest.cs
index 5516cd3377..6815839992 100644
--- a/src/csharp/Grpc.Core.Tests/PerformanceTest.cs
+++ b/src/csharp/Grpc.Core.Tests/PerformanceTest.cs
@@ -67,10 +67,10 @@ namespace Grpc.Core.Tests
             channel.ShutdownAsync().Wait();
             server.ShutdownAsync().Wait();
         }
-
-        // Test attribute commented out to prevent running as part of the default test suite.
-        //[Test]
-        //[Category("Performance")]
+            
+        [Test]
+        [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public void UnaryCallPerformance()
         {
             var profiler = new BasicProfiler();
diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs
new file mode 100644
index 0000000000..343ab1e85a
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs
@@ -0,0 +1,125 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class SanityTest
+    {
+        /// <summary>
+        /// Because we depend on a native library, sometimes when things go wrong, the
+        /// entire NUnit test process crashes. To be able to track down problems better,
+        /// the NUnit tests are run by run_tests.py script in a separate process per test class.
+        /// The list of tests to run is stored in src/csharp/tests.json.
+        /// This test checks that the tests.json file is up to date by discovering all the
+        /// existing NUnit tests in all test assemblies and comparing to contents of tests.json.
+        /// </summary>
+        [Test]
+        public void TestsJsonUpToDate()
+        {
+            var testClasses = DiscoverAllTestClasses();
+            string testsJson = GetTestsJson();
+
+            // we don't have a JSON parser at hand, but check that the test class
+            // name is contained in the file instead.
+            foreach (var className in testClasses) {
+                Assert.IsTrue(testsJson.Contains(className),
+                    string.Format("Test class \"{0}\" is missing in C# tests.json file", className));
+            }
+        }
+
+        /// <summary>
+        /// Gets list of all test classes obtained by inspecting all the test assemblies.
+        /// </summary>
+        private List<string> DiscoverAllTestClasses()
+        {
+            var assemblies = GetTestAssemblies();
+
+            var testClasses = new List<string>();
+            foreach (var assembly in assemblies)
+            {
+                foreach (var t in assembly.GetTypes())
+                {
+                    foreach (var m in t.GetMethods())
+                    {
+                        var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true);
+                        if (attributes.Length > 0)
+                        {
+                            testClasses.Add(t.FullName);
+                            break;
+                        }
+
+                    }
+                }
+            }
+            testClasses.Sort();
+            return testClasses;
+        }
+
+        private string GetTestsJson()
+        {
+            var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+            var testsJsonFile = Path.Combine(assemblyDir, "..", "..", "..", "tests.json");
+
+            return File.ReadAllText(testsJsonFile);
+        }
+
+        private List<Assembly> GetTestAssemblies()
+        {
+            var result = new List<Assembly>();
+            var executingAssembly = Assembly.GetExecutingAssembly();
+
+            result.Add(executingAssembly);
+
+            var otherAssemblies = new[] {
+                "Grpc.Examples.Tests",
+                "Grpc.HealthCheck.Tests",
+                "Grpc.IntegrationTesting"
+            };
+            foreach (var assemblyName in otherAssemblies)
+            {
+                var location = executingAssembly.Location.Replace("Grpc.Core.Tests", assemblyName);
+                result.Add(Assembly.LoadFrom(location));
+            }
+            return result;
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core.Tests/ShutdownTest.cs b/src/csharp/Grpc.Core.Tests/ShutdownTest.cs
index a2be7ddd5e..10d666d109 100644
--- a/src/csharp/Grpc.Core.Tests/ShutdownTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ShutdownTest.cs
@@ -61,17 +61,20 @@ namespace Grpc.Core.Tests
         }
 
         [Test]
-        public async Task AbandonedCall()
+        public async Task AbandonedCall_ServerKillAsync()
         {
+            var readyToShutdown = new TaskCompletionSource<object>();
             helper.DuplexStreamingHandler = new DuplexStreamingServerMethod<string, string>(async (requestStream, responseStream, context) =>
             {
+                readyToShutdown.SetResult(null);
                 await requestStream.ToListAsync();
             });
 
-            var call = Calls.AsyncDuplexStreamingCall(helper.CreateDuplexStreamingCall(new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(1))));
+            var call = Calls.AsyncDuplexStreamingCall(helper.CreateDuplexStreamingCall());
+            await readyToShutdown.Task;  // make sure handler is running
 
-            channel.ShutdownAsync().Wait();
-            server.ShutdownAsync().Wait();
+            await channel.ShutdownAsync();  // channel.ShutdownAsync() works even if there's a pending call.
+            await server.KillAsync();  // server.ShutdownAsync() would hang waiting for the call to finish.
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/CallOptions.cs b/src/csharp/Grpc.Core/CallOptions.cs
index c0f94c63c2..1fda80cb90 100644
--- a/src/csharp/Grpc.Core/CallOptions.cs
+++ b/src/csharp/Grpc.Core/CallOptions.cs
@@ -184,6 +184,7 @@ namespace Grpc.Core
                 {
                     Preconditions.CheckArgument(!newOptions.cancellationToken.CanBeCanceled,
                         "Cannot propagate cancellation token from parent call. The cancellation token has already been set to a non-default value.");
+                    newOptions.cancellationToken = propagationToken.ParentCancellationToken;
                 }
             }
                 
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
index 953f61aa1e..92f8d77e85 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
@@ -238,20 +238,6 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected Exception TrySerialize(TWrite msg, out byte[] payload)
-        {
-            try
-            {
-                payload = serializer(msg);
-                return null;
-            }
-            catch (Exception e)
-            {
-                payload = null;
-                return e;
-            }
-        }
-
         protected Exception TryDeserialize(byte[] payload, out TRead msg)
         {
             using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.TryDeserialize"))
diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
index 0ade701a53..de66759b94 100644
--- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
@@ -284,7 +284,7 @@ namespace Grpc.Core.Internal
             var finishedTask = asyncCall.ServerSideCallAsync();
             var responseStream = new ServerResponseStream<byte[], byte[]>(asyncCall);
 
-            await responseStream.WriteStatusAsync(new Status(StatusCode.Unimplemented, "No such method."), Metadata.Empty).ConfigureAwait(false);
+            await responseStream.WriteStatusAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty).ConfigureAwait(false);
             await finishedTask.ConfigureAwait(false);
         }
     }
diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs
index 38fc067d9f..3031175c8a 100644
--- a/src/csharp/Grpc.Core/Internal/Timespec.cs
+++ b/src/csharp/Grpc.Core/Internal/Timespec.cs
@@ -63,20 +63,18 @@ namespace Grpc.Core.Internal
         [DllImport("grpc_csharp_ext.dll")]
         static extern int gprsharp_sizeof_timespec();
 
-        public Timespec(IntPtr tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, GPRClockType.Realtime)
+        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, GPRClockType.Realtime)
         {
         }
 
-        public Timespec(IntPtr tv_sec, int tv_nsec, GPRClockType clock_type)
+        public Timespec(long tv_sec, int tv_nsec, GPRClockType clock_type)
         {
             this.tv_sec = tv_sec;
             this.tv_nsec = tv_nsec;
             this.clock_type = clock_type;
         }
 
-        // NOTE: on linux 64bit  sizeof(gpr_timespec) = 16, on windows 32bit sizeof(gpr_timespec) = 8
-        // so IntPtr seems to have the right size to work on both.
-        private System.IntPtr tv_sec;
+        private long tv_sec;
         private int tv_nsec;
         private GPRClockType clock_type;
 
@@ -116,7 +114,7 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Seconds since unix epoch.
         /// </summary>
-        public IntPtr TimevalSeconds
+        public long TimevalSeconds
         {
             get
             {
@@ -176,18 +174,18 @@ namespace Grpc.Core.Internal
             {
                 // convert nanos to ticks, round up to the nearest tick
                 long ticksFromNanos = tv_nsec / NanosPerTick + ((tv_nsec % NanosPerTick != 0) ? 1 : 0);
-                long ticksTotal = checked(tv_sec.ToInt64() * TicksPerSecond + ticksFromNanos);
+                long ticksTotal = checked(tv_sec * TicksPerSecond + ticksFromNanos);
                 return UnixEpoch.AddTicks(ticksTotal);
             }
             catch (OverflowException)
             {
                 // ticks out of long range
-                return tv_sec.ToInt64() > 0 ? DateTime.MaxValue : DateTime.MinValue;
+                return tv_sec > 0 ? DateTime.MaxValue : DateTime.MinValue;
             }
             catch (ArgumentOutOfRangeException)
             {
                 // resulting date time would be larger than MaxValue
-                return tv_sec.ToInt64() > 0 ? DateTime.MaxValue : DateTime.MinValue;
+                return tv_sec > 0 ? DateTime.MaxValue : DateTime.MinValue;
             }
         }
             
@@ -226,12 +224,7 @@ namespace Grpc.Core.Internal
                     seconds--;
                     nanos += (int)NanosPerSecond;
                 }
-                // new IntPtr possibly throws OverflowException
-                return new Timespec(new IntPtr(seconds), nanos);
-            }
-            catch (OverflowException)
-            {
-                return dateTime > UnixEpoch ? Timespec.InfFuture : Timespec.InfPast;
+                return new Timespec(seconds, nanos);
             }
             catch (ArgumentOutOfRangeException)
             {
diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs
index 0fadabe554..d120f95fdf 100644
--- a/src/csharp/Grpc.Core/Server.cs
+++ b/src/csharp/Grpc.Core/Server.cs
@@ -171,6 +171,8 @@ namespace Grpc.Core
             handle.CancelAllCalls();
             await shutdownTcs.Task.ConfigureAwait(false);
             DisposeHandle();
+
+            await Task.Run(() => GrpcEnvironment.Release()).ConfigureAwait(false);
         }
 
         internal void AddCallReference(object call)
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
index 012de45524..b0d920a34c 100644
--- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
@@ -41,6 +41,9 @@
     <Reference Include="CommandLine">
       <HintPath>..\packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll</HintPath>
     </Reference>
+    <Reference Include="Moq">
+      <HintPath>..\packages\Moq.4.2.1510.2205\lib\net40\Moq.dll</HintPath>
+    </Reference>
     <Reference Include="nunit.framework">
       <HintPath>..\packages\NUnit.2.6.4\lib\nunit.framework.dll</HintPath>
     </Reference>
@@ -83,6 +86,7 @@
     <Compile Include="..\Grpc.Core\Version.cs">
       <Link>Version.cs</Link>
     </Compile>
+    <Compile Include="HeaderInterceptorTest.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Empty.cs" />
     <Compile Include="Messages.cs" />
diff --git a/src/csharp/Grpc.IntegrationTesting/HeaderInterceptorTest.cs b/src/csharp/Grpc.IntegrationTesting/HeaderInterceptorTest.cs
new file mode 100644
index 0000000000..1d758b7540
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting/HeaderInterceptorTest.cs
@@ -0,0 +1,113 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Grpc.Core.Utils;
+using Grpc.Testing;
+using NUnit.Framework;
+
+namespace Grpc.IntegrationTesting
+{
+    public class HeaderInterceptorTest
+    {
+        const string Host = "localhost";
+        Server server;
+        Channel channel;
+        TestService.TestServiceClient client;
+
+        [TestFixtureSetUp]
+        public void Init()
+        {
+            server = new Server
+            {
+                Services = { TestService.BindService(new TestServiceImpl()) },
+                Ports = { { Host, ServerPort.PickUnused, ServerCredentials.Insecure } }
+            };
+            server.Start();
+
+            channel = new Channel(Host, server.Ports.Single().BoundPort, ChannelCredentials.Insecure);
+            client = TestService.NewClient(channel);
+        }
+
+        [TestFixtureTearDown]
+        public void Cleanup()
+        {
+            channel.ShutdownAsync().Wait();
+            server.ShutdownAsync().Wait();
+        }
+
+        [Test]
+        public async Task HeaderInterceptor_CreateMetadata()
+        {
+            var key = "x-grpc-test-echo-initial";
+            client.HeaderInterceptor = new HeaderInterceptor((method, metadata) =>
+            {
+                metadata.Add(key, "ABC");
+            });
+
+            var call = client.UnaryCallAsync(new SimpleRequest());
+            await call;
+
+           var responseHeaders = await call.ResponseHeadersAsync;
+           Assert.AreEqual("ABC", responseHeaders.First((entry) => entry.Key == key).Value);
+        }
+
+        [Test]
+        public async Task HeaderInterceptor_AppendMetadata()
+        {
+            var initialKey = "x-grpc-test-echo-initial";
+            var trailingKey = "x-grpc-test-echo-trailing-bin";
+
+            client.HeaderInterceptor = new HeaderInterceptor((method, metadata) =>
+            {
+                metadata.Add(initialKey, "ABC");
+            });
+
+            var headers = new Metadata
+            {
+                { trailingKey, new byte[] {0xaa} }
+            };
+            var call = client.UnaryCallAsync(new SimpleRequest(), headers: headers);
+            await call;
+
+            var responseHeaders = await call.ResponseHeadersAsync;
+            Assert.AreEqual("ABC", responseHeaders.First((entry) => entry.Key == initialKey).Value);
+            CollectionAssert.AreEqual(new byte[] {0xaa}, call.GetTrailers().First((entry) => entry.Key == trailingKey).ValueBytes);
+        }
+    }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
index 5eec11abf7..b0e33e49f7 100644
--- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
+++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
@@ -34,6 +34,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -130,8 +131,7 @@ namespace Grpc.IntegrationTesting
                 };
             }
             var channel = new Channel(options.ServerHost, options.ServerPort, credentials, channelOptions);
-            TestService.TestServiceClient client = new TestService.TestServiceClient(channel);
-            await RunTestCaseAsync(client, options);
+            await RunTestCaseAsync(channel, options);
             await channel.ShutdownAsync();
         }
 
@@ -159,8 +159,9 @@ namespace Grpc.IntegrationTesting
             return credentials;
         }
 
-        private async Task RunTestCaseAsync(TestService.TestServiceClient client, ClientOptions options)
+        private async Task RunTestCaseAsync(Channel channel, ClientOptions options)
         {
+            var client = new TestService.TestServiceClient(channel);
             switch (options.TestCase)
             {
                 case "empty_unary":
@@ -202,8 +203,14 @@ namespace Grpc.IntegrationTesting
                 case "timeout_on_sleeping_server":
                     await RunTimeoutOnSleepingServerAsync(client);
                     break;
-                case "benchmark_empty_unary":
-                    RunBenchmarkEmptyUnary(client);
+                case "custom_metadata":
+                    await RunCustomMetadataAsync(client);
+                    break;
+                case "status_code_and_message":
+                    await RunStatusCodeAndMessageAsync(client);
+                    break;
+                case "unimplemented_method":
+                    RunUnimplementedMethod(new UnimplementedService.UnimplementedServiceClient(channel));
                     break;
                 default:
                     throw new ArgumentException("Unknown test case " + options.TestCase);
@@ -227,7 +234,6 @@ namespace Grpc.IntegrationTesting
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828)
             };
-
             var response = client.UnaryCall(request);
 
             Assert.AreEqual(PayloadType.COMPRESSABLE, response.Payload.Type);
@@ -493,11 +499,95 @@ namespace Grpc.IntegrationTesting
             Console.WriteLine("Passed!");
         }
 
-        // This is not an official interop test, but it's useful.
-        public static void RunBenchmarkEmptyUnary(TestService.ITestServiceClient client)
+        public static async Task RunCustomMetadataAsync(TestService.ITestServiceClient client)
         {
-            BenchmarkUtil.RunBenchmark(10000, 10000,
-                                       () => { client.EmptyCall(new Empty()); });
+            Console.WriteLine("running custom_metadata");
+            {
+                // step 1: test unary call
+                var request = new SimpleRequest
+                {
+                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseSize = 314159,
+                    Payload = CreateZerosPayload(271828)
+                };
+
+                var call = client.UnaryCallAsync(request, headers: CreateTestMetadata());
+                await call.ResponseAsync;
+
+                var responseHeaders = await call.ResponseHeadersAsync;
+                var responseTrailers = call.GetTrailers();
+
+                Assert.AreEqual("test_initial_metadata_value", responseHeaders.First((entry) => entry.Key == "x-grpc-test-echo-initial").Value);
+                CollectionAssert.AreEqual(new byte[] { 0xab, 0xab, 0xab }, responseTrailers.First((entry) => entry.Key == "x-grpc-test-echo-trailing-bin").ValueBytes);
+            }
+
+            {
+                // step 2: test full duplex call
+                var request = new StreamingOutputCallRequest
+                {
+                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseParameters = { new ResponseParameters { Size = 31415 } },
+                    Payload = CreateZerosPayload(27182)
+                };
+
+                var call = client.FullDuplexCall(headers: CreateTestMetadata());
+                var responseHeaders = await call.ResponseHeadersAsync;
+
+                await call.RequestStream.WriteAsync(request);
+                await call.RequestStream.CompleteAsync();
+                await call.ResponseStream.ToListAsync();
+
+                var responseTrailers = call.GetTrailers();
+
+                Assert.AreEqual("test_initial_metadata_value", responseHeaders.First((entry) => entry.Key == "x-grpc-test-echo-initial").Value);
+                CollectionAssert.AreEqual(new byte[] { 0xab, 0xab, 0xab }, responseTrailers.First((entry) => entry.Key == "x-grpc-test-echo-trailing-bin").ValueBytes);
+            }
+
+            Console.WriteLine("Passed!");
+        }
+
+        public static async Task RunStatusCodeAndMessageAsync(TestService.ITestServiceClient client)
+        {
+            Console.WriteLine("running status_code_and_message");
+            var echoStatus = new EchoStatus
+            {
+                Code = 2,
+                Message = "test status message"
+            };
+
+            {
+                // step 1: test unary call
+                var request = new SimpleRequest { ResponseStatus = echoStatus };
+
+                var e = Assert.Throws<RpcException>(() => client.UnaryCall(request));
+                Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
+                Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+            }
+
+            {
+                // step 2: test full duplex call
+                var request = new StreamingOutputCallRequest { ResponseStatus = echoStatus };
+
+                var call = client.FullDuplexCall();
+                await call.RequestStream.WriteAsync(request);
+                await call.RequestStream.CompleteAsync();
+
+                var e = Assert.Throws<RpcException>(async () => await call.ResponseStream.ToListAsync());
+                Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
+                Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+            }
+
+            Console.WriteLine("Passed!");
+        }
+
+        public static void RunUnimplementedMethod(UnimplementedService.IUnimplementedServiceClient client)
+        {
+            Console.WriteLine("running unimplemented_method");
+            var e = Assert.Throws<RpcException>(() => client.UnimplementedCall(new Empty()));
+
+            Assert.AreEqual(StatusCode.Unimplemented, e.Status.StatusCode);
+            Assert.AreEqual("", e.Status.Detail);
+            Console.WriteLine("Passed!");
         }
 
         private static Payload CreateZerosPayload(int size)
@@ -516,5 +606,14 @@ namespace Grpc.IntegrationTesting
             Assert.IsTrue(email.Length > 0);  // spec requires nonempty client email.
             return email;
         }
+
+        private static Metadata CreateTestMetadata()
+        {
+            return new Metadata
+            {
+                {"x-grpc-test-echo-initial", "test_initial_metadata_value"},
+                {"x-grpc-test-echo-trailing-bin", new byte[] {0xab, 0xab, 0xab}}
+            };
+        }
     }
 }
diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
index 837ae74c45..18168f9970 100644
--- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
@@ -128,9 +128,29 @@ namespace Grpc.IntegrationTesting
         }
 
         [Test]
-        public async Task TimeoutOnSleepingServerAsync()
+        public async Task TimeoutOnSleepingServer()
         {
             await InteropClient.RunTimeoutOnSleepingServerAsync(client);
         }
+
+        [Test]
+        public async Task CustomMetadata()
+        {
+            await InteropClient.RunCustomMetadataAsync(client);
+        }
+
+        [Test]
+        [Ignore("TODO: see #4427")]
+        public async Task StatusCodeAndMessage()
+        {
+            await InteropClient.RunStatusCodeAndMessageAsync(client);
+        }
+
+        [Test]
+        [Ignore("TODO: see #4427")]
+        public void UnimplementedMethod()
+        {
+            InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel));
+        }
     }
 }
diff --git a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
index 35230f48c1..1c8bfed1f6 100644
--- a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
@@ -40,6 +40,7 @@ using System.Threading.Tasks;
 using Grpc.Core;
 using Grpc.Core.Utils;
 using Grpc.Testing;
+using Moq;
 using NUnit.Framework;
 
 namespace Grpc.IntegrationTesting
@@ -50,37 +51,37 @@ namespace Grpc.IntegrationTesting
         Server server;
         Channel channel;
         TestService.ITestServiceClient client;
+        List<ChannelOption> options;
+        Mock<TestService.ITestService> serviceMock;
+        AsyncAuthInterceptor asyncAuthInterceptor;
 
-        [TestFixtureSetUp]
+        [SetUp]
         public void Init()
         {
-            var serverCredentials = new SslServerCredentials(new[] { new KeyCertificatePair(File.ReadAllText(TestCredentials.ServerCertChainPath), File.ReadAllText(TestCredentials.ServerPrivateKeyPath)) });
+            serviceMock = new Mock<TestService.ITestService>();
+            serviceMock.Setup(m => m.UnaryCall(It.IsAny<SimpleRequest>(), It.IsAny<ServerCallContext>()))
+                .Returns(new Func<SimpleRequest, ServerCallContext, Task<SimpleResponse>>(UnaryCallHandler));
+
             server = new Server
             {
-                Services = { TestService.BindService(new TestServiceImpl()) },
-                Ports = { { Host, ServerPort.PickUnused, serverCredentials } }
+                Services = { TestService.BindService(serviceMock.Object) },
+                Ports = { { Host, ServerPort.PickUnused, TestCredentials.CreateSslServerCredentials() } }
             };
             server.Start();
 
-            var options = new List<ChannelOption>
+            options = new List<ChannelOption>
             {
                 new ChannelOption(ChannelOptions.SslTargetNameOverride, TestCredentials.DefaultHostOverride)
             };
 
-            var asyncAuthInterceptor = new AsyncAuthInterceptor(async (context, metadata) =>
+            asyncAuthInterceptor = new AsyncAuthInterceptor(async (context, metadata) =>
             {
-                await Task.Delay(100);  // make sure the operation is asynchronous.
+                await Task.Delay(100).ConfigureAwait(false);  // make sure the operation is asynchronous.
                 metadata.Add("authorization", "SECRET_TOKEN");
             });
-
-            var clientCredentials = ChannelCredentials.Create(
-                new SslCredentials(File.ReadAllText(TestCredentials.ClientCertAuthorityPath)),
-                CallCredentials.FromInterceptor(asyncAuthInterceptor));
-            channel = new Channel(Host, server.Ports.Single().BoundPort, clientCredentials, options);
-            client = TestService.NewClient(channel);
         }
 
-        [TestFixtureTearDown]
+        [TearDown]
         public void Cleanup()
         {
             channel.ShutdownAsync().Wait();
@@ -90,8 +91,29 @@ namespace Grpc.IntegrationTesting
         [Test]
         public void MetadataCredentials()
         {
-            var response = client.UnaryCall(new SimpleRequest { ResponseSize = 10 });
-            Assert.AreEqual(10, response.Payload.Body.Length);
+            var channelCredentials = ChannelCredentials.Create(TestCredentials.CreateSslCredentials(),
+                CallCredentials.FromInterceptor(asyncAuthInterceptor));
+            channel = new Channel(Host, server.Ports.Single().BoundPort, channelCredentials, options);
+            client = TestService.NewClient(channel);
+
+            client.UnaryCall(new SimpleRequest {});
+        }
+
+        [Test]
+        public void MetadataCredentials_PerCall()
+        {
+            channel = new Channel(Host, server.Ports.Single().BoundPort, TestCredentials.CreateSslCredentials(), options);
+            client = TestService.NewClient(channel);
+
+            var callCredentials = CallCredentials.FromInterceptor(asyncAuthInterceptor);
+            client.UnaryCall(new SimpleRequest { }, new CallOptions(credentials: callCredentials));
+        }
+
+        private Task<SimpleResponse> UnaryCallHandler(SimpleRequest request, ServerCallContext context)
+        {
+            var authToken = context.RequestHeaders.First((entry) => entry.Key == "authorization").Value;
+            Assert.AreEqual("SECRET_TOKEN", authToken);
+            return Task.FromResult(new SimpleResponse());
         }
     }
 }
diff --git a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
index 2b51526c88..3dd91b7948 100644
--- a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
@@ -75,9 +75,10 @@ namespace Grpc.IntegrationTesting
             serverRunner.StopAsync().Wait();
         }
 
-        // Test attribute commented out to prevent running as part of the default test suite.
-        //[Test]
-        //[Category("Performance")]
+
+        [Test]
+        [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public async Task ClientServerRunner()
         {
             var config = new ClientConfig
diff --git a/src/csharp/Grpc.IntegrationTesting/TestServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/TestServiceImpl.cs
index c5bfcf08c0..5a1b4cf319 100644
--- a/src/csharp/Grpc.IntegrationTesting/TestServiceImpl.cs
+++ b/src/csharp/Grpc.IntegrationTesting/TestServiceImpl.cs
@@ -33,6 +33,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Google.Protobuf;
@@ -51,14 +52,20 @@ namespace Grpc.Testing
             return Task.FromResult(new Empty());
         }
 
-        public Task<SimpleResponse> UnaryCall(SimpleRequest request, ServerCallContext context)
+        public async Task<SimpleResponse> UnaryCall(SimpleRequest request, ServerCallContext context)
         {
+            await EnsureEchoMetadataAsync(context);
+            EnsureEchoStatus(request.ResponseStatus, context);
+
             var response = new SimpleResponse { Payload = CreateZerosPayload(request.ResponseSize) };
-            return Task.FromResult(response);
+            return response;
         }
 
         public async Task StreamingOutputCall(StreamingOutputCallRequest request, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
         {
+            await EnsureEchoMetadataAsync(context);
+            EnsureEchoStatus(request.ResponseStatus, context);
+
             foreach (var responseParam in request.ResponseParameters)
             {
                 var response = new StreamingOutputCallResponse { Payload = CreateZerosPayload(responseParam.Size) };
@@ -68,6 +75,8 @@ namespace Grpc.Testing
 
         public async Task<StreamingInputCallResponse> StreamingInputCall(IAsyncStreamReader<StreamingInputCallRequest> requestStream, ServerCallContext context)
         {
+            await EnsureEchoMetadataAsync(context);
+
             int sum = 0;
             await requestStream.ForEachAsync(async request =>
             {
@@ -78,8 +87,11 @@ namespace Grpc.Testing
 
         public async Task FullDuplexCall(IAsyncStreamReader<StreamingOutputCallRequest> requestStream, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
         {
+            await EnsureEchoMetadataAsync(context);
+
             await requestStream.ForEachAsync(async request =>
             {
+                EnsureEchoStatus(request.ResponseStatus, context);
                 foreach (var responseParam in request.ResponseParameters)
                 {
                     var response = new StreamingOutputCallResponse { Payload = CreateZerosPayload(responseParam.Size) };
@@ -97,5 +109,28 @@ namespace Grpc.Testing
         {
             return new Payload { Body = ByteString.CopyFrom(new byte[size]) };
         }
+
+        private static async Task EnsureEchoMetadataAsync(ServerCallContext context)
+        {
+            var echoInitialList = context.RequestHeaders.Where((entry) => entry.Key == "x-grpc-test-echo-initial").ToList();
+            if (echoInitialList.Any()) {
+                var entry = echoInitialList.Single();
+                await context.WriteResponseHeadersAsync(new Metadata { entry });
+            }
+
+            var echoTrailingList = context.RequestHeaders.Where((entry) => entry.Key == "x-grpc-test-echo-trailing-bin").ToList();
+            if (echoTrailingList.Any()) {
+                context.ResponseTrailers.Add(echoTrailingList.Single());
+            }
+        }
+
+        private static void EnsureEchoStatus(EchoStatus responseStatus, ServerCallContext context)
+        {
+            if (responseStatus != null)
+            {
+                var statusCode = (StatusCode)responseStatus.Code;
+                context.Status = new Status(statusCode, responseStatus.Message);
+            }
+        }
     }
 }
diff --git a/src/csharp/Grpc.IntegrationTesting/packages.config b/src/csharp/Grpc.IntegrationTesting/packages.config
index bdb3dadf44..5c59af1b7d 100644
--- a/src/csharp/Grpc.IntegrationTesting/packages.config
+++ b/src/csharp/Grpc.IntegrationTesting/packages.config
@@ -11,6 +11,7 @@
   <package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
   <package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
   <package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net45" />
+  <package id="Moq" version="4.2.1510.2205" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
   <package id="NUnit" version="2.6.4" targetFramework="net45" />
 </packages>
\ No newline at end of file
diff --git a/src/csharp/tests.json b/src/csharp/tests.json
new file mode 100644
index 0000000000..4aa93668ad
--- /dev/null
+++ b/src/csharp/tests.json
@@ -0,0 +1,45 @@
+{
+  "assemblies": [
+    "Grpc.Core.Tests",
+    "Grpc.Examples.Tests",
+    "Grpc.HealthCheck.Tests",
+    "Grpc.IntegrationTesting"
+  ],
+  "tests": [
+    "Grpc.Core.Internal.Tests.AsyncCallTest",
+    "Grpc.Core.Internal.Tests.ChannelArgsSafeHandleTest",
+    "Grpc.Core.Internal.Tests.CompletionQueueEventTest",
+    "Grpc.Core.Internal.Tests.CompletionQueueSafeHandleTest",
+    "Grpc.Core.Internal.Tests.MetadataArraySafeHandleTest",
+    "Grpc.Core.Internal.Tests.TimespecTest",
+    "Grpc.Core.Tests.CallCredentialsTest",
+    "Grpc.Core.Tests.CallOptionsTest",
+    "Grpc.Core.Tests.ChannelCredentialsTest",
+    "Grpc.Core.Tests.ChannelOptionsTest",
+    "Grpc.Core.Tests.ChannelTest",
+    "Grpc.Core.Tests.ClientServerTest",
+    "Grpc.Core.Tests.CompressionTest",
+    "Grpc.Core.Tests.ContextPropagationTest",
+    "Grpc.Core.Tests.GrpcEnvironmentTest",
+    "Grpc.Core.Tests.MarshallingErrorsTest",
+    "Grpc.Core.Tests.MetadataTest",
+    "Grpc.Core.Tests.NUnitVersionTest",
+    "Grpc.Core.Tests.PerformanceTest",
+    "Grpc.Core.Tests.PInvokeTest",
+    "Grpc.Core.Tests.ResponseHeadersTest",
+    "Grpc.Core.Tests.SanityTest",
+    "Grpc.Core.Tests.ServerTest",
+    "Grpc.Core.Tests.ShutdownTest",
+    "Grpc.Core.Tests.TimeoutsTest",
+    "Grpc.Core.Tests.UserAgentStringTest",
+    "Math.Tests.MathClientServerTest",
+    "Grpc.HealthCheck.Tests.HealthClientServerTest",
+    "Grpc.HealthCheck.Tests.HealthServiceImplTest",
+    "Grpc.IntegrationTesting.HeaderInterceptorTest",
+    "Grpc.IntegrationTesting.HistogramTest",
+    "Grpc.IntegrationTesting.InteropClientServerTest",
+    "Grpc.IntegrationTesting.MetadataCredentialsTest",
+    "Grpc.IntegrationTesting.RunnerClientServerTest",
+    "Grpc.IntegrationTesting.SslCredentialsTest"
+  ]
+}
\ No newline at end of file
diff --git a/src/node/ext/channel_credentials.cc b/src/node/ext/channel_credentials.cc
index b77ff80af2..059bc8a890 100644
--- a/src/node/ext/channel_credentials.cc
+++ b/src/node/ext/channel_credentials.cc
@@ -154,6 +154,12 @@ NAN_METHOD(ChannelCredentials::CreateSsl) {
     return Nan::ThrowTypeError(
         "createSSl's third argument must be a Buffer if provided");
   }
+  if ((key_cert_pair.private_key == NULL) !=
+      (key_cert_pair.cert_chain == NULL)) {
+    return Nan::ThrowError(
+        "createSsl's second and third arguments must be"
+        " provided or omitted together");
+  }
   grpc_channel_credentials *creds = grpc_ssl_credentials_create(
       root_certs, key_cert_pair.private_key == NULL ? NULL : &key_cert_pair,
       NULL);
diff --git a/src/node/ext/server_credentials.cc b/src/node/ext/server_credentials.cc
index e6c55e263c..5285d53df4 100644
--- a/src/node/ext/server_credentials.cc
+++ b/src/node/ext/server_credentials.cc
@@ -167,30 +167,18 @@ NAN_METHOD(ServerCredentials::CreateSsl) {
       return Nan::ThrowTypeError("Key/cert pairs must be objects");
     }
     Local<Object> pair_obj = Nan::To<Object>(pair_val).ToLocalChecked();
-    MaybeLocal<Value> maybe_key = Nan::Get(pair_obj, key_key);
-    if (maybe_key.IsEmpty()) {
-      delete key_cert_pairs;
-      return Nan::ThrowTypeError(
-          "Key/cert pairs must have a private_key and a cert_chain");
-    }
-    MaybeLocal<Value> maybe_cert = Nan::Get(pair_obj, cert_key);
-    if (maybe_cert.IsEmpty()) {
-      delete key_cert_pairs;
-      return Nan::ThrowTypeError(
-          "Key/cert pairs must have a private_key and a cert_chain");
-    }
-    if (!::node::Buffer::HasInstance(maybe_key.ToLocalChecked())) {
+    Local<Value> maybe_key = Nan::Get(pair_obj, key_key).ToLocalChecked();
+    Local<Value> maybe_cert = Nan::Get(pair_obj, cert_key).ToLocalChecked();
+    if (!::node::Buffer::HasInstance(maybe_key)) {
       delete key_cert_pairs;
       return Nan::ThrowTypeError("private_key must be a Buffer");
     }
-    if (!::node::Buffer::HasInstance(maybe_cert.ToLocalChecked())) {
+    if (!::node::Buffer::HasInstance(maybe_cert)) {
       delete key_cert_pairs;
       return Nan::ThrowTypeError("cert_chain must be a Buffer");
     }
-    key_cert_pairs[i].private_key = ::node::Buffer::Data(
-        maybe_key.ToLocalChecked());
-    key_cert_pairs[i].cert_chain = ::node::Buffer::Data(
-        maybe_cert.ToLocalChecked());
+    key_cert_pairs[i].private_key = ::node::Buffer::Data(maybe_key);
+    key_cert_pairs[i].cert_chain = ::node::Buffer::Data(maybe_cert);
   }
   grpc_server_credentials *creds = grpc_ssl_server_credentials_create(
       root_certs, key_cert_pairs, key_cert_pair_count, force_client_auth, NULL);
diff --git a/src/node/interop/async_delay_queue.js b/src/node/interop/async_delay_queue.js
new file mode 100644
index 0000000000..2bd3ca4da3
--- /dev/null
+++ b/src/node/interop/async_delay_queue.js
@@ -0,0 +1,79 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+'use strict';
+
+var _ = require('lodash');
+
+/**
+ * This class represents a queue of callbacks that must happen sequentially, each
+ * with a specific delay after the previous event.
+ */
+function AsyncDelayQueue() {
+  this.queue = [];
+
+  this.callback_pending = false;
+}
+
+/**
+ * Run the next callback after its corresponding delay, if there are any
+ * remaining.
+ */
+AsyncDelayQueue.prototype.runNext = function() {
+  var next = this.queue.shift();
+  var continueCallback = _.bind(this.runNext, this);
+  if (next) {
+    this.callback_pending = true;
+    setTimeout(function() {
+      next.callback(continueCallback);
+    }, next.delay);
+  } else {
+    this.callback_pending = false;
+  }
+};
+
+/**
+ * Add a callback to be called with a specific delay after now or after the
+ * current last item in the queue or current pending callback, whichever is
+ * latest.
+ * @param {function(function())} callback The callback
+ * @param {Number} The delay to apply, in milliseconds
+ */
+AsyncDelayQueue.prototype.add = function(callback, delay) {
+  this.queue.push({callback: callback, delay: delay});
+  if (!this.callback_pending) {
+    this.runNext();
+  }
+};
+
+module.exports = AsyncDelayQueue;
diff --git a/src/node/interop/interop_server.js b/src/node/interop/interop_server.js
index 5321005c86..9526b5d183 100644
--- a/src/node/interop/interop_server.js
+++ b/src/node/interop/interop_server.js
@@ -36,6 +36,7 @@
 var fs = require('fs');
 var path = require('path');
 var _ = require('lodash');
+var AsyncDelayQueue = require('./async_delay_queue');
 var grpc = require('..');
 var testProto = grpc.load({
   root: __dirname + '/../../..',
@@ -155,6 +156,7 @@ function handleStreamingInput(call, callback) {
  */
 function handleStreamingOutput(call) {
   echoHeader(call);
+  var delay_queue = new AsyncDelayQueue();
   var req = call.request;
   if (req.response_status) {
     var status = req.response_status;
@@ -163,9 +165,15 @@ function handleStreamingOutput(call) {
     return;
   }
   _.each(req.response_parameters, function(resp_param) {
-    call.write({payload: getPayload(req.response_type, resp_param.size)});
+    delay_queue.add(function(next) {
+      call.write({payload: getPayload(req.response_type, resp_param.size)});
+      next();
+    }, resp_param.interval_us);
+  });
+  delay_queue.add(function(next) {
+    call.end(getEchoTrailer(call));
+    next();
   });
-  call.end(getEchoTrailer(call));
 }
 
 /**
@@ -175,6 +183,7 @@ function handleStreamingOutput(call) {
  */
 function handleFullDuplex(call) {
   echoHeader(call);
+  var delay_queue = new AsyncDelayQueue();
   call.on('data', function(value) {
     if (value.response_status) {
       var status = value.response_status;
@@ -183,11 +192,17 @@ function handleFullDuplex(call) {
       return;
     }
     _.each(value.response_parameters, function(resp_param) {
-      call.write({payload: getPayload(value.response_type, resp_param.size)});
+      delay_queue.add(function(next) {
+        call.write({payload: getPayload(value.response_type, resp_param.size)});
+        next();
+      }, resp_param.interval_us);
     });
   });
   call.on('end', function() {
-    call.end(getEchoTrailer(call));
+    delay_queue.add(function(next) {
+      call.end(getEchoTrailer(call));
+      next();
+    });
   });
 }
 
diff --git a/src/node/test/credentials_test.js b/src/node/test/credentials_test.js
index 647f648ca9..294600c85a 100644
--- a/src/node/test/credentials_test.js
+++ b/src/node/test/credentials_test.js
@@ -76,6 +76,146 @@ var fakeFailingGoogleCredentials = {
   }
 };
 
+var key_data, pem_data, ca_data;
+
+before(function() {
+  var key_path = path.join(__dirname, './data/server1.key');
+  var pem_path = path.join(__dirname, './data/server1.pem');
+  var ca_path = path.join(__dirname, '../test/data/ca.pem');
+  key_data = fs.readFileSync(key_path);
+  pem_data = fs.readFileSync(pem_path);
+  ca_data = fs.readFileSync(ca_path);
+});
+
+describe('channel credentials', function() {
+  describe('#createSsl', function() {
+    it('works with no arguments', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.credentials.createSsl();
+      });
+      assert.notEqual(creds, null);
+    });
+    it('works with just one Buffer argument', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.credentials.createSsl(ca_data);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('works with 3 Buffer arguments', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.credentials.createSsl(ca_data, key_data, pem_data);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('works if the first argument is null', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.credentials.createSsl(null, key_data, pem_data);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('fails if the first argument is a non-Buffer value', function() {
+      assert.throws(function() {
+        grpc.credentials.createSsl('test');
+      }, TypeError);
+    });
+    it('fails if the second argument is a non-Buffer value', function() {
+      assert.throws(function() {
+        grpc.credentials.createSsl(null, 'test', pem_data);
+      }, TypeError);
+    });
+    it('fails if the third argument is a non-Buffer value', function() {
+      assert.throws(function() {
+        grpc.credentials.createSsl(null, key_data, 'test');
+      }, TypeError);
+    });
+    it('fails if only 1 of the last 2 arguments is provided', function() {
+      assert.throws(function() {
+        grpc.credentials.createSsl(null, key_data);
+      });
+      assert.throws(function() {
+        grpc.credentials.createSsl(null, null, pem_data);
+      });
+    });
+  });
+});
+
+describe('server credentials', function() {
+  describe('#createSsl', function() {
+    it('accepts a buffer and array as the first 2 arguments', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.ServerCredentials.createSsl(ca_data, []);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('accepts a boolean as the third argument', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.ServerCredentials.createSsl(ca_data, [], true);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('accepts an object with two buffers in the second argument', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.ServerCredentials.createSsl(null,
+                                                 [{private_key: key_data,
+                                                   cert_chain: pem_data}]);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('accepts multiple objects in the second argument', function() {
+      var creds;
+      assert.doesNotThrow(function() {
+        creds = grpc.ServerCredentials.createSsl(null,
+                                                 [{private_key: key_data,
+                                                   cert_chain: pem_data},
+                                                  {private_key: key_data,
+                                                   cert_chain: pem_data}]);
+      });
+      assert.notEqual(creds, null);
+    });
+    it('fails if the second argument is not an Array', function() {
+      assert.throws(function() {
+        grpc.ServerCredentials.createSsl(ca_data, 'test');
+      }, TypeError);
+    });
+    it('fails if the first argument is a non-Buffer value', function() {
+      assert.throws(function() {
+        grpc.ServerCredentials.createSsl('test', []);
+      }, TypeError);
+    });
+    it('fails if the third argument is a non-boolean value', function() {
+      assert.throws(function() {
+        grpc.ServerCredentials.createSsl(ca_data, [], 'test');
+      }, TypeError);
+    });
+    it('fails if the array elements are not objects', function() {
+      assert.throws(function() {
+        grpc.ServerCredentials.createSsl(ca_data, 'test');
+      }, TypeError);
+    });
+    it('fails if the object does not have a Buffer private_key', function() {
+      assert.throws(function() {
+        grpc.ServerCredentials.createSsl(null,
+                                         [{private_key: 'test',
+                                           cert_chain: pem_data}]);
+      }, TypeError);
+    });
+    it('fails if the object does not have a Buffer cert_chain', function() {
+      assert.throws(function() {
+        grpc.ServerCredentials.createSsl(null,
+                                         [{private_key: key_data,
+                                           cert_chain: 'test'}]);
+      }, TypeError);
+    });
+  });
+});
+
 describe('client credentials', function() {
   var Client;
   var server;
@@ -109,20 +249,13 @@ describe('client credentials', function() {
         });
       }
     });
-    var key_path = path.join(__dirname, './data/server1.key');
-    var pem_path = path.join(__dirname, './data/server1.pem');
-    var key_data = fs.readFileSync(key_path);
-    var pem_data = fs.readFileSync(pem_path);
     var creds = grpc.ServerCredentials.createSsl(null,
                                                  [{private_key: key_data,
                                                    cert_chain: pem_data}]);
-    //creds = grpc.ServerCredentials.createInsecure();
     port = server.bind('localhost:0', creds);
     server.start();
 
     Client = proto.TestService;
-    var ca_path = path.join(__dirname, '../test/data/ca.pem');
-    var ca_data = fs.readFileSync(ca_path);
     client_ssl_creds = grpc.credentials.createSsl(ca_data);
     var host_override = 'foo.test.google.fr';
     client_options['grpc.ssl_target_name_override'] = host_override;
diff --git a/src/node/test/server_test.js b/src/node/test/server_test.js
index 999a183b3c..592f47e145 100644
--- a/src/node/test/server_test.js
+++ b/src/node/test/server_test.js
@@ -45,9 +45,30 @@ describe('server', function() {
         new grpc.Server();
       });
     });
-    it('should work with an empty list argument', function() {
+    it('should work with an empty object argument', function() {
       assert.doesNotThrow(function() {
-        new grpc.Server([]);
+        new grpc.Server({});
+      });
+    });
+    it('should work without the new keyword', function() {
+      var server;
+      assert.doesNotThrow(function() {
+        server = grpc.Server();
+      });
+      assert(server instanceof grpc.Server);
+    });
+    it('should only accept objects with string or int values', function() {
+      assert.doesNotThrow(function() {
+        new grpc.Server({'key' : 'value'});
+      });
+      assert.doesNotThrow(function() {
+        new grpc.Server({'key' : 5});
+      });
+      assert.throws(function() {
+        new grpc.Server({'key' : null});
+      });
+      assert.throws(function() {
+        new grpc.Server({'key' : new Date()});
       });
     });
   });
diff --git a/src/php/bin/run_tests.sh b/src/php/bin/run_tests.sh
index 2a7335e20a..235d161ca7 100755
--- a/src/php/bin/run_tests.sh
+++ b/src/php/bin/run_tests.sh
@@ -30,7 +30,7 @@
 
 # Loads the local shared library, and runs all of the test cases in tests/
 # against it
-set -e
+set -ex
 cd $(dirname $0)/../../..
 root=$(pwd)
 cd src/php/bin
diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c
index 3b99de7538..7ba14a38d8 100644
--- a/src/php/ext/grpc/call.c
+++ b/src/php/ext/grpc/call.c
@@ -42,13 +42,13 @@
 #include <ext/standard/info.h>
 #include <ext/spl/spl_exceptions.h>
 #include "php_grpc.h"
+#include "call_credentials.h"
 
 #include <zend_exceptions.h>
 #include <zend_hash.h>
 
 #include <stdbool.h>
 
-#include <grpc/support/log.h>
 #include <grpc/support/alloc.h>
 #include <grpc/grpc.h>
 
@@ -515,11 +515,41 @@ PHP_METHOD(Call, cancel) {
   grpc_call_cancel(call->wrapped, NULL);
 }
 
+/**
+ * Set the CallCredentials for this call.
+ * @param CallCredentials creds_obj The CallCredentials object
+ * @param int The error code
+ */
+PHP_METHOD(Call, setCredentials) {
+  zval *creds_obj;
+
+  /* "O" == 1 Object */
+  if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "O", &creds_obj,
+                            grpc_ce_call_credentials) == FAILURE) {
+    zend_throw_exception(spl_ce_InvalidArgumentException,
+                         "setCredentials expects 1 CallCredentials",
+                         1 TSRMLS_CC);
+    return;
+  }
+
+  wrapped_grpc_call_credentials *creds =
+      (wrapped_grpc_call_credentials *)zend_object_store_get_object(
+          creds_obj TSRMLS_CC);
+
+  wrapped_grpc_call *call =
+      (wrapped_grpc_call *)zend_object_store_get_object(getThis() TSRMLS_CC);
+
+  grpc_call_error error = GRPC_CALL_ERROR;
+  error = grpc_call_set_credentials(call->wrapped, creds->wrapped);
+  RETURN_LONG(error);
+}
+
 static zend_function_entry call_methods[] = {
     PHP_ME(Call, __construct, NULL, ZEND_ACC_PUBLIC | ZEND_ACC_CTOR)
     PHP_ME(Call, startBatch, NULL, ZEND_ACC_PUBLIC)
     PHP_ME(Call, getPeer, NULL, ZEND_ACC_PUBLIC)
     PHP_ME(Call, cancel, NULL, ZEND_ACC_PUBLIC)
+    PHP_ME(Call, setCredentials, NULL, ZEND_ACC_PUBLIC)
     PHP_FE_END};
 
 void grpc_init_call(TSRMLS_D) {
diff --git a/src/php/ext/grpc/call.h b/src/php/ext/grpc/call.h
index f3ef89dc97..73efadae35 100644
--- a/src/php/ext/grpc/call.h
+++ b/src/php/ext/grpc/call.h
@@ -66,4 +66,8 @@ zval *grpc_php_wrap_call(grpc_call *wrapped, bool owned);
  * call metadata */
 zval *grpc_parse_metadata_array(grpc_metadata_array *metadata_array);
 
+/* Populates a grpc_metadata_array with the data in a PHP array object.
+   Returns true on success and false on failure */
+bool create_metadata_array(zval *array, grpc_metadata_array *metadata);
+
 #endif /* NET_GRPC_PHP_GRPC_CHANNEL_H_ */
diff --git a/src/php/ext/grpc/call_credentials.c b/src/php/ext/grpc/call_credentials.c
index 17f167befb..285c4e7c85 100644
--- a/src/php/ext/grpc/call_credentials.c
+++ b/src/php/ext/grpc/call_credentials.c
@@ -43,6 +43,7 @@
 #include <ext/standard/info.h>
 #include <ext/spl/spl_exceptions.h>
 #include "php_grpc.h"
+#include "call.h"
 
 #include <zend_exceptions.h>
 #include <zend_hash.h>
@@ -103,7 +104,7 @@ PHP_METHOD(CallCredentials, createComposite) {
   zval *cred1_obj;
   zval *cred2_obj;
 
-  /* "OO" == 3 Objects */
+  /* "OO" == 2 Objects */
   if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "OO", &cred1_obj,
                             grpc_ce_call_credentials, &cred2_obj,
                             grpc_ce_call_credentials) == FAILURE) {
@@ -125,9 +126,107 @@ PHP_METHOD(CallCredentials, createComposite) {
   RETURN_DESTROY_ZVAL(creds_object);
 }
 
+/**
+ * Create a call credentials object from the plugin API
+ * @param function callback The callback function
+ * @return CallCredentials The new call credentials object
+ */
+PHP_METHOD(CallCredentials, createFromPlugin) {
+  zend_fcall_info *fci;
+  zend_fcall_info_cache *fci_cache;
+
+  fci = (zend_fcall_info *)emalloc(sizeof(zend_fcall_info));
+  fci_cache = (zend_fcall_info_cache *)emalloc(sizeof(zend_fcall_info_cache));
+  memset(fci, 0, sizeof(zend_fcall_info));
+  memset(fci_cache, 0, sizeof(zend_fcall_info_cache));
+
+  /* "f" == 1 function */
+  if (zend_parse_parameters(ZEND_NUM_ARGS(), "f", fci,
+                            fci_cache,
+                            fci->params,
+                            fci->param_count) == FAILURE) {
+    zend_throw_exception(spl_ce_InvalidArgumentException,
+                         "createFromPlugin expects 1 callback",
+                         1 TSRMLS_CC);
+    return;
+  }
+
+  plugin_state *state;
+  state = (plugin_state *)emalloc(sizeof(plugin_state));
+  memset(state, 0, sizeof(plugin_state));
+
+  /* save the user provided PHP callback function */
+  state->fci = fci;
+  state->fci_cache = fci_cache;
+
+  grpc_metadata_credentials_plugin plugin;
+  plugin.get_metadata = plugin_get_metadata;
+  plugin.destroy = plugin_destroy_state;
+  plugin.state = (void *)state;
+  plugin.type = "";
+
+  grpc_call_credentials *creds = grpc_metadata_credentials_create_from_plugin(
+      plugin, NULL);
+  zval *creds_object = grpc_php_wrap_call_credentials(creds);
+  RETURN_DESTROY_ZVAL(creds_object);
+}
+
+/* Callback function for plugin creds API */
+void plugin_get_metadata(void *ptr, grpc_auth_metadata_context context,
+                         grpc_credentials_plugin_metadata_cb cb,
+                         void *user_data) {
+  plugin_state *state = (plugin_state *)ptr;
+
+  /* prepare to call the user callback function with info from the
+   * grpc_auth_metadata_context */
+  zval **params[1];
+  zval *arg;
+  zval *retval;
+  MAKE_STD_ZVAL(arg);
+  object_init(arg);
+  add_property_string(arg, "service_url", context.service_url, true);
+  add_property_string(arg, "method_name", context.method_name, true);
+  params[0] = &arg;
+  state->fci->param_count = 1;
+  state->fci->params = params;
+  state->fci->retval_ptr_ptr = &retval;
+
+  /* call the user callback function */
+  zend_call_function(state->fci, state->fci_cache);
+
+  if (Z_TYPE_P(retval) != IS_ARRAY) {
+    zend_throw_exception(spl_ce_InvalidArgumentException,
+                         "plugin callback must return metadata array",
+                         1 TSRMLS_CC);
+  }
+
+  grpc_metadata_array metadata;
+  if (!create_metadata_array(retval, &metadata)) {
+    zend_throw_exception(spl_ce_InvalidArgumentException,
+                         "invalid metadata", 1 TSRMLS_CC);
+    grpc_metadata_array_destroy(&metadata);
+  }
+
+  /* TODO: handle error */
+  grpc_status_code code = GRPC_STATUS_OK;
+
+  /* Pass control back to core */
+  cb(user_data, metadata.metadata, metadata.count, code, NULL);
+}
+
+/* Cleanup function for plugin creds API */
+void plugin_destroy_state(void *ptr) {
+  plugin_state *state = (plugin_state *)ptr;
+  efree(state->fci);
+  efree(state->fci_cache);
+  efree(state);
+}
+
 static zend_function_entry call_credentials_methods[] = {
   PHP_ME(CallCredentials, createComposite, NULL,
          ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
+  PHP_ME(CallCredentials, createFromPlugin, NULL,
+         ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
   PHP_FE_END};
 
 void grpc_init_call_credentials(TSRMLS_D) {
diff --git a/src/php/ext/grpc/call_credentials.h b/src/php/ext/grpc/call_credentials.h
index 8f35ac68bc..d2f6a92449 100755
--- a/src/php/ext/grpc/call_credentials.h
+++ b/src/php/ext/grpc/call_credentials.h
@@ -57,6 +57,20 @@ typedef struct wrapped_grpc_call_credentials {
   grpc_call_credentials *wrapped;
 } wrapped_grpc_call_credentials;
 
+/* Struct to hold callback function for plugin creds API */
+typedef struct plugin_state {
+  zend_fcall_info *fci;
+  zend_fcall_info_cache *fci_cache;
+} plugin_state;
+
+/* Callback function for plugin creds API */
+void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
+                         grpc_credentials_plugin_metadata_cb cb,
+                         void *user_data);
+
+/* Cleanup function for plugin creds API */
+void plugin_destroy_state(void *ptr);
+
 /* Initializes the CallCredentials PHP class */
 void grpc_init_call_credentials(TSRMLS_D);
 
diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c
index f8c4f0423f..60c94412dc 100644
--- a/src/php/ext/grpc/channel.c
+++ b/src/php/ext/grpc/channel.c
@@ -154,16 +154,20 @@ PHP_METHOD(Channel, __construct) {
     array_hash = Z_ARRVAL_P(args_array);
     if (zend_hash_find(array_hash, "credentials", sizeof("credentials"),
                        (void **)&creds_obj) == SUCCESS) {
-      if (zend_get_class_entry(*creds_obj TSRMLS_CC) !=
+      if (Z_TYPE_P(*creds_obj) == IS_NULL) {
+        creds = NULL;
+        zend_hash_del(array_hash, "credentials", 12);
+      } else if (zend_get_class_entry(*creds_obj TSRMLS_CC) !=
           grpc_ce_channel_credentials) {
         zend_throw_exception(spl_ce_InvalidArgumentException,
                              "credentials must be a ChannelCredentials object",
                              1 TSRMLS_CC);
         return;
+      } else {
+        creds = (wrapped_grpc_channel_credentials *)zend_object_store_get_object(
+            *creds_obj TSRMLS_CC);
+        zend_hash_del(array_hash, "credentials", 12);
       }
-      creds = (wrapped_grpc_channel_credentials *)zend_object_store_get_object(
-          *creds_obj TSRMLS_CC);
-      zend_hash_del(array_hash, "credentials", 12);
     }
     php_grpc_read_args_array(args_array, &args);
     if (creds == NULL) {
diff --git a/src/php/ext/grpc/channel_credentials.c b/src/php/ext/grpc/channel_credentials.c
index df4a5d5162..ae9a9897fc 100644
--- a/src/php/ext/grpc/channel_credentials.c
+++ b/src/php/ext/grpc/channel_credentials.c
@@ -169,6 +169,14 @@ PHP_METHOD(ChannelCredentials, createComposite) {
   RETURN_DESTROY_ZVAL(creds_object);
 }
 
+/**
+ * Create insecure channel credentials
+ * @return null
+ */
+PHP_METHOD(ChannelCredentials, createInsecure) {
+  RETURN_NULL();
+}
+
 static zend_function_entry channel_credentials_methods[] = {
   PHP_ME(ChannelCredentials, createDefault, NULL,
          ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
@@ -176,6 +184,8 @@ static zend_function_entry channel_credentials_methods[] = {
          ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
   PHP_ME(ChannelCredentials, createComposite, NULL,
          ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
+  PHP_ME(ChannelCredentials, createInsecure, NULL,
+         ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
   PHP_FE_END};
 
 void grpc_init_channel_credentials(TSRMLS_D) {
diff --git a/src/php/ext/grpc/package.xml b/src/php/ext/grpc/package.xml
index a2e3e2826a..9c98f82540 100644
--- a/src/php/ext/grpc/package.xml
+++ b/src/php/ext/grpc/package.xml
@@ -10,11 +10,11 @@
   <email>grpc-packages@google.com</email>
   <active>yes</active>
  </lead>
- <date>2015-10-21</date>
- <time>17:04:32</time>
+ <date>2015-12-16</date>
+ <time>12:56:11</time>
  <version>
-  <release>0.6.1</release>
-  <api>0.6.0</api>
+  <release>0.7.0</release>
+  <api>0.7.0</api>
  </version>
  <stability>
   <release>beta</release>
@@ -22,19 +22,22 @@
  </stability>
  <license>BSD</license>
  <notes>
-- fixed undefined constant fatal error when run with apache/nginx #2275
+- Breaking change to Credentials class (removed) #3765
+- Replaced by ChannelCredentials and CallCredentials class #3765
+- New plugin based metadata auth API #4394
+- Explicit ChannelCredentials::createInsecure() call
  </notes>
  <contents>
   <dir baseinstalldir="/" name="/">
    <file baseinstalldir="/" md5sum="6f19828fb869b7b8a590cbb76b4f996d" name="byte_buffer.c" role="src" />
    <file baseinstalldir="/" md5sum="c8de0f819499c48adfc8d7f472c0196b" name="byte_buffer.h" role="src" />
-   <file baseinstalldir="/" md5sum="d64c9005993de02abac55664b0b9e0b2" name="call.c" role="src" />
-   <file baseinstalldir="/" md5sum="26acbf04c30162c2d2aca4688bb2aec8" name="call.h" role="src" />
-   <file baseinstalldir="/" md5sum="6fa13d260dfde216f795225644f04e7a" name="call_credentials.c" role="src" />
-   <file baseinstalldir="/" md5sum="e45269975f9a30fd349a90daf6b31aa2" name="call_credentials.h" role="src" />
-   <file baseinstalldir="/" md5sum="0779db3b196c98081b2260ceec22cd4d" name="channel.c" role="src" />
+   <file baseinstalldir="/" md5sum="ee7eb7757f9e6f0e36f8f616b6bd0af5" name="call.c" role="src" />
+   <file baseinstalldir="/" md5sum="44c56bd9912d2538cbd6059e3e0452b6" name="call.h" role="src" />
+   <file baseinstalldir="/" md5sum="ff90f6c03ed44b5f4170bf3259a6704e" name="call_credentials.c" role="src" />
+   <file baseinstalldir="/" md5sum="3c3860e1d84f43cb6b2fbaa8d2ae1ab7" name="call_credentials.h" role="src" />
+   <file baseinstalldir="/" md5sum="aee9b63f790522aec2c682055240cc61" name="channel.c" role="src" />
    <file baseinstalldir="/" md5sum="ed4b00c0cf3702b115d0cfa87450dc09" name="channel.h" role="src" />
-   <file baseinstalldir="/" md5sum="1b40f50fa6184ad7d24a961ac76151ff" name="channel_credentials.c" role="src" />
+   <file baseinstalldir="/" md5sum="1a51c76d0b7b7d3ab570ed7d60c2ea46" name="channel_credentials.c" role="src" />
    <file baseinstalldir="/" md5sum="a86250e03f610ce6c2c7595a84e08821" name="channel_credentials.h" role="src" />
    <file baseinstalldir="/" md5sum="55ab7a42f9dd9bfc7e28a61cfc5fca63" name="completion_queue.c" role="src" />
    <file baseinstalldir="/" md5sum="f10b5bb232d74a6878e829e2e76cdaa2" name="completion_queue.h" role="src" />
@@ -130,5 +133,23 @@ Update to wrap gRPC C Core version 0.10.0
 - fixed undefined constant fatal error when run with apache/nginx #2275
    </notes>
   </release>
+  <release>
+   <version>
+    <release>0.7.0</release>
+    <api>0.7.0</api>
+   </version>
+   <stability>
+    <release>beta</release>
+    <api>beta</api>
+   </stability>
+   <date>2015-12-16</date>
+   <license>BSD</license>
+   <notes>
+- Breaking change to Credentials class (removed) #3765
+- Replaced by ChannelCredentials and CallCredentials class #3765
+- New plugin based metadata auth API #4394
+- Explicit ChannelCredentials::createInsecure() call
+   </notes>
+  </release>
  </changelog>
 </package>
diff --git a/src/php/lib/Grpc/AbstractCall.php b/src/php/lib/Grpc/AbstractCall.php
index 53849d51fc..712af91eb2 100644
--- a/src/php/lib/Grpc/AbstractCall.php
+++ b/src/php/lib/Grpc/AbstractCall.php
@@ -43,19 +43,20 @@ abstract class AbstractCall
     /**
      * Create a new Call wrapper object.
      *
-     * @param Channel         $channel     The channel to communicate on
-     * @param string          $method      The method to call on the
-     *                                     remote server
-     * @param callback        $deserialize A callback function to deserialize
-     *                                     the response
-     * @param (optional) long $timeout     Timeout in microseconds
+     * @param Channel  $channel     The channel to communicate on
+     * @param string   $method      The method to call on the
+     *                              remote server
+     * @param callback $deserialize A callback function to deserialize
+     *                              the response
+     * @param array    $options     Call options (optional)
      */
     public function __construct(Channel $channel,
                                 $method,
                                 $deserialize,
-                                $timeout = false)
+                                $options = [])
     {
-        if ($timeout) {
+        if (isset($options['timeout']) &&
+            is_numeric($timeout = $options['timeout'])) {
             $now = Timeval::now();
             $delta = new Timeval($timeout);
             $deadline = $now->add($delta);
@@ -65,6 +66,13 @@ abstract class AbstractCall
         $this->call = new Call($channel, $method, $deadline);
         $this->deserialize = $deserialize;
         $this->metadata = null;
+        if (isset($options['call_credentials_callback']) &&
+            is_callable($call_credentials_callback =
+                        $options['call_credentials_callback'])) {
+            $call_credentials = CallCredentials::createFromPlugin(
+                $call_credentials_callback);
+            $this->call->setCredentials($call_credentials);
+        }
     }
 
     /**
@@ -106,4 +114,15 @@ abstract class AbstractCall
 
         return call_user_func($this->deserialize, $value);
     }
+
+    /**
+     * Set the CallCredentials for the underlying Call.
+     *
+     * @param CallCredentials $call_credentials The CallCredentials
+     *                                          object
+     */
+    public function setCallCredentials($call_credentials)
+    {
+        $this->call->setCredentials($call_credentials);
+    }
 }
diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php
index 7b2627f516..2de1b337e5 100755
--- a/src/php/lib/Grpc/BaseStub.php
+++ b/src/php/lib/Grpc/BaseStub.php
@@ -72,6 +72,11 @@ class BaseStub
         }
         $opts['grpc.primary_user_agent'] .=
             'grpc-php/'.$package_config['version'];
+        if (!array_key_exists('credentials', $opts)) {
+            throw new \Exception("The opts['credentials'] key is now ".
+                                 'required. Please see one of the '.
+                                 'ChannelCredentials::create methods');
+        }
         $this->channel = new Channel($hostname, $opts);
     }
 
@@ -159,25 +164,6 @@ class BaseStub
     }
 
     /**
-     * extract $timeout from $metadata.
-     *
-     * @param $metadata The metadata map
-     *
-     * @return list($metadata_copy, $timeout)
-     */
-    private function _extract_timeout_from_metadata($metadata)
-    {
-        $timeout = false;
-        $metadata_copy = $metadata;
-        if (isset($metadata['timeout'])) {
-            $timeout = $metadata['timeout'];
-            unset($metadata_copy['timeout']);
-        }
-
-        return [$metadata_copy, $timeout];
-    }
-
-    /**
      * validate and normalize the metadata array.
      *
      * @param $metadata The metadata map
@@ -220,21 +206,19 @@ class BaseStub
                                    $metadata = [],
                                    $options = [])
     {
-        list($actual_metadata, $timeout) =
-            $this->_extract_timeout_from_metadata($metadata);
         $call = new UnaryCall($this->channel,
                               $method,
                               $deserialize,
-                              $timeout);
+                              $options);
         $jwt_aud_uri = $this->_get_jwt_aud_uri($method);
         if (is_callable($this->update_metadata)) {
-            $actual_metadata = call_user_func($this->update_metadata,
-                                        $actual_metadata,
+            $metadata = call_user_func($this->update_metadata,
+                                        $metadata,
                                         $jwt_aud_uri);
         }
-        $actual_metadata = $this->_validate_and_normalize_metadata(
-            $actual_metadata);
-        $call->start($argument, $actual_metadata, $options);
+        $metadata = $this->_validate_and_normalize_metadata(
+            $metadata);
+        $call->start($argument, $metadata, $options);
 
         return $call;
     }
@@ -253,23 +237,22 @@ class BaseStub
      */
     public function _clientStreamRequest($method,
                                          callable $deserialize,
-                                         $metadata = [])
+                                         $metadata = [],
+                                         $options = [])
     {
-        list($actual_metadata, $timeout) =
-            $this->_extract_timeout_from_metadata($metadata);
         $call = new ClientStreamingCall($this->channel,
                                         $method,
                                         $deserialize,
-                                        $timeout);
+                                        $options);
         $jwt_aud_uri = $this->_get_jwt_aud_uri($method);
         if (is_callable($this->update_metadata)) {
-            $actual_metadata = call_user_func($this->update_metadata,
-                                        $actual_metadata,
+            $metadata = call_user_func($this->update_metadata,
+                                        $metadata,
                                         $jwt_aud_uri);
         }
-        $actual_metadata = $this->_validate_and_normalize_metadata(
-            $actual_metadata);
-        $call->start($actual_metadata);
+        $metadata = $this->_validate_and_normalize_metadata(
+            $metadata);
+        $call->start($metadata);
 
         return $call;
     }
@@ -291,21 +274,19 @@ class BaseStub
                                          $metadata = [],
                                          $options = [])
     {
-        list($actual_metadata, $timeout) =
-            $this->_extract_timeout_from_metadata($metadata);
         $call = new ServerStreamingCall($this->channel,
                                         $method,
                                         $deserialize,
-                                        $timeout);
+                                        $options);
         $jwt_aud_uri = $this->_get_jwt_aud_uri($method);
         if (is_callable($this->update_metadata)) {
-            $actual_metadata = call_user_func($this->update_metadata,
-                                        $actual_metadata,
+            $metadata = call_user_func($this->update_metadata,
+                                        $metadata,
                                         $jwt_aud_uri);
         }
-        $actual_metadata = $this->_validate_and_normalize_metadata(
-            $actual_metadata);
-        $call->start($argument, $actual_metadata, $options);
+        $metadata = $this->_validate_and_normalize_metadata(
+            $metadata);
+        $call->start($argument, $metadata, $options);
 
         return $call;
     }
@@ -321,23 +302,22 @@ class BaseStub
      */
     public function _bidiRequest($method,
                                  callable $deserialize,
-                                 $metadata = [])
+                                 $metadata = [],
+                                 $options = [])
     {
-        list($actual_metadata, $timeout) =
-            $this->_extract_timeout_from_metadata($metadata);
         $call = new BidiStreamingCall($this->channel,
                                       $method,
                                       $deserialize,
-                                      $timeout);
+                                      $options);
         $jwt_aud_uri = $this->_get_jwt_aud_uri($method);
         if (is_callable($this->update_metadata)) {
-            $actual_metadata = call_user_func($this->update_metadata,
-                                        $actual_metadata,
+            $metadata = call_user_func($this->update_metadata,
+                                        $metadata,
                                         $jwt_aud_uri);
         }
-        $actual_metadata = $this->_validate_and_normalize_metadata(
-            $actual_metadata);
-        $call->start($actual_metadata);
+        $metadata = $this->_validate_and_normalize_metadata(
+            $metadata);
+        $call->start($metadata);
 
         return $call;
     }
diff --git a/src/php/tests/generated_code/AbstractGeneratedCodeTest.php b/src/php/tests/generated_code/AbstractGeneratedCodeTest.php
index 4a0bd6a1f1..1fe81b9d54 100644
--- a/src/php/tests/generated_code/AbstractGeneratedCodeTest.php
+++ b/src/php/tests/generated_code/AbstractGeneratedCodeTest.php
@@ -41,7 +41,6 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase
      * running on $GRPC_TEST_HOST.
      */
     protected static $client;
-    protected static $timeout;
 
     public function testWaitForNotReady()
     {
@@ -93,7 +92,7 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase
     public function testTimeout()
     {
         $div_arg = new math\DivArgs();
-        $call = self::$client->Div($div_arg, ['timeout' => 100]);
+        $call = self::$client->Div($div_arg, [], ['timeout' => 100]);
         list($response, $status) = $call->wait();
         $this->assertSame(\Grpc\STATUS_DEADLINE_EXCEEDED, $status->code);
     }
@@ -112,7 +111,9 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase
      */
     public function testInvalidMethodName()
     {
-        $invalid_client = new DummyInvalidClient('host', []);
+        $invalid_client = new DummyInvalidClient('host', [
+            'credentials' => Grpc\ChannelCredentials::createInsecure(),
+        ]);
         $div_arg = new math\DivArgs();
         $invalid_client->InvalidUnaryCall($div_arg);
     }
diff --git a/src/php/tests/generated_code/GeneratedCodeTest.php b/src/php/tests/generated_code/GeneratedCodeTest.php
index 7043e8e1d1..0cdce6cf92 100755
--- a/src/php/tests/generated_code/GeneratedCodeTest.php
+++ b/src/php/tests/generated_code/GeneratedCodeTest.php
@@ -38,10 +38,12 @@ class GeneratedCodeTest extends AbstractGeneratedCodeTest
     public function setUp()
     {
         self::$client = new math\MathClient(
-        getenv('GRPC_TEST_HOST'), []);
+            getenv('GRPC_TEST_HOST'), [
+                'credentials' => Grpc\ChannelCredentials::createInsecure(),
+            ]);
     }
 
-    public static function tearDownAfterClass()
+    public function tearDown()
     {
         self::$client->close();
     }
diff --git a/src/php/tests/generated_code/GeneratedCodeWithCallbackTest.php b/src/php/tests/generated_code/GeneratedCodeWithCallbackTest.php
index 5a20e684c7..6bb1955ccb 100644
--- a/src/php/tests/generated_code/GeneratedCodeWithCallbackTest.php
+++ b/src/php/tests/generated_code/GeneratedCodeWithCallbackTest.php
@@ -39,7 +39,8 @@ class GeneratedCodeWithCallbackTest extends AbstractGeneratedCodeTest
     {
         self::$client = new math\MathClient(
         getenv('GRPC_TEST_HOST'),
-        ['update_metadata' => function ($a_hash,
+        ['credentials' => Grpc\ChannelCredentials::createInsecure(),
+         'update_metadata' => function ($a_hash,
                                         $client = []) {
                                 $a_copy = $a_hash;
                                 $a_copy['foo'] = ['bar'];
@@ -48,7 +49,7 @@ class GeneratedCodeWithCallbackTest extends AbstractGeneratedCodeTest
                               }]);
     }
 
-    public static function tearDownAfterClass()
+    public function tearDown()
     {
         self::$client->close();
     }
diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php
index 9aab5c966c..aebf80f6bf 100755
--- a/src/php/tests/interop/interop_client.php
+++ b/src/php/tests/interop/interop_client.php
@@ -84,7 +84,7 @@ function largeUnary($stub)
  * @param $fillOauthScope boolean whether to fill result with oauth scope
  */
 function performLargeUnary($stub, $fillUsername = false, $fillOauthScope = false,
-                           $metadata = [])
+                           $callback = false)
 {
     $request_len = 271828;
     $response_len = 314159;
@@ -99,7 +99,12 @@ function performLargeUnary($stub, $fillUsername = false, $fillOauthScope = false
     $request->setFillUsername($fillUsername);
     $request->setFillOauthScope($fillOauthScope);
 
-    list($result, $status) = $stub->UnaryCall($request, $metadata)->wait();
+    $options = false;
+    if ($callback) {
+        $options['call_credentials_callback'] = $callback;
+    }
+
+    list($result, $status) = $stub->UnaryCall($request, [], $options)->wait();
     hardAssert($status->code === Grpc\STATUS_OK, 'Call did not complete successfully');
     hardAssert($result !== null, 'Call returned a null response');
     $payload = $result->getPayload();
@@ -186,6 +191,15 @@ function oauth2AuthToken($stub, $args)
              'invalid email returned');
 }
 
+function updateAuthMetadataCallback($context)
+{
+    $authUri = $context->service_url;
+    $methodName = $context->method_name;
+    $auth_credentials = ApplicationDefaultCredentials::getCredentials();
+
+    return $auth_credentials->updateMetadata($metadata = [], $authUri);
+}
+
 /**
  * Run the per_rpc_creds auth test.
  *
@@ -197,15 +211,9 @@ function perRpcCreds($stub, $args)
     $jsonKey = json_decode(
         file_get_contents(getenv(CredentialsLoader::ENV_VAR)),
         true);
-    $auth_credentials = ApplicationDefaultCredentials::getCredentials(
-        $args['oauth_scope']
-    );
-    $token = $auth_credentials->fetchAuthToken();
-    $metadata = [CredentialsLoader::AUTH_METADATA_KEY => [sprintf('%s %s',
-                          $token['token_type'],
-                          $token['access_token'])]];
+
     $result = performLargeUnary($stub, $fillUsername = true, $fillOauthScope = true,
-                              $metadata);
+                                'updateAuthMetadataCallback');
     hardAssert($result->getUsername() == $jsonKey['client_email'],
              'invalid email returned');
 }
@@ -363,7 +371,7 @@ function cancelAfterFirstResponse($stub)
 
 function timeoutOnSleepingServer($stub)
 {
-    $call = $stub->FullDuplexCall(['timeout' => 1000]);
+    $call = $stub->FullDuplexCall([], ['timeout' => 1000]);
     $request = new grpc\testing\StreamingOutputCallRequest();
     $request->setResponseType(grpc\testing\PayloadType::COMPRESSABLE);
     $response_parameters = new grpc\testing\ResponseParameters();
@@ -430,6 +438,8 @@ if ($use_tls) {
     }
     $opts['credentials'] = $ssl_credentials;
     $opts['grpc.ssl_target_name_override'] = $host_override;
+} else {
+    $opts['credentials'] = Grpc\ChannelCredentials::createInsecure();
 }
 
 if (in_array($test_case, ['service_account_creds',
diff --git a/src/php/tests/unit_tests/CallCredentialsTest.php b/src/php/tests/unit_tests/CallCredentialsTest.php
new file mode 100644
index 0000000000..0918412781
--- /dev/null
+++ b/src/php/tests/unit_tests/CallCredentialsTest.php
@@ -0,0 +1,137 @@
+<?php
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+class CallCredentialsTest extends PHPUnit_Framework_TestCase
+{
+    public function setUp()
+    {
+        $credentials = Grpc\ChannelCredentials::createSsl(
+            file_get_contents(dirname(__FILE__).'/../data/ca.pem'));
+        $call_credentials = Grpc\CallCredentials::createFromPlugin(
+            array($this, 'callbackFunc'));
+        $credentials = Grpc\ChannelCredentials::createComposite(
+            $credentials,
+            $call_credentials
+        );
+        $server_credentials = Grpc\ServerCredentials::createSsl(
+            null,
+            file_get_contents(dirname(__FILE__).'/../data/server1.key'),
+            file_get_contents(dirname(__FILE__).'/../data/server1.pem'));
+        $this->server = new Grpc\Server();
+        $this->port = $this->server->addSecureHttp2Port('0.0.0.0:0',
+                                              $server_credentials);
+        $this->server->start();
+        $this->host_override = 'foo.test.google.fr';
+        $this->channel = new Grpc\Channel(
+            'localhost:'.$this->port,
+            [
+            'grpc.ssl_target_name_override' => $this->host_override,
+            'grpc.default_authority' => $this->host_override,
+            'credentials' => $credentials,
+            ]
+        );
+    }
+
+    public function tearDown()
+    {
+        unset($this->channel);
+        unset($this->server);
+    }
+
+    public function callbackFunc($context)
+    {
+        $this->assertTrue(is_string($context->service_url));
+        $this->assertTrue(is_string($context->method_name));
+
+        return ['k1' => ['v1'], 'k2' => ['v2']];
+    }
+
+    public function testCreateFromPlugin()
+    {
+        $deadline = Grpc\Timeval::infFuture();
+        $status_text = 'xyz';
+        $call = new Grpc\Call($this->channel,
+                              '/abc/dummy_method',
+                              $deadline,
+                              $this->host_override);
+
+        $event = $call->startBatch([
+            Grpc\OP_SEND_INITIAL_METADATA => [],
+            Grpc\OP_SEND_CLOSE_FROM_CLIENT => true,
+        ]);
+
+        $this->assertTrue($event->send_metadata);
+        $this->assertTrue($event->send_close);
+
+        $event = $this->server->requestCall();
+
+        $this->assertTrue(is_array($event->metadata));
+        $metadata = $event->metadata;
+        $this->assertTrue(array_key_exists('k1', $metadata));
+        $this->assertTrue(array_key_exists('k2', $metadata));
+        $this->assertSame($metadata['k1'], ['v1']);
+        $this->assertSame($metadata['k2'], ['v2']);
+
+        $this->assertSame('/abc/dummy_method', $event->method);
+        $server_call = $event->call;
+
+        $event = $server_call->startBatch([
+            Grpc\OP_SEND_INITIAL_METADATA => [],
+            Grpc\OP_SEND_STATUS_FROM_SERVER => [
+                'metadata' => [],
+                'code' => Grpc\STATUS_OK,
+                'details' => $status_text,
+            ],
+            Grpc\OP_RECV_CLOSE_ON_SERVER => true,
+        ]);
+
+        $this->assertTrue($event->send_metadata);
+        $this->assertTrue($event->send_status);
+        $this->assertFalse($event->cancelled);
+
+        $event = $call->startBatch([
+            Grpc\OP_RECV_INITIAL_METADATA => true,
+            Grpc\OP_RECV_STATUS_ON_CLIENT => true,
+        ]);
+
+        $this->assertSame([], $event->metadata);
+        $status = $event->status;
+        $this->assertSame([], $status->metadata);
+        $this->assertSame(Grpc\STATUS_OK, $status->code);
+        $this->assertSame($status_text, $status->details);
+
+        unset($call);
+        unset($server_call);
+    }
+}
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxd b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxd
index 10c948cd0a..643cdc9e3d 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxd
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxd
@@ -60,6 +60,8 @@ cdef extern from "grpc/support/port_platform.h":
   # type exactly; just close enough that the operations will be supported in the
   # underlying C layers.
   ctypedef unsigned int gpr_uint32
+  ctypedef int gpr_int32
+  ctypedef long int gpr_int64
 
 
 cdef extern from "grpc/support/time.h":
@@ -71,8 +73,8 @@ cdef extern from "grpc/support/time.h":
     GPR_TIMESPAN
 
   ctypedef struct gpr_timespec:
-    libc.time.time_t seconds "tv_sec"
-    int nanoseconds "tv_nsec"
+    gpr_int64 seconds "tv_sec"
+    gpr_int32 nanoseconds "tv_nsec"
     gpr_clock_type clock_type
 
   gpr_timespec gpr_time_0(gpr_clock_type type)
diff --git a/src/python/grpcio/setup.py b/src/python/grpcio/setup.py
index a948ca1fac..b8a98c3d85 100644
--- a/src/python/grpcio/setup.py
+++ b/src/python/grpcio/setup.py
@@ -163,7 +163,7 @@ else:
 
 setuptools.setup(
     name='grpcio',
-    version='0.11.0b2',
+    version='0.12.0b0',
     ext_modules=CYTHON_EXTENSION_MODULES,
     packages=list(PACKAGES),
     package_dir=PACKAGE_DIRECTORIES,
diff --git a/src/python/grpcio/tests/interop/client.py b/src/python/grpcio/tests/interop/client.py
index 5c00bce014..573ec2bd71 100644
--- a/src/python/grpcio/tests/interop/client.py
+++ b/src/python/grpcio/tests/interop/client.py
@@ -90,7 +90,7 @@ def _stub(args):
     if args.use_test_ca:
       root_certificates = resources.test_root_certificates()
     else:
-      root_certificates = resources.prod_root_certificates()
+      root_certificates = None  # will load default roots.
 
     channel = test_utilities.not_really_secure_channel(
         args.server_host, args.server_port,
diff --git a/src/python/grpcio/tests/interop/resources.py b/src/python/grpcio/tests/interop/resources.py
index 1122499418..c424385cf6 100644
--- a/src/python/grpcio/tests/interop/resources.py
+++ b/src/python/grpcio/tests/interop/resources.py
@@ -44,10 +44,6 @@ def test_root_certificates():
       __name__, _ROOT_CERTIFICATES_RESOURCE_PATH)
 
 
-def prod_root_certificates():
-  return open(os.environ['SSL_CERT_FILE'], mode='rb').read()
-
-
 def private_key():
   return pkg_resources.resource_string(__name__, _PRIVATE_KEY_RESOURCE_PATH)
 
diff --git a/src/python/grpcio/tests/unit/resources.py b/src/python/grpcio/tests/unit/resources.py
index 2c3045313d..023cdb155f 100644
--- a/src/python/grpcio/tests/unit/resources.py
+++ b/src/python/grpcio/tests/unit/resources.py
@@ -43,10 +43,6 @@ def test_root_certificates():
       __name__, _ROOT_CERTIFICATES_RESOURCE_PATH)
 
 
-def prod_root_certificates():
-  return open(os.environ['SSL_CERT_FILE'], mode='rb').read()
-
-
 def private_key():
   return pkg_resources.resource_string(__name__, _PRIVATE_KEY_RESOURCE_PATH)
 
diff --git a/src/ruby/bin/apis/pubsub_demo.rb b/src/ruby/bin/apis/pubsub_demo.rb
index 003e91a6b3..143ecc7a8f 100755
--- a/src/ruby/bin/apis/pubsub_demo.rb
+++ b/src/ruby/bin/apis/pubsub_demo.rb
@@ -32,7 +32,6 @@
 # pubsub_demo demos accesses the Google PubSub API via its gRPC interface
 #
 # $ GOOGLE_APPLICATION_CREDENTIALS=<path_to_service_account_key_file> \
-#   SSL_CERT_FILE=<path/to/ssl/certs> \
 #   path/to/pubsub_demo.rb \
 #   [--action=<chosen_demo_action> ]
 #
@@ -55,18 +54,9 @@ require 'google/protobuf/empty'
 require 'tech/pubsub/proto/pubsub'
 require 'tech/pubsub/proto/pubsub_services'
 
-# loads the certificates used to access the test server securely.
-def load_prod_cert
-  fail 'could not find a production cert' if ENV['SSL_CERT_FILE'].nil?
-  p "loading prod certs from #{ENV['SSL_CERT_FILE']}"
-  File.open(ENV['SSL_CERT_FILE']) do |f|
-    return f.read
-  end
-end
-
 # creates a SSL Credentials from the production certificates.
 def ssl_creds
-  GRPC::Core::ChannelCredentials.new(load_prod_cert)
+  GRPC::Core::ChannelCredentials.new()
 end
 
 # Builds the metadata authentication update proc.
@@ -80,8 +70,9 @@ def publisher_stub(opts)
   address = "#{opts.host}:#{opts.port}"
   stub_clz = Tech::Pubsub::PublisherService::Stub # shorter
   GRPC.logger.info("... access PublisherService at #{address}")
-  stub_clz.new(address,
-               creds: ssl_creds, update_metadata: auth_proc(opts),
+  call_creds = GRPC::Core::CallCredentials.new(auth_proc(opts))
+  combined_creds = ssl_creds.compose(call_creds)
+  stub_clz.new(address, creds: combined_creds,
                GRPC::Core::Channel::SSL_TARGET => opts.host)
 end
 
@@ -90,8 +81,9 @@ def subscriber_stub(opts)
   address = "#{opts.host}:#{opts.port}"
   stub_clz = Tech::Pubsub::SubscriberService::Stub # shorter
   GRPC.logger.info("... access SubscriberService at #{address}")
-  stub_clz.new(address,
-               creds: ssl_creds, update_metadata: auth_proc(opts),
+  call_creds = GRPC::Core::CallCredentials.new(auth_proc(opts))
+  combined_creds = ssl_creds.compose(call_creds)
+  stub_clz.new(address, creds: combined_creds,
                GRPC::Core::Channel::SSL_TARGET => opts.host)
 end
 
diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c
index 40364328ee..1647d9b484 100644
--- a/src/ruby/ext/grpc/rb_call.c
+++ b/src/ruby/ext/grpc/rb_call.c
@@ -39,6 +39,7 @@
 #include <grpc/support/alloc.h>
 
 #include "rb_byte_buffer.h"
+#include "rb_call_credentials.h"
 #include "rb_completion_queue.h"
 #include "rb_grpc.h"
 
@@ -279,6 +280,26 @@ static VALUE grpc_rb_call_set_write_flag(VALUE self, VALUE write_flag) {
   return rb_ivar_set(self, id_write_flag, write_flag);
 }
 
+/*
+  call-seq:
+  call.set_credentials call_credentials
+
+  Sets credentials on a call */
+static VALUE grpc_rb_call_set_credentials(VALUE self, VALUE credentials) {
+  grpc_call *call = NULL;
+  grpc_call_credentials *creds;
+  grpc_call_error err;
+  TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
+  creds = grpc_rb_get_wrapped_call_credentials(credentials);
+  err = grpc_call_set_credentials(call, creds);
+  if (err != GRPC_CALL_OK) {
+    rb_raise(grpc_rb_eCallError,
+             "grpc_call_set_credentials failed with %s (code=%d)",
+             grpc_call_error_detail_of(err), err);
+  }
+  return Qnil;
+}
+
 /* grpc_rb_md_ary_fill_hash_cb is the hash iteration callback used
    to fill grpc_metadata_array.
 
@@ -347,7 +368,7 @@ static int grpc_rb_md_ary_capacity_hash_cb(VALUE key, VALUE val,
 /* grpc_rb_md_ary_convert converts a ruby metadata hash into
    a grpc_metadata_array.
 */
-static void grpc_rb_md_ary_convert(VALUE md_ary_hash,
+void grpc_rb_md_ary_convert(VALUE md_ary_hash,
                                    grpc_metadata_array *md_ary) {
   VALUE md_ary_obj = Qnil;
   if (md_ary_hash == Qnil) {
@@ -795,6 +816,8 @@ void Init_grpc_call() {
   rb_define_method(grpc_rb_cCall, "write_flag", grpc_rb_call_get_write_flag, 0);
   rb_define_method(grpc_rb_cCall, "write_flag=", grpc_rb_call_set_write_flag,
                    1);
+  rb_define_method(grpc_rb_cCall, "set_credentials!",
+                   grpc_rb_call_set_credentials, 1);
 
   /* Ids used to support call attributes */
   id_metadata = rb_intern("metadata");
diff --git a/src/ruby/ext/grpc/rb_call.h b/src/ruby/ext/grpc/rb_call.h
index 1d2fbc3580..24adb3477b 100644
--- a/src/ruby/ext/grpc/rb_call.h
+++ b/src/ruby/ext/grpc/rb_call.h
@@ -50,6 +50,12 @@ const char* grpc_call_error_detail_of(grpc_call_error err);
 /* Converts a metadata array to a hash. */
 VALUE grpc_rb_md_ary_to_h(grpc_metadata_array *md_ary);
 
+/* grpc_rb_md_ary_convert converts a ruby metadata hash into
+   a grpc_metadata_array.
+*/
+void grpc_rb_md_ary_convert(VALUE md_ary_hash,
+                            grpc_metadata_array *md_ary);
+
 /* grpc_rb_eCallError is the ruby class of the exception thrown during call
    operations. */
 extern VALUE grpc_rb_eCallError;
diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c
new file mode 100644
index 0000000000..acc5472799
--- /dev/null
+++ b/src/ruby/ext/grpc/rb_call_credentials.c
@@ -0,0 +1,312 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "rb_call_credentials.h"
+
+#include <ruby/ruby.h>
+#include <ruby/thread.h>
+
+#include <grpc/grpc.h>
+#include <grpc/grpc_security.h>
+
+#include "rb_call.h"
+#include "rb_grpc.h"
+
+/* grpc_rb_cCallCredentials is the ruby class that proxies
+ * grpc_call_credentials */
+static VALUE grpc_rb_cCallCredentials = Qnil;
+
+/* grpc_rb_call_credentials wraps a grpc_call_credentials. It provides a peer
+ * ruby object, 'mark' to minimize copying when a credential is created from
+ * ruby. */
+typedef struct grpc_rb_call_credentials {
+  /* Holder of ruby objects involved in contructing the credentials */
+  VALUE mark;
+
+  /* The actual credentials */
+  grpc_call_credentials *wrapped;
+} grpc_rb_call_credentials;
+
+typedef struct callback_params {
+  VALUE get_metadata;
+  grpc_auth_metadata_context context;
+  void *user_data;
+  grpc_credentials_plugin_metadata_cb callback;
+} callback_params;
+
+static VALUE grpc_rb_call_credentials_callback(VALUE callback_args) {
+  VALUE result = rb_hash_new();
+  VALUE metadata = rb_funcall(rb_ary_entry(callback_args, 0), rb_intern("call"),
+                              1, rb_ary_entry(callback_args, 1));
+  rb_hash_aset(result, rb_str_new2("metadata"), metadata);
+  rb_hash_aset(result, rb_str_new2("status"), INT2NUM(GRPC_STATUS_OK));
+  rb_hash_aset(result, rb_str_new2("details"), rb_str_new2(""));
+  return result;
+}
+
+static VALUE grpc_rb_call_credentials_callback_rescue(VALUE args,
+                                                      VALUE exception_object) {
+  VALUE result = rb_hash_new();
+  rb_hash_aset(result, rb_str_new2("metadata"), Qnil);
+  /* Currently only gives the exception class name. It should be possible get
+     more details */
+  rb_hash_aset(result, rb_str_new2("status"),
+               INT2NUM(GRPC_STATUS_PERMISSION_DENIED));
+  rb_hash_aset(result, rb_str_new2("details"),
+               rb_str_new2(rb_obj_classname(exception_object)));
+  return result;
+}
+
+static void *grpc_rb_call_credentials_callback_with_gil(void *param) {
+  callback_params *const params = (callback_params *)param;
+  VALUE auth_uri = rb_str_new_cstr(params->context.service_url);
+  /* Pass the arguments to the proc in a hash, which currently only has they key
+     'auth_uri' */
+  VALUE callback_args = rb_ary_new();
+  VALUE args = rb_hash_new();
+  VALUE result;
+  grpc_metadata_array md_ary;
+  grpc_status_code status;
+  VALUE details;
+  char *error_details;
+  grpc_metadata_array_init(&md_ary);
+  rb_hash_aset(args, ID2SYM(rb_intern("jwt_aud_uri")), auth_uri);
+  rb_ary_push(callback_args, params->get_metadata);
+  rb_ary_push(callback_args, args);
+  result = rb_rescue(grpc_rb_call_credentials_callback, callback_args,
+                     grpc_rb_call_credentials_callback_rescue, Qnil);
+  // Both callbacks return a hash, so result should be a hash
+  grpc_rb_md_ary_convert(rb_hash_aref(result, rb_str_new2("metadata")), &md_ary);
+  status = NUM2INT(rb_hash_aref(result, rb_str_new2("status")));
+  details = rb_hash_aref(result, rb_str_new2("details"));
+  error_details = StringValueCStr(details);
+  params->callback(params->user_data, md_ary.metadata, md_ary.count, status,
+                   error_details);
+  grpc_metadata_array_destroy(&md_ary);
+
+  return NULL;
+}
+
+static void grpc_rb_call_credentials_plugin_get_metadata(
+    void *state, grpc_auth_metadata_context context,
+    grpc_credentials_plugin_metadata_cb cb, void *user_data) {
+  callback_params params;
+  params.get_metadata = (VALUE)state;
+  params.context = context;
+  params.user_data = user_data;
+  params.callback = cb;
+
+  rb_thread_call_with_gvl(grpc_rb_call_credentials_callback_with_gil,
+                          (void*)(&params));
+}
+
+static void grpc_rb_call_credentials_plugin_destroy(void *state) {
+  // Not sure what needs to be done here
+}
+
+/* Destroys the credentials instances. */
+static void grpc_rb_call_credentials_free(void *p) {
+  grpc_rb_call_credentials *wrapper;
+  if (p == NULL) {
+    return;
+  }
+  wrapper = (grpc_rb_call_credentials *)p;
+
+  /* Delete the wrapped object if the mark object is Qnil, which indicates that
+   * no other object is the actual owner. */
+  if (wrapper->wrapped != NULL && wrapper->mark == Qnil) {
+    grpc_call_credentials_release(wrapper->wrapped);
+    wrapper->wrapped = NULL;
+  }
+
+  xfree(p);
+}
+
+/* Protects the mark object from GC */
+static void grpc_rb_call_credentials_mark(void *p) {
+  grpc_rb_call_credentials *wrapper = NULL;
+  if (p == NULL) {
+    return;
+  }
+  wrapper = (grpc_rb_call_credentials *)p;
+
+  /* If it's not already cleaned up, mark the mark object */
+  if (wrapper->mark != Qnil) {
+    rb_gc_mark(wrapper->mark);
+  }
+}
+
+static rb_data_type_t grpc_rb_call_credentials_data_type = {
+  "grpc_call_credentials",
+  {grpc_rb_call_credentials_mark, grpc_rb_call_credentials_free,
+   GRPC_RB_MEMSIZE_UNAVAILABLE, {NULL, NULL}},
+  NULL,
+  NULL,
+#ifdef RUBY_TYPED_FREE_IMMEDIATELY
+  RUBY_TYPED_FREE_IMMEDIATELY
+#endif
+};
+
+/* Allocates CallCredentials instances.
+   Provides safe initial defaults for the instance fields. */
+static VALUE grpc_rb_call_credentials_alloc(VALUE cls) {
+  grpc_rb_call_credentials *wrapper = ALLOC(grpc_rb_call_credentials);
+  wrapper->wrapped = NULL;
+  wrapper->mark = Qnil;
+  return TypedData_Wrap_Struct(cls, &grpc_rb_call_credentials_data_type, wrapper);
+}
+
+/* Creates a wrapping object for a given call credentials. This should only be
+ * called with grpc_call_credentials objects that are not already associated
+ * with any Ruby object */
+VALUE grpc_rb_wrap_call_credentials(grpc_call_credentials *c) {
+  VALUE rb_wrapper;
+  grpc_rb_call_credentials *wrapper;
+  if (c == NULL) {
+    return Qnil;
+  }
+  rb_wrapper = grpc_rb_call_credentials_alloc(grpc_rb_cCallCredentials);
+  TypedData_Get_Struct(rb_wrapper, grpc_rb_call_credentials,
+                       &grpc_rb_call_credentials_data_type, wrapper);
+  wrapper->wrapped = c;
+  return rb_wrapper;
+}
+
+/* Clones CallCredentials instances.
+   Gives CallCredentials a consistent implementation of Ruby's object copy/dup
+   protocol. */
+static VALUE grpc_rb_call_credentials_init_copy(VALUE copy, VALUE orig) {
+  grpc_rb_call_credentials *orig_cred = NULL;
+  grpc_rb_call_credentials *copy_cred = NULL;
+
+  if (copy == orig) {
+    return copy;
+  }
+
+  /* Raise an error if orig is not a credentials object or a subclass. */
+  if (TYPE(orig) != T_DATA ||
+      RDATA(orig)->dfree != (RUBY_DATA_FUNC)grpc_rb_call_credentials_free) {
+    rb_raise(rb_eTypeError, "not a %s",
+             rb_obj_classname(grpc_rb_cCallCredentials));
+  }
+
+  TypedData_Get_Struct(orig, grpc_rb_call_credentials,
+                       &grpc_rb_call_credentials_data_type, orig_cred);
+  TypedData_Get_Struct(copy, grpc_rb_call_credentials,
+                       &grpc_rb_call_credentials_data_type, copy_cred);
+
+  /* use ruby's MEMCPY to make a byte-for-byte copy of the credentials
+   * wrapper object. */
+  MEMCPY(copy_cred, orig_cred, grpc_rb_call_credentials, 1);
+  return copy;
+}
+
+/* The attribute used on the mark object to hold the callback */
+static ID id_callback;
+
+/*
+  call-seq:
+    creds = Credentials.new auth_proc
+  proc: (required) Proc that generates auth metadata
+  Initializes CallCredential instances. */
+static VALUE grpc_rb_call_credentials_init(VALUE self, VALUE proc) {
+  grpc_rb_call_credentials *wrapper = NULL;
+  grpc_call_credentials *creds = NULL;
+  grpc_metadata_credentials_plugin plugin;
+
+  TypedData_Get_Struct(self, grpc_rb_call_credentials,
+                       &grpc_rb_call_credentials_data_type, wrapper);
+
+  plugin.get_metadata = grpc_rb_call_credentials_plugin_get_metadata;
+  plugin.destroy = grpc_rb_call_credentials_plugin_destroy;
+  if (!rb_obj_is_proc(proc)) {
+    rb_raise(rb_eTypeError, "Argument to CallCredentials#new must be a proc");
+    return Qnil;
+  }
+  plugin.state = (void*)proc;
+  plugin.type = "";
+
+  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  if (creds == NULL) {
+    rb_raise(rb_eRuntimeError, "could not create a credentials, not sure why");
+    return Qnil;
+  }
+
+  wrapper->wrapped = creds;
+  rb_ivar_set(self, id_callback, proc);
+
+  return self;
+}
+
+static VALUE grpc_rb_call_credentials_compose(int argc, VALUE *argv,
+                                              VALUE self) {
+  grpc_call_credentials *creds;
+  grpc_call_credentials *other;
+  if (argc == 0) {
+    return self;
+  }
+  creds = grpc_rb_get_wrapped_call_credentials(self);
+  for (int i = 0; i < argc; i++) {
+    other = grpc_rb_get_wrapped_call_credentials(argv[i]);
+    creds = grpc_composite_call_credentials_create(creds, other, NULL);
+  }
+  return grpc_rb_wrap_call_credentials(creds);
+}
+
+void Init_grpc_call_credentials() {
+  grpc_rb_cCallCredentials =
+      rb_define_class_under(grpc_rb_mGrpcCore, "CallCredentials", rb_cObject);
+
+  /* Allocates an object managed by the ruby runtime */
+  rb_define_alloc_func(grpc_rb_cCallCredentials,
+                       grpc_rb_call_credentials_alloc);
+
+  /* Provides a ruby constructor and support for dup/clone. */
+  rb_define_method(grpc_rb_cCallCredentials, "initialize",
+                   grpc_rb_call_credentials_init, 1);
+  rb_define_method(grpc_rb_cCallCredentials, "initialize_copy",
+                   grpc_rb_call_credentials_init_copy, 1);
+  rb_define_method(grpc_rb_cCallCredentials, "compose",
+                   grpc_rb_call_credentials_compose, -1);
+
+  id_callback = rb_intern("__callback");
+}
+
+/* Gets the wrapped grpc_call_credentials from the ruby wrapper */
+grpc_call_credentials *grpc_rb_get_wrapped_call_credentials(VALUE v) {
+  grpc_rb_call_credentials *wrapper = NULL;
+  TypedData_Get_Struct(v, grpc_rb_call_credentials,
+                       &grpc_rb_call_credentials_data_type,
+                       wrapper);
+  return wrapper->wrapped;
+}
diff --git a/src/ruby/ext/grpc/rb_call_credentials.h b/src/ruby/ext/grpc/rb_call_credentials.h
new file mode 100644
index 0000000000..5350a8f7ff
--- /dev/null
+++ b/src/ruby/ext/grpc/rb_call_credentials.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_RB_CALL_CREDENTIALS_H_
+#define GRPC_RB_CALL_CREDENTIALS_H_
+
+#include <ruby/ruby.h>
+
+#include <grpc/grpc_security.h>
+
+/* Initializes the ruby CallCredentials class. */
+void Init_grpc_call_credentials();
+
+grpc_call_credentials* grpc_rb_get_wrapped_call_credentials(VALUE v);
+
+#endif /* GRPC_RB_CALL_CREDENTIALS_H_ */
diff --git a/src/ruby/ext/grpc/rb_channel_credentials.c b/src/ruby/ext/grpc/rb_channel_credentials.c
index 072a6f54ab..3530081c6b 100644
--- a/src/ruby/ext/grpc/rb_channel_credentials.c
+++ b/src/ruby/ext/grpc/rb_channel_credentials.c
@@ -37,7 +37,9 @@
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
+#include <grpc/support/log.h>
 
+#include "rb_call_credentials.h"
 #include "rb_grpc.h"
 
 /* grpc_rb_cChannelCredentials is the ruby class that proxies
@@ -107,6 +109,22 @@ static VALUE grpc_rb_channel_credentials_alloc(VALUE cls) {
   return TypedData_Wrap_Struct(cls, &grpc_rb_channel_credentials_data_type, wrapper);
 }
 
+/* Creates a wrapping object for a given channel credentials. This should only
+ * be called with grpc_channel_credentials objects that are not already
+ * associated with any Ruby object. */
+VALUE grpc_rb_wrap_channel_credentials(grpc_channel_credentials *c) {
+  VALUE rb_wrapper;
+  grpc_rb_channel_credentials *wrapper;
+  if (c == NULL) {
+    return Qnil;
+  }
+  rb_wrapper = grpc_rb_channel_credentials_alloc(grpc_rb_cChannelCredentials);
+  TypedData_Get_Struct(rb_wrapper, grpc_rb_channel_credentials,
+                       &grpc_rb_channel_credentials_data_type, wrapper);
+  wrapper->wrapped = c;
+  return rb_wrapper;
+}
+
 /* Clones ChannelCredentials instances.
    Gives ChannelCredentials a consistent implementation of Ruby's object copy/dup
    protocol. */
@@ -148,11 +166,13 @@ static ID id_pem_cert_chain;
 
 /*
   call-seq:
-    creds1 = Credentials.new(pem_root_certs)
+    creds1 = Credentials.new()
+    ...
+    creds2 = Credentials.new(pem_root_certs)
     ...
-    creds2 = Credentials.new(pem_root_certs, pem_private_key,
+    creds3 = Credentials.new(pem_root_certs, pem_private_key,
                              pem_cert_chain)
-    pem_root_certs: (required) PEM encoding of the server root certificate
+    pem_root_certs: (optional) PEM encoding of the server root certificate
     pem_private_key: (optional) PEM encoding of the client's private key
     pem_cert_chain: (optional) PEM encoding of the client's cert chain
     Initializes Credential instances. */
@@ -163,26 +183,23 @@ static VALUE grpc_rb_channel_credentials_init(int argc, VALUE *argv, VALUE self)
   grpc_rb_channel_credentials *wrapper = NULL;
   grpc_channel_credentials *creds = NULL;
   grpc_ssl_pem_key_cert_pair key_cert_pair;
+  const char *pem_root_certs_cstr = NULL;
   MEMZERO(&key_cert_pair, grpc_ssl_pem_key_cert_pair, 1);
-  /* TODO: Remove mandatory arg when we support default roots. */
-  /* "12" == 1 mandatory arg, 2 (credentials) is optional */
-  rb_scan_args(argc, argv, "12", &pem_root_certs, &pem_private_key,
+  /* "03" == no mandatory arg, 3 optional */
+  rb_scan_args(argc, argv, "03", &pem_root_certs, &pem_private_key,
                &pem_cert_chain);
 
   TypedData_Get_Struct(self, grpc_rb_channel_credentials,
                        &grpc_rb_channel_credentials_data_type, wrapper);
-  if (pem_root_certs == Qnil) {
-    rb_raise(rb_eRuntimeError,
-             "could not create a credential: nil pem_root_certs");
-    return Qnil;
+  if (pem_root_certs != Qnil) {
+    pem_root_certs_cstr = RSTRING_PTR(pem_root_certs);
   }
   if (pem_private_key == Qnil && pem_cert_chain == Qnil) {
-    creds =
-        grpc_ssl_credentials_create(RSTRING_PTR(pem_root_certs), NULL, NULL);
+    creds = grpc_ssl_credentials_create(pem_root_certs_cstr, NULL, NULL);
   } else {
     key_cert_pair.private_key = RSTRING_PTR(pem_private_key);
     key_cert_pair.cert_chain = RSTRING_PTR(pem_cert_chain);
-    creds = grpc_ssl_credentials_create(RSTRING_PTR(pem_root_certs),
+    creds = grpc_ssl_credentials_create(pem_root_certs_cstr,
                                         &key_cert_pair, NULL);
   }
   if (creds == NULL) {
@@ -199,6 +216,25 @@ static VALUE grpc_rb_channel_credentials_init(int argc, VALUE *argv, VALUE self)
   return self;
 }
 
+static VALUE grpc_rb_channel_credentials_compose(int argc, VALUE *argv,
+                                                 VALUE self) {
+  grpc_channel_credentials *creds;
+  grpc_call_credentials *other;
+  if (argc == 0) {
+    return self;
+  }
+  creds = grpc_rb_get_wrapped_channel_credentials(self);
+  for (int i = 0; i < argc; i++) {
+    other = grpc_rb_get_wrapped_call_credentials(argv[i]);
+    creds = grpc_composite_channel_credentials_create(creds, other, NULL);
+    if (creds == NULL) {
+      rb_raise(rb_eRuntimeError,
+               "Failed to compose channel and call credentials");
+    }
+  }
+  return grpc_rb_wrap_channel_credentials(creds);
+}
+
 void Init_grpc_channel_credentials() {
   grpc_rb_cChannelCredentials =
       rb_define_class_under(grpc_rb_mGrpcCore, "ChannelCredentials", rb_cObject);
@@ -212,6 +248,8 @@ void Init_grpc_channel_credentials() {
                    grpc_rb_channel_credentials_init, -1);
   rb_define_method(grpc_rb_cChannelCredentials, "initialize_copy",
                    grpc_rb_channel_credentials_init_copy, 1);
+  rb_define_method(grpc_rb_cChannelCredentials, "compose",
+                   grpc_rb_channel_credentials_compose, -1);
 
   id_pem_cert_chain = rb_intern("__pem_cert_chain");
   id_pem_private_key = rb_intern("__pem_private_key");
diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c
index 7c7c2d3440..a752a5f879 100644
--- a/src/ruby/ext/grpc/rb_grpc.c
+++ b/src/ruby/ext/grpc/rb_grpc.c
@@ -41,6 +41,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/time.h>
 #include "rb_call.h"
+#include "rb_call_credentials.h"
 #include "rb_channel.h"
 #include "rb_channel_credentials.h"
 #include "rb_completion_queue.h"
@@ -91,7 +92,7 @@ static ID id_tv_sec;
 static ID id_tv_nsec;
 
 /**
- * grpc_rb_time_timeval creates a time_eval from a ruby time object.
+ * grpc_rb_time_timeval creates a timeval from a ruby time object.
  *
  * This func is copied from ruby source, MRI/source/time.c, which is published
  * under the same license as the ruby.h, on which the entire extensions is
@@ -137,7 +138,7 @@ gpr_timespec grpc_rb_time_timeval(VALUE time, int interval) {
           d += 1;
           f -= 1;
         }
-        t.tv_sec = (time_t)f;
+        t.tv_sec = (gpr_int64)f;
         if (f != t.tv_sec) {
           rb_raise(rb_eRangeError, "%f out of Time range",
                    RFLOAT_VALUE(time));
@@ -318,6 +319,7 @@ void Init_grpc() {
   Init_grpc_channel();
   Init_grpc_completion_queue();
   Init_grpc_call();
+  Init_grpc_call_credentials();
   Init_grpc_channel_credentials();
   Init_grpc_server();
   Init_grpc_server_credentials();
diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb
index 90aaa026ec..13100a614c 100644
--- a/src/ruby/lib/grpc/generic/client_stub.rb
+++ b/src/ruby/lib/grpc/generic/client_stub.rb
@@ -57,21 +57,6 @@ module GRPC
       Core::Channel.new(host, kw, creds)
     end
 
-    def self.update_with_jwt_aud_uri(a_hash, host, method)
-      last_slash_idx, res = method.rindex('/'), a_hash.clone
-      return res if last_slash_idx.nil?
-      service_name = method[0..(last_slash_idx - 1)]
-      res[:jwt_aud_uri] = "https://#{host}#{service_name}"
-      res
-    end
-
-    # check_update_metadata is used by #initialize verify that it's a Proc.
-    def self.check_update_metadata(update_metadata)
-      return update_metadata if update_metadata.nil?
-      fail(TypeError, '!is_a?Proc') unless update_metadata.is_a?(Proc)
-      update_metadata
-    end
-
     # Allows users of the stub to modify the propagate mask.
     #
     # This is an advanced feature for use when making calls to another gRPC
@@ -99,29 +84,21 @@ module GRPC
     # - :timeout
     # when present, this is the default timeout used for calls
     #
-    # - :update_metadata
-    # when present, this a func that takes a hash and returns a hash
-    # it can be used to update metadata, i.e, remove, or amend
-    # metadata values.
-    #
     # @param host [String] the host the stub connects to
     # @param q [Core::CompletionQueue] used to wait for events
     # @param channel_override [Core::Channel] a pre-created channel
     # @param timeout [Number] the default timeout to use in requests
     # @param creds [Core::ChannelCredentials] the channel credentials
-    # @param update_metadata a func that updates metadata as described above
     # @param kw [KeywordArgs]the channel arguments
     def initialize(host, q,
                    channel_override: nil,
                    timeout: nil,
                    creds: nil,
                    propagate_mask: nil,
-                   update_metadata: nil,
                    **kw)
       fail(TypeError, '!CompletionQueue') unless q.is_a?(Core::CompletionQueue)
       @queue = q
       @ch = ClientStub.setup_channel(channel_override, host, creds, **kw)
-      @update_metadata = ClientStub.check_update_metadata(update_metadata)
       alt_host = kw[Core::Channel::SSL_TARGET]
       @host = alt_host.nil? ? host : alt_host
       @propagate_mask = propagate_mask
@@ -166,6 +143,8 @@ module GRPC
     # @param deadline [Time] (optional) the time the request should complete
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
+    # @param credentials [Core::CallCredentials] credentials to use when making
+    #   the call
     # @param return_op [true|false] return an Operation if true
     # @return [Object] the response received from the server
     def request_response(method, req, marshal, unmarshal,
@@ -173,19 +152,20 @@ module GRPC
                          timeout: nil,
                          return_op: false,
                          parent: nil,
+                         credentials: nil,
                          **kw)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
                           timeout: timeout,
-                          parent: parent)
-      md = update_metadata(kw, method)
-      return c.request_response(req, **md) unless return_op
+                          parent: parent,
+                          credentials: credentials)
+      return c.request_response(req, **kw) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #request_response.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.request_response(req, **md)
+        c.request_response(req, **kw)
       end
       op
     end
@@ -234,25 +214,28 @@ module GRPC
     # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
+    # @param credentials [Core::CallCredentials] credentials to use when making
+    #   the call
     # @return [Object|Operation] the response received from the server
     def client_streamer(method, requests, marshal, unmarshal,
                         deadline: nil,
                         timeout: nil,
                         return_op: false,
                         parent: nil,
+                        credentials: nil,
                         **kw)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
                           timeout: timeout,
-                          parent: parent)
-      md = update_metadata(kw, method)
-      return c.client_streamer(requests, **md) unless return_op
+                          parent: parent,
+                          credentials: credentials)
+      return c.client_streamer(requests, **kw) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #client_streamer.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.client_streamer(requests, **md)
+        c.client_streamer(requests, **kw)
       end
       op
     end
@@ -309,6 +292,8 @@ module GRPC
     # @param return_op [true|false]return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
+    # @param credentials [Core::CallCredentials] credentials to use when making
+    #   the call
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|Operation|nil] as discussed above
     def server_streamer(method, req, marshal, unmarshal,
@@ -316,20 +301,21 @@ module GRPC
                         timeout: nil,
                         return_op: false,
                         parent: nil,
+                        credentials: nil,
                         **kw,
                         &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
                           timeout: timeout,
-                          parent: parent)
-      md = update_metadata(kw, method)
-      return c.server_streamer(req, **md, &blk) unless return_op
+                          parent: parent,
+                          credentials: credentials)
+      return c.server_streamer(req, **kw, &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #server_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.server_streamer(req, **md, &blk)
+        c.server_streamer(req, **kw, &blk)
       end
       op
     end
@@ -424,6 +410,8 @@ module GRPC
     # @param deadline [Time] (optional) the time the request should complete
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
+    # @param credentials [Core::CallCredentials] credentials to use when making
+    #   the call
     # @param return_op [true|false] return an Operation if true
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|nil|Operation] as discussed above
@@ -432,36 +420,28 @@ module GRPC
                       timeout: nil,
                       return_op: false,
                       parent: nil,
+                      credentials: nil,
                       **kw,
                       &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
                           timeout: timeout,
-                          parent: parent)
-      md = update_metadata(kw, method)
-      return c.bidi_streamer(requests, **md, &blk) unless return_op
+                          parent: parent,
+                          credentials: credentials)
+
+      return c.bidi_streamer(requests, **kw, &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #bidi_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.bidi_streamer(requests, **md, &blk)
+        c.bidi_streamer(requests, **kw, &blk)
       end
       op
     end
 
     private
 
-    def update_metadata(kw, method)
-      return kw if @update_metadata.nil?
-      just_jwt_uri = self.class.update_with_jwt_aud_uri({}, @host, method)
-      updated = @update_metadata.call(just_jwt_uri)
-
-      # keys should be lowercase
-      updated = Hash[updated.each_pair.map { |k, v|  [k.downcase, v] }]
-      kw.merge(updated)
-    end
-
     # Creates a new active stub
     #
     # @param method [string] the method being called.
@@ -473,7 +453,8 @@ module GRPC
     def new_active_call(method, marshal, unmarshal,
                         deadline: nil,
                         timeout: nil,
-                        parent: nil)
+                        parent: nil,
+                        credentials: nil)
       if deadline.nil?
         deadline = from_relative_time(timeout.nil? ? @timeout : timeout)
       end
@@ -483,6 +464,7 @@ module GRPC
                              method,
                              nil, # host use nil,
                              deadline)
+      call.set_credentials credentials unless credentials.nil?
       ActiveCall.new(call, @queue, marshal, unmarshal, deadline, started: false)
     end
   end
diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb
index 0e318bd53b..410156ff03 100644
--- a/src/ruby/lib/grpc/generic/rpc_server.rb
+++ b/src/ruby/lib/grpc/generic/rpc_server.rb
@@ -48,6 +48,8 @@ module GRPC
         return false
       when 'TERM'
         return false
+      when nil
+        return true
       end
     end
     true
diff --git a/src/ruby/pb/test/client.rb b/src/ruby/pb/test/client.rb
index 329e2dc98b..6cc616e5cb 100755
--- a/src/ruby/pb/test/client.rb
+++ b/src/ruby/pb/test/client.rb
@@ -93,13 +93,6 @@ def load_test_certs
   files.map { |f| File.open(File.join(data_dir, f)).read }
 end
 
-# loads the certificates used to access the test server securely.
-def load_prod_cert
-  fail 'could not find a production cert' if ENV['SSL_CERT_FILE'].nil?
-  GRPC.logger.info("loading prod certs from #{ENV['SSL_CERT_FILE']}")
-  File.open(ENV['SSL_CERT_FILE']).read
-end
-
 # creates SSL Credentials from the test certificates.
 def test_creds
   certs = load_test_certs
@@ -108,8 +101,7 @@ end
 
 # creates SSL Credentials from the production certificates.
 def prod_creds
-  cert_text = load_prod_cert
-  GRPC::Core::ChannelCredentials.new(cert_text)
+  GRPC::Core::ChannelCredentials.new()
 end
 
 # creates the SSL Credentials.
@@ -132,7 +124,8 @@ def create_stub(opts)
     if wants_creds.include?(opts.test_case)
       unless opts.oauth_scope.nil?
         auth_creds = Google::Auth.get_application_default(opts.oauth_scope)
-        stub_opts[:update_metadata] = auth_creds.updater_proc
+        call_creds = GRPC::Core::CallCredentials.new(auth_creds.updater_proc)
+        stub_opts[:creds] = stub_opts[:creds].compose call_creds
       end
     end
 
@@ -141,12 +134,14 @@ def create_stub(opts)
       kw = auth_creds.updater_proc.call({})  # gives as an auth token
 
       # use a metadata update proc that just adds the auth token.
-      stub_opts[:update_metadata] = proc { |md| md.merge(kw) }
+      call_creds = GRPC::Core::CallCredentials.new(proc { |md| md.merge(kw) })
+      stub_opts[:creds] = stub_opts[:creds].compose call_creds
     end
 
     if opts.test_case == 'jwt_token_creds'  # don't use a scope
       auth_creds = Google::Auth.get_application_default
-      stub_opts[:update_metadata] = auth_creds.updater_proc
+      call_creds = GRPC::Core::CallCredentials.new(auth_creds.updater_proc)
+      stub_opts[:creds] = stub_opts[:creds].compose call_creds
     end
 
     GRPC.logger.info("... connecting securely to #{address}")
diff --git a/src/ruby/spec/call_credentials_spec.rb b/src/ruby/spec/call_credentials_spec.rb
new file mode 100644
index 0000000000..32a0ad44b7
--- /dev/null
+++ b/src/ruby/spec/call_credentials_spec.rb
@@ -0,0 +1,57 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'grpc'
+
+describe GRPC::Core::CallCredentials do
+  CallCredentials = GRPC::Core::CallCredentials
+
+  let(:auth_proc) { proc { { 'plugin_key' => 'plugin_value' } } }
+
+  describe '#new' do
+    it 'can successfully create a CallCredentials from a proc' do
+      expect { CallCredentials.new(auth_proc) }.not_to raise_error
+    end
+  end
+
+  describe '#compose' do
+    it 'can compose with another CallCredentials' do
+      creds1 = CallCredentials.new(auth_proc)
+      creds2 = CallCredentials.new(auth_proc)
+      expect { creds1.compose creds2 }.not_to raise_error
+    end
+
+    it 'can compose with multiple CallCredentials' do
+      creds1 = CallCredentials.new(auth_proc)
+      creds2 = CallCredentials.new(auth_proc)
+      creds3 = CallCredentials.new(auth_proc)
+      expect { creds1.compose(creds2, creds3) }.not_to raise_error
+    end
+  end
+end
diff --git a/src/ruby/spec/call_spec.rb b/src/ruby/spec/call_spec.rb
index dd3c45f754..6629570fba 100644
--- a/src/ruby/spec/call_spec.rb
+++ b/src/ruby/spec/call_spec.rb
@@ -144,6 +144,15 @@ describe GRPC::Core::Call do
     end
   end
 
+  describe '#set_credentials!' do
+    it 'can set a valid CallCredentials object' do
+      call = make_test_call
+      auth_proc = proc { { 'plugin_key' => 'plugin_value' } }
+      creds = GRPC::Core::CallCredentials.new auth_proc
+      expect { call.set_credentials! creds }.not_to raise_error
+    end
+  end
+
   def make_test_call
     @ch.create_call(client_queue, nil, nil, 'dummy_method', nil, deadline)
   end
diff --git a/src/ruby/spec/channel_credentials_spec.rb b/src/ruby/spec/channel_credentials_spec.rb
index b2bdf7032e..0bcfc752a0 100644
--- a/src/ruby/spec/channel_credentials_spec.rb
+++ b/src/ruby/spec/channel_credentials_spec.rb
@@ -31,6 +31,7 @@ require 'grpc'
 
 describe GRPC::Core::ChannelCredentials do
   ChannelCredentials = GRPC::Core::ChannelCredentials
+  CallCredentials = GRPC::Core::CallCredentials
 
   def load_test_certs
     test_root = File.join(File.dirname(__FILE__), 'testdata')
@@ -54,10 +55,43 @@ describe GRPC::Core::ChannelCredentials do
       expect { ChannelCredentials.new(root_cert) }.not_to raise_error
     end
 
-    it 'cannot be constructed with a nil server roots' do
+    it 'can be constructed with a nil server roots' do
       _, client_key, client_chain = load_test_certs
       blk = proc { ChannelCredentials.new(nil, client_key, client_chain) }
-      expect(&blk).to raise_error
+      expect(&blk).not_to raise_error
+    end
+
+    it 'can be constructed with no params' do
+      blk = proc { ChannelCredentials.new(nil) }
+      expect(&blk).not_to raise_error
+    end
+  end
+
+  describe '#compose' do
+    it 'can compose with a CallCredentials' do
+      certs = load_test_certs
+      channel_creds = ChannelCredentials.new(*certs)
+      auth_proc = proc { { 'plugin_key' => 'plugin_value' } }
+      call_creds = CallCredentials.new auth_proc
+      expect { channel_creds.compose call_creds }.not_to raise_error
+    end
+
+    it 'can compose with multiple CallCredentials' do
+      certs = load_test_certs
+      channel_creds = ChannelCredentials.new(*certs)
+      auth_proc = proc { { 'plugin_key' => 'plugin_value' } }
+      call_creds1 = CallCredentials.new auth_proc
+      call_creds2 = CallCredentials.new auth_proc
+      expect do
+        channel_creds.compose(call_creds1, call_creds2)
+      end.not_to raise_error
+    end
+
+    it 'cannot compose with ChannelCredentials' do
+      certs = load_test_certs
+      channel_creds1 = ChannelCredentials.new(*certs)
+      channel_creds2 = ChannelCredentials.new(*certs)
+      expect { channel_creds1.compose channel_creds2 }.to raise_error(TypeError)
     end
   end
 end
diff --git a/src/ruby/spec/client_server_spec.rb b/src/ruby/spec/client_server_spec.rb
index 734f176e94..7cce2076c9 100644
--- a/src/ruby/spec/client_server_spec.rb
+++ b/src/ruby/spec/client_server_spec.rb
@@ -413,6 +413,8 @@ describe 'the http client/server' do
 end
 
 describe 'the secure http client/server' do
+  include_context 'setup: tags'
+
   def load_test_certs
     test_root = File.join(File.dirname(__FILE__), 'testdata')
     files = ['ca.pem', 'server1.key', 'server1.pem']
@@ -443,4 +445,31 @@ describe 'the secure http client/server' do
 
   it_behaves_like 'GRPC metadata delivery works OK' do
   end
+
+  it 'modifies metadata with CallCredentials' do
+    auth_proc = proc { { 'k1' => 'updated-v1' } }
+    call_creds = GRPC::Core::CallCredentials.new(auth_proc)
+    md = { 'k2' => 'v2' }
+    expected_md = { 'k1' => 'updated-v1', 'k2' => 'v2' }
+    recvd_rpc = nil
+    rcv_thread = Thread.new do
+      recvd_rpc = @server.request_call(@server_queue, @server_tag, deadline)
+    end
+
+    call = new_client_call
+    call.set_credentials! call_creds
+    client_ops = {
+      CallOps::SEND_INITIAL_METADATA => md
+    }
+    batch_result = call.run_batch(@client_queue, @client_tag, deadline,
+                                  client_ops)
+    expect(batch_result.send_metadata).to be true
+
+    # confirm the server can receive the client metadata
+    rcv_thread.join
+    expect(recvd_rpc).to_not eq nil
+    recvd_md = recvd_rpc.metadata
+    replace_symbols = Hash[expected_md.each_pair.collect { |x, y| [x.to_s, y] }]
+    expect(recvd_md).to eq(recvd_md.merge(replace_symbols))
+  end
 end
diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb
index da5bc6c9e5..40550230dd 100644
--- a/src/ruby/spec/generic/client_stub_spec.rb
+++ b/src/ruby/spec/generic/client_stub_spec.rb
@@ -145,34 +145,6 @@ describe 'ClientStub' do
         th.join
       end
 
-      it 'should update the sent metadata with a provided metadata updater' do
-        server_port = create_test_server
-        host = "localhost:#{server_port}"
-        th = run_request_response(@sent_msg, @resp, @pass,
-                                  k1: 'updated-v1', k2: 'v2')
-        update_md = proc do |md|
-          md[:k1] = 'updated-v1'
-          md
-        end
-        stub = GRPC::ClientStub.new(host, @cq, update_metadata: update_md)
-        expect(get_response(stub)).to eq(@resp)
-        th.join
-      end
-
-      it 'should downcase the keys provided by the metadata updater' do
-        server_port = create_test_server
-        host = "localhost:#{server_port}"
-        th = run_request_response(@sent_msg, @resp, @pass,
-                                  k1: 'downcased-key-v1', k2: 'v2')
-        update_md = proc do |md|
-          md[:K1] = 'downcased-key-v1'
-          md
-        end
-        stub = GRPC::ClientStub.new(host, @cq, update_metadata: update_md)
-        expect(get_response(stub)).to eq(@resp)
-        th.join
-      end
-
       it 'should send a request when configured using an override channel' do
         server_port = create_test_server
         alt_host = "localhost:#{server_port}"
@@ -241,20 +213,6 @@ describe 'ClientStub' do
         th.join
       end
 
-      it 'should update the sent metadata with a provided metadata updater' do
-        server_port = create_test_server
-        host = "localhost:#{server_port}"
-        th = run_client_streamer(@sent_msgs, @resp, @pass,
-                                 k1: 'updated-v1', k2: 'v2')
-        update_md = proc do |md|
-          md[:k1] = 'updated-v1'
-          md
-        end
-        stub = GRPC::ClientStub.new(host, @cq, update_metadata: update_md)
-        expect(get_response(stub)).to eq(@resp)
-        th.join
-      end
-
       it 'should raise an error if the status is not ok' do
         server_port = create_test_server
         host = "localhost:#{server_port}"
@@ -323,21 +281,6 @@ describe 'ClientStub' do
         expect { e.collect { |r| r } }.to raise_error(GRPC::BadStatus)
         th.join
       end
-
-      it 'should update the sent metadata with a provided metadata updater' do
-        server_port = create_test_server
-        host = "localhost:#{server_port}"
-        th = run_server_streamer(@sent_msg, @replys, @pass,
-                                 k1: 'updated-v1', k2: 'v2')
-        update_md = proc do |md|
-          md[:k1] = 'updated-v1'
-          md
-        end
-        stub = GRPC::ClientStub.new(host, @cq, update_metadata: update_md)
-        e = get_responses(stub)
-        expect(e.collect { |r| r }).to eq(@replys)
-        th.join
-      end
     end
 
     describe 'without a call operation' do
diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb
index efe07f734e..d95a021311 100644
--- a/src/ruby/spec/generic/rpc_server_spec.rb
+++ b/src/ruby/spec/generic/rpc_server_spec.rb
@@ -422,25 +422,6 @@ describe GRPC::RpcServer do
         t.join
       end
 
-      it 'should receive updated metadata', server: true do
-        service = EchoService.new
-        @srv.handle(service)
-        t = Thread.new { @srv.run }
-        @srv.wait_till_running
-        req = EchoMsg.new
-        client_opts[:update_metadata] = proc do |md|
-          md[:k1] = 'updated-v1'
-          md
-        end
-        stub = EchoStub.new(@host, **client_opts)
-        expect(stub.an_rpc(req, k1: 'v1', k2: 'v2')).to be_a(EchoMsg)
-        wanted_md = [{ 'k1' => 'updated-v1', 'k2' => 'v2',
-                       'jwt_aud_uri' => "https://#{@host}/EchoService" }]
-        check_md(wanted_md, service.received_md)
-        @srv.stop
-        t.join
-      end
-
       it 'should handle multiple parallel requests', server: true do
         @srv.handle(EchoService)
         t = Thread.new { @srv.run }