Merge branch 'compression-accept-encoding' into compression-interop

107 files changed, 3276 insertions, 1437 deletions

diff --git a/src/core/census/grpc_context.c b/src/core/census/grpc_context.c
index 0ed63469b6..11f1eb3d5d 100644
--- a/src/core/census/grpc_context.c
+++ b/src/core/census/grpc_context.c
@@ -32,14 +32,15 @@
  */
 
 #include <grpc/census.h>
-#include "src/core/census/grpc_context.h"
+#include <grpc/grpc.h>
+#include "src/core/surface/call.h"
 
 static void grpc_census_context_destroy(void *context) {
   census_context_destroy((census_context *)context);
 }
 
 void grpc_census_call_set_context(grpc_call *call, census_context *context) {
-  if (!census_available()) {
+  if (census_enabled() == CENSUS_FEATURE_NONE) {
     return;
   }
   if (context == NULL) {
diff --git a/src/core/census/initialize.c b/src/core/census/initialize.c
index 8016520641..8d60f790eb 100644
--- a/src/core/census/initialize.c
+++ b/src/core/census/initialize.c
@@ -33,20 +33,25 @@
 
 #include <grpc/census.h>
 
-static int census_fns_enabled = CENSUS_NONE;
+static int features_enabled = CENSUS_FEATURE_NONE;
 
-int census_initialize(int functions) {
-  if (census_fns_enabled != CENSUS_NONE) {
+int census_initialize(int features) {
+  if (features_enabled != CENSUS_FEATURE_NONE) {
     return 1;
   }
-  if (functions != CENSUS_NONE) {
+  if (features != CENSUS_FEATURE_NONE) {
     return 1;
   } else {
-    census_fns_enabled = functions;
+    features_enabled = features;
     return 0;
   }
 }
 
-void census_shutdown() { census_fns_enabled = CENSUS_NONE; }
+void census_shutdown(void) { features_enabled = CENSUS_FEATURE_NONE; }
 
-int census_available() { return (census_fns_enabled != CENSUS_NONE); }
+int census_supported(void) {
+  /* TODO(aveitch): improve this as we implement features... */
+  return CENSUS_FEATURE_NONE;
+}
+
+int census_enabled(void) { return features_enabled; }
diff --git a/src/core/httpcli/httpcli_security_connector.h b/src/core/census/record_stat.c
index c50f25905e..3dd918618b 100644
--- a/src/core/httpcli/httpcli_security_connector.h
+++ b/src/core/census/record_stat.c
@@ -31,13 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_SECURITY_CONNECTOR_H
-#define GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_SECURITY_CONNECTOR_H
+#include <grpc/census.h>
+#include "src/core/census/rpc_stat_id.h"
 
-#include "src/core/security/security_connector.h"
-
-grpc_security_status grpc_httpcli_ssl_channel_security_connector_create(
-    const unsigned char *pem_root_certs, size_t pem_root_certs_size,
-    const char *secure_peer_name, grpc_channel_security_connector **sc);
-
-#endif  /* GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_SECURITY_CONNECTOR_H */
+void census_record_stat(census_context *context, census_stat *stats,
+                        size_t nstats) {}
diff --git a/src/core/census/rpc_stat_id.h b/src/core/census/rpc_stat_id.h
new file mode 100644
index 0000000000..fc0aa6f43f
--- /dev/null
+++ b/src/core/census/rpc_stat_id.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef CENSUS_RPC_STAT_ID_H
+#define CENSUS_RPC_STAT_ID_H
+
+/* Stats ID's used for RPC measurements. */
+#define CENSUS_INVALID_STAT_ID 0     /* ID 0 is always invalid */
+#define CENSUS_RPC_CLIENT_REQUESTS 1 /* Count of client requests sent. */
+#define CENSUS_RPC_SERVER_REQUESTS 2 /* Count of server requests sent. */
+#define CENSUS_RPC_CLIENT_ERRORS 3   /* Client error counts. */
+#define CENSUS_RPC_SERVER_ERRORS 4   /* Server error counts. */
+#define CENSUS_RPC_CLIENT_LATENCY 5  /* Client side request latency. */
+#define CENSUS_RPC_SERVER_LATENCY 6  /* Server side request latency. */
+
+#endif /* CENSUS_RPC_STAT_ID_H */
diff --git a/src/core/channel/channel_stack.c b/src/core/channel/channel_stack.c
index e38dcb58b7..cd7c182ef2 100644
--- a/src/core/channel/channel_stack.c
+++ b/src/core/channel/channel_stack.c
@@ -191,6 +191,11 @@ void grpc_call_next_op(grpc_call_element *elem, grpc_transport_stream_op *op) {
   next_elem->filter->start_transport_stream_op(next_elem, op);
 }
 
+char *grpc_call_next_get_peer(grpc_call_element *elem) {
+  grpc_call_element *next_elem = elem + 1;
+  return next_elem->filter->get_peer(next_elem);
+}
+
 void grpc_channel_next_op(grpc_channel_element *elem, grpc_transport_op *op) {
   grpc_channel_element *next_elem = elem + 1;
   next_elem->filter->start_transport_op(next_elem, op);
diff --git a/src/core/channel/channel_stack.h b/src/core/channel/channel_stack.h
index 785be8925b..4a608b956e 100644
--- a/src/core/channel/channel_stack.h
+++ b/src/core/channel/channel_stack.h
@@ -104,6 +104,9 @@ typedef struct {
      The filter does not need to do any chaining */
   void (*destroy_channel_elem)(grpc_channel_element *elem);
 
+  /* Implement grpc_call_get_peer() */
+  char *(*get_peer)(grpc_call_element *elem);
+
   /* The name of this filter */
   const char *name;
 } grpc_channel_filter;
@@ -173,6 +176,8 @@ void grpc_call_next_op(grpc_call_element *elem, grpc_transport_stream_op *op);
 /* Call the next operation (depending on call directionality) in a channel
    stack */
 void grpc_channel_next_op(grpc_channel_element *elem, grpc_transport_op *op);
+/* Pass through a request to get_peer to the next child element */
+char *grpc_call_next_get_peer(grpc_call_element *elem);
 
 /* Given the top element of a channel stack, get the channel stack itself */
 grpc_channel_stack *grpc_channel_stack_from_top_element(
diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c
index c1aa580b2d..a293c93ec6 100644
--- a/src/core/channel/client_channel.c
+++ b/src/core/channel/client_channel.c
@@ -40,7 +40,6 @@
 #include "src/core/channel/connected_channel.h"
 #include "src/core/surface/channel.h"
 #include "src/core/iomgr/iomgr.h"
-#include "src/core/iomgr/pollset_set.h"
 #include "src/core/support/string.h"
 #include "src/core/transport/connectivity_state.h"
 #include <grpc/support/alloc.h>
@@ -57,6 +56,8 @@ typedef struct {
   grpc_mdctx *mdctx;
   /** resolver for this channel */
   grpc_resolver *resolver;
+  /** have we started resolving this channel */
+  int started_resolving;
   /** master channel - the grpc_channel instance that ultimately owns
       this channel_data via its channel stack.
       We occasionally use this to bump the refcount on the master channel
@@ -77,8 +78,22 @@ typedef struct {
   grpc_iomgr_closure on_config_changed;
   /** connectivity state being tracked */
   grpc_connectivity_state_tracker state_tracker;
+  /** when an lb_policy arrives, should we try to exit idle */
+  int exit_idle_when_lb_policy_arrives;
+  /** pollset_set of interested parties in a new connection */
+  grpc_pollset_set pollset_set;
 } channel_data;
 
+/** We create one watcher for each new lb_policy that is returned from a resolver,
+    to watch for state changes from the lb_policy. When a state change is seen, we
+    update the channel, and create a new watcher */
+typedef struct {
+  channel_data *chand;
+  grpc_iomgr_closure on_changed;
+  grpc_connectivity_state state;
+  grpc_lb_policy *lb_policy;
+} lb_policy_connectivity_watcher;
+
 typedef enum {
   CALL_CREATED,
   CALL_WAITING_FOR_SEND,
@@ -265,6 +280,26 @@ static grpc_iomgr_closure *merge_into_waiting_op(
   return consumed_op;
 }
 
+static char *cc_get_peer(grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
+  channel_data *chand = elem->channel_data;
+  grpc_subchannel_call *subchannel_call;
+  char *result;
+
+  gpr_mu_lock(&calld->mu_state);
+  if (calld->state == CALL_ACTIVE) {
+    subchannel_call = calld->subchannel_call;
+    GRPC_SUBCHANNEL_CALL_REF(subchannel_call, "get_peer");
+    gpr_mu_unlock(&calld->mu_state);
+    result = grpc_subchannel_call_get_peer(subchannel_call);
+    GRPC_SUBCHANNEL_CALL_UNREF(subchannel_call, "get_peer");
+    return result;
+  } else {
+    gpr_mu_unlock(&calld->mu_state);
+    return grpc_channel_get_target(chand->master);
+  }
+}
+
 static void perform_transport_stream_op(grpc_call_element *elem,
                                         grpc_transport_stream_op *op,
                                         int continuation) {
@@ -365,6 +400,13 @@ static void perform_transport_stream_op(grpc_call_element *elem,
           } else if (chand->resolver != NULL) {
             calld->state = CALL_WAITING_FOR_CONFIG;
             add_to_lb_policy_wait_queue_locked_state_config(elem);
+            if (!chand->started_resolving && chand->resolver != NULL) {
+              GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+              chand->started_resolving = 1;
+              grpc_resolver_next(chand->resolver,
+                                 &chand->incoming_configuration,
+                                 &chand->on_config_changed);
+            }
             gpr_mu_unlock(&chand->mu_config);
             gpr_mu_unlock(&calld->mu_state);
           } else {
@@ -388,16 +430,53 @@ static void cc_start_transport_stream_op(grpc_call_element *elem,
   perform_transport_stream_op(elem, op, 0);
 }
 
+static void watch_lb_policy(channel_data *chand, grpc_lb_policy *lb_policy, grpc_connectivity_state current_state);
+
+static void on_lb_policy_state_changed(void *arg, int iomgr_success) {
+  lb_policy_connectivity_watcher *w = arg;
+
+  gpr_mu_lock(&w->chand->mu_config);
+  /* check if the notification is for a stale policy */
+  if (w->lb_policy == w->chand->lb_policy) {
+    grpc_connectivity_state_set(&w->chand->state_tracker, w->state,
+                                "lb_changed");
+    if (w->state != GRPC_CHANNEL_FATAL_FAILURE) {
+      watch_lb_policy(w->chand, w->lb_policy, w->state);
+    }
+  }
+  gpr_mu_unlock(&w->chand->mu_config);
+
+  GRPC_CHANNEL_INTERNAL_UNREF(w->chand->master, "watch_lb_policy");
+  gpr_free(w);
+}
+
+static void watch_lb_policy(channel_data *chand, grpc_lb_policy *lb_policy, grpc_connectivity_state current_state) {
+  lb_policy_connectivity_watcher *w = gpr_malloc(sizeof(*w));
+  GRPC_CHANNEL_INTERNAL_REF(chand->master, "watch_lb_policy");
+
+  w->chand = chand;
+  grpc_iomgr_closure_init(&w->on_changed, on_lb_policy_state_changed, w);
+  w->state = current_state;
+  w->lb_policy = lb_policy;
+  grpc_lb_policy_notify_on_state_change(lb_policy, &w->state, &w->on_changed);
+}
+
 static void cc_on_config_changed(void *arg, int iomgr_success) {
   channel_data *chand = arg;
   grpc_lb_policy *lb_policy = NULL;
   grpc_lb_policy *old_lb_policy;
   grpc_resolver *old_resolver;
   grpc_iomgr_closure *wakeup_closures = NULL;
+  grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE;
+  int exit_idle = 0;
 
   if (chand->incoming_configuration != NULL) {
     lb_policy = grpc_client_config_get_lb_policy(chand->incoming_configuration);
-    GRPC_LB_POLICY_REF(lb_policy, "channel");
+    if (lb_policy != NULL) {
+      GRPC_LB_POLICY_REF(lb_policy, "channel");
+      GRPC_LB_POLICY_REF(lb_policy, "config_change");
+      state = grpc_lb_policy_check_connectivity(lb_policy);
+    }
 
     grpc_client_config_unref(chand->incoming_configuration);
   }
@@ -411,13 +490,12 @@ static void cc_on_config_changed(void *arg, int iomgr_success) {
     wakeup_closures = chand->waiting_for_config_closures;
     chand->waiting_for_config_closures = NULL;
   }
-  gpr_mu_unlock(&chand->mu_config);
-
-  if (old_lb_policy) {
-    GRPC_LB_POLICY_UNREF(old_lb_policy, "channel");
+  if (lb_policy != NULL && chand->exit_idle_when_lb_policy_arrives) {
+    GRPC_LB_POLICY_REF(lb_policy, "exit_idle");
+    exit_idle = 1;
+    chand->exit_idle_when_lb_policy_arrives = 0;
   }
 
-  gpr_mu_lock(&chand->mu_config);
   if (iomgr_success && chand->resolver) {
     grpc_resolver *resolver = chand->resolver;
     GRPC_RESOLVER_REF(resolver, "channel-next");
@@ -426,11 +504,16 @@ static void cc_on_config_changed(void *arg, int iomgr_success) {
     grpc_resolver_next(resolver, &chand->incoming_configuration,
                        &chand->on_config_changed);
     GRPC_RESOLVER_UNREF(resolver, "channel-next");
+    grpc_connectivity_state_set(&chand->state_tracker, state,
+                                "new_lb+resolver");
+    if (lb_policy != NULL) {
+      watch_lb_policy(chand, lb_policy, state);
+    }
   } else {
     old_resolver = chand->resolver;
     chand->resolver = NULL;
     grpc_connectivity_state_set(&chand->state_tracker,
-                                GRPC_CHANNEL_FATAL_FAILURE);
+                                GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone");
     gpr_mu_unlock(&chand->mu_config);
     if (old_resolver != NULL) {
       grpc_resolver_shutdown(old_resolver);
@@ -438,12 +521,24 @@ static void cc_on_config_changed(void *arg, int iomgr_success) {
     }
   }
 
+  if (exit_idle) {
+    grpc_lb_policy_exit_idle(lb_policy);
+    GRPC_LB_POLICY_UNREF(lb_policy, "exit_idle");
+  }
+
+  if (old_lb_policy != NULL) {
+    GRPC_LB_POLICY_UNREF(old_lb_policy, "channel");
+  }
+
   while (wakeup_closures) {
     grpc_iomgr_closure *next = wakeup_closures->next;
-    grpc_iomgr_add_callback(wakeup_closures);
+    wakeup_closures->cb(wakeup_closures->cb_arg, 1);
     wakeup_closures = next;
   }
 
+  if (lb_policy != NULL) {
+    GRPC_LB_POLICY_UNREF(lb_policy, "config_change");
+  }
   GRPC_CHANNEL_INTERNAL_UNREF(chand->master, "resolver");
 }
 
@@ -467,20 +562,22 @@ static void cc_start_transport_op(grpc_channel_element *elem,
     op->connectivity_state = NULL;
   }
 
+  if (!is_empty(op, sizeof(*op))) {
+    lb_policy = chand->lb_policy;
+    if (lb_policy) {
+      GRPC_LB_POLICY_REF(lb_policy, "broadcast");
+    }
+  }
+
   if (op->disconnect && chand->resolver != NULL) {
     grpc_connectivity_state_set(&chand->state_tracker,
-                                GRPC_CHANNEL_FATAL_FAILURE);
+                                GRPC_CHANNEL_FATAL_FAILURE, "disconnect");
     destroy_resolver = chand->resolver;
     chand->resolver = NULL;
     if (chand->lb_policy != NULL) {
       grpc_lb_policy_shutdown(chand->lb_policy);
-    }
-  }
-
-  if (!is_empty(op, sizeof(*op))) {
-    lb_policy = chand->lb_policy;
-    if (lb_policy) {
-      GRPC_LB_POLICY_REF(lb_policy, "broadcast");
+      GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
+      chand->lb_policy = NULL;
     }
   }
   gpr_mu_unlock(&chand->mu_config);
@@ -561,10 +658,11 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
   gpr_mu_init(&chand->mu_config);
   chand->mdctx = metadata_context;
   chand->master = master;
+  grpc_pollset_set_init(&chand->pollset_set);
   grpc_iomgr_closure_init(&chand->on_config_changed, cc_on_config_changed,
                           chand);
 
-  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE);
+  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE, "client_channel");
 }
 
 /* Destructor for channel_data */
@@ -578,6 +676,8 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
   if (chand->lb_policy != NULL) {
     GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
   }
+  grpc_connectivity_state_destroy(&chand->state_tracker);
+  grpc_pollset_set_destroy(&chand->pollset_set);
   gpr_mu_destroy(&chand->mu_config);
 }
 
@@ -590,6 +690,7 @@ const grpc_channel_filter grpc_client_channel_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
+    cc_get_peer,
     "client-channel",
 };
 
@@ -598,10 +699,66 @@ void grpc_client_channel_set_resolver(grpc_channel_stack *channel_stack,
   /* post construction initialization: set the transport setup pointer */
   grpc_channel_element *elem = grpc_channel_stack_last_element(channel_stack);
   channel_data *chand = elem->channel_data;
+  gpr_mu_lock(&chand->mu_config);
   GPR_ASSERT(!chand->resolver);
   chand->resolver = resolver;
-  GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
   GRPC_RESOLVER_REF(resolver, "channel");
-  grpc_resolver_next(resolver, &chand->incoming_configuration,
-                     &chand->on_config_changed);
+  if (chand->waiting_for_config_closures != NULL ||
+      chand->exit_idle_when_lb_policy_arrives) {
+    chand->started_resolving = 1;
+    GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+    grpc_resolver_next(resolver, &chand->incoming_configuration,
+                       &chand->on_config_changed);
+  }
+  gpr_mu_unlock(&chand->mu_config);
+}
+
+grpc_connectivity_state grpc_client_channel_check_connectivity_state(
+    grpc_channel_element *elem, int try_to_connect) {
+  channel_data *chand = elem->channel_data;
+  grpc_connectivity_state out;
+  gpr_mu_lock(&chand->mu_config);
+  out = grpc_connectivity_state_check(&chand->state_tracker);
+  if (out == GRPC_CHANNEL_IDLE && try_to_connect) {
+    if (chand->lb_policy != NULL) {
+      grpc_lb_policy_exit_idle(chand->lb_policy);
+    } else {
+      chand->exit_idle_when_lb_policy_arrives = 1;
+      if (!chand->started_resolving && chand->resolver != NULL) {
+        GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver");
+        chand->started_resolving = 1;
+        grpc_resolver_next(chand->resolver, &chand->incoming_configuration,
+                           &chand->on_config_changed);
+      }
+    }
+  }
+  gpr_mu_unlock(&chand->mu_config);
+  return out;
+}
+
+void grpc_client_channel_watch_connectivity_state(
+    grpc_channel_element *elem, grpc_connectivity_state *state,
+    grpc_iomgr_closure *on_complete) {
+  channel_data *chand = elem->channel_data;
+  gpr_mu_lock(&chand->mu_config);
+  grpc_connectivity_state_notify_on_state_change(&chand->state_tracker, state,
+                                                 on_complete);
+  gpr_mu_unlock(&chand->mu_config);
+}
+
+grpc_pollset_set *grpc_client_channel_get_connecting_pollset_set(grpc_channel_element *elem) {
+  channel_data *chand = elem->channel_data;
+  return &chand->pollset_set;
+}
+
+void grpc_client_channel_add_interested_party(grpc_channel_element *elem,
+                                          grpc_pollset *pollset) {
+  channel_data *chand = elem->channel_data;
+  grpc_pollset_set_add_pollset(&chand->pollset_set, pollset);
+}
+
+void grpc_client_channel_del_interested_party(grpc_channel_element *elem,
+                                          grpc_pollset *pollset) {
+  channel_data *chand = elem->channel_data;
+  grpc_pollset_set_del_pollset(&chand->pollset_set, pollset);
 }
diff --git a/src/core/channel/client_channel.h b/src/core/channel/client_channel.h
index fd2be46145..cd81294eb3 100644
--- a/src/core/channel/client_channel.h
+++ b/src/core/channel/client_channel.h
@@ -52,4 +52,18 @@ extern const grpc_channel_filter grpc_client_channel_filter;
 void grpc_client_channel_set_resolver(grpc_channel_stack *channel_stack,
                                       grpc_resolver *resolver);
 
+grpc_connectivity_state grpc_client_channel_check_connectivity_state(
+    grpc_channel_element *elem, int try_to_connect);
+
+void grpc_client_channel_watch_connectivity_state(
+    grpc_channel_element *elem, grpc_connectivity_state *state,
+    grpc_iomgr_closure *on_complete);
+
+grpc_pollset_set *grpc_client_channel_get_connecting_pollset_set(grpc_channel_element *elem);
+
+void grpc_client_channel_add_interested_party(grpc_channel_element *channel,
+                                          grpc_pollset *pollset);
+void grpc_client_channel_del_interested_party(grpc_channel_element *channel,
+                                          grpc_pollset *pollset);
+
 #endif /* GRPC_INTERNAL_CORE_CHANNEL_CLIENT_CHANNEL_H */
diff --git a/src/core/channel/compress_filter.c b/src/core/channel/compress_filter.c
index 1bd20661d6..2fd4c8cae6 100644
--- a/src/core/channel/compress_filter.c
+++ b/src/core/channel/compress_filter.c
@@ -301,24 +301,24 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
       grpc_channel_args_get_compression_algorithm(args);
 
   channeld->mdstr_request_compression_algorithm_key =
-      grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY);
+      grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY, 0);
 
   channeld->mdstr_outgoing_compression_algorithm_key =
-      grpc_mdstr_from_string(mdctx, "grpc-encoding");
+      grpc_mdstr_from_string(mdctx, "grpc-encoding", 0);
 
   channeld->mdstr_compression_capabilities_key =
-      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding");
+      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding", 0);
 
   for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
-    char *algorith_name;
-    GPR_ASSERT(grpc_compression_algorithm_name(algo_idx, &algorith_name) != 0);
+    char *algorithm_name;
+    GPR_ASSERT(grpc_compression_algorithm_name(algo_idx, &algorithm_name) != 0);
     channeld->mdelem_compression_algorithms[algo_idx] =
         grpc_mdelem_from_metadata_strings(
             mdctx,
             GRPC_MDSTR_REF(channeld->mdstr_outgoing_compression_algorithm_key),
-            grpc_mdstr_from_string(mdctx, algorith_name));
+            grpc_mdstr_from_string(mdctx, algorithm_name, 0));
     if (algo_idx > 0) {
-      supported_algorithms_names[algo_idx-1] = algorith_name;
+      supported_algorithms_names[algo_idx-1] = algorithm_name;
     }
   }
 
@@ -334,7 +334,7 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
       grpc_mdelem_from_metadata_strings(
           mdctx,
           GRPC_MDSTR_REF(channeld->mdstr_compression_capabilities_key),
-          grpc_mdstr_from_string(mdctx, accept_encoding_str));
+          grpc_mdstr_from_string(mdctx, accept_encoding_str, 0));
   gpr_free(accept_encoding_str);
 
   GPR_ASSERT(!is_last);
@@ -364,4 +364,5 @@ const grpc_channel_filter grpc_compress_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
+    grpc_call_next_get_peer,
     "compress"};
diff --git a/src/core/channel/connected_channel.c b/src/core/channel/connected_channel.c
index 34d07de519..b95ed06f2b 100644
--- a/src/core/channel/connected_channel.c
+++ b/src/core/channel/connected_channel.c
@@ -119,6 +119,11 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
   grpc_transport_destroy(cd->transport);
 }
 
+static char *con_get_peer(grpc_call_element *elem) {
+  channel_data *chand = elem->channel_data;
+  return grpc_transport_get_peer(chand->transport);
+}
+
 const grpc_channel_filter grpc_connected_channel_filter = {
     con_start_transport_stream_op,
     con_start_transport_op,
@@ -128,6 +133,7 @@ const grpc_channel_filter grpc_connected_channel_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
+    con_get_peer,
     "connected",
 };
 
diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c
index 6e93103a6a..48c623d359 100644
--- a/src/core/channel/http_client_filter.c
+++ b/src/core/channel/http_client_filter.c
@@ -40,10 +40,12 @@
 typedef struct call_data {
   grpc_linked_mdelem method;
   grpc_linked_mdelem scheme;
+  grpc_linked_mdelem authority;
   grpc_linked_mdelem te_trailers;
   grpc_linked_mdelem content_type;
   grpc_linked_mdelem user_agent;
   int sent_initial_metadata;
+  int sent_authority;
 
   int got_initial_metadata;
   grpc_stream_op_buffer *recv_ops;
@@ -62,6 +64,7 @@ typedef struct channel_data {
   grpc_mdelem *scheme;
   grpc_mdelem *content_type;
   grpc_mdelem *status;
+  grpc_mdelem *default_authority;
   /** complete user agent mdelem */
   grpc_mdelem *user_agent;
 } channel_data;
@@ -100,6 +103,7 @@ static void hc_on_recv(void *user_data, int success) {
 
 static grpc_mdelem *client_strip_filter(void *user_data, grpc_mdelem *md) {
   grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
   channel_data *channeld = elem->channel_data;
   /* eat the things we'd like to set ourselves */
   if (md->key == channeld->method->key) return NULL;
@@ -107,6 +111,10 @@ static grpc_mdelem *client_strip_filter(void *user_data, grpc_mdelem *md) {
   if (md->key == channeld->te_trailers->key) return NULL;
   if (md->key == channeld->content_type->key) return NULL;
   if (md->key == channeld->user_agent->key) return NULL;
+  if (channeld->default_authority &&
+      channeld->default_authority->key == md->key) {
+    calld->sent_authority = 1;
+  }
   return md;
 }
 
@@ -130,6 +138,11 @@ static void hc_mutate_op(grpc_call_element *elem,
                                    GRPC_MDELEM_REF(channeld->method));
       grpc_metadata_batch_add_head(&op->data.metadata, &calld->scheme,
                                    GRPC_MDELEM_REF(channeld->scheme));
+      if (channeld->default_authority && !calld->sent_authority) {
+        grpc_metadata_batch_add_head(
+            &op->data.metadata, &calld->authority,
+            GRPC_MDELEM_REF(channeld->default_authority));
+      }
       grpc_metadata_batch_add_tail(&op->data.metadata, &calld->te_trailers,
                                    GRPC_MDELEM_REF(channeld->te_trailers));
       grpc_metadata_batch_add_tail(&op->data.metadata, &calld->content_type,
@@ -162,6 +175,7 @@ static void init_call_elem(grpc_call_element *elem,
   call_data *calld = elem->call_data;
   calld->sent_initial_metadata = 0;
   calld->got_initial_metadata = 0;
+  calld->sent_authority = 0;
   calld->on_done_recv = NULL;
   grpc_iomgr_closure_init(&calld->hc_on_recv, hc_on_recv, elem);
   if (initial_op) hc_mutate_op(elem, initial_op);
@@ -233,7 +247,7 @@ static grpc_mdstr *user_agent_from_args(grpc_mdctx *mdctx,
 
   tmp = gpr_strvec_flatten(&v, NULL);
   gpr_strvec_destroy(&v);
-  result = grpc_mdstr_from_string(mdctx, tmp);
+  result = grpc_mdstr_from_string(mdctx, tmp, 0);
   gpr_free(tmp);
 
   return result;
@@ -241,8 +255,10 @@ static grpc_mdstr *user_agent_from_args(grpc_mdctx *mdctx,
 
 /* Constructor for channel_data */
 static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
-                              const grpc_channel_args *args, grpc_mdctx *mdctx,
-                              int is_first, int is_last) {
+                              const grpc_channel_args *channel_args,
+                              grpc_mdctx *mdctx, int is_first, int is_last) {
+  size_t i;
+
   /* grab pointers to our data from the channel element */
   channel_data *channeld = elem->channel_data;
 
@@ -251,17 +267,32 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
      path */
   GPR_ASSERT(!is_last);
 
+  channeld->default_authority = NULL;
+  if (channel_args) {
+    for (i = 0; i < channel_args->num_args; i++) {
+      if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_DEFAULT_AUTHORITY)) {
+        if (channel_args->args[i].type != GRPC_ARG_STRING) {
+          gpr_log(GPR_ERROR, "%s: must be an string",
+                  GRPC_ARG_DEFAULT_AUTHORITY);
+        } else {
+          channeld->default_authority = grpc_mdelem_from_strings(
+              mdctx, ":authority", channel_args->args[i].value.string);
+        }
+      }
+    }
+  }
+
   /* initialize members */
   channeld->te_trailers = grpc_mdelem_from_strings(mdctx, "te", "trailers");
   channeld->method = grpc_mdelem_from_strings(mdctx, ":method", "POST");
-  channeld->scheme =
-      grpc_mdelem_from_strings(mdctx, ":scheme", scheme_from_args(args));
+  channeld->scheme = grpc_mdelem_from_strings(mdctx, ":scheme",
+                                              scheme_from_args(channel_args));
   channeld->content_type =
       grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc");
   channeld->status = grpc_mdelem_from_strings(mdctx, ":status", "200");
   channeld->user_agent = grpc_mdelem_from_metadata_strings(
-      mdctx, grpc_mdstr_from_string(mdctx, "user-agent"),
-      user_agent_from_args(mdctx, args));
+      mdctx, grpc_mdstr_from_string(mdctx, "user-agent", 0),
+      user_agent_from_args(mdctx, channel_args));
 }
 
 /* Destructor for channel data */
@@ -275,9 +306,13 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
   GRPC_MDELEM_UNREF(channeld->content_type);
   GRPC_MDELEM_UNREF(channeld->status);
   GRPC_MDELEM_UNREF(channeld->user_agent);
+  if (channeld->default_authority) {
+    GRPC_MDELEM_UNREF(channeld->default_authority);
+  }
 }
 
 const grpc_channel_filter grpc_http_client_filter = {
     hc_start_transport_op, grpc_channel_next_op, sizeof(call_data),
     init_call_elem,        destroy_call_elem,    sizeof(channel_data),
-    init_channel_elem,     destroy_channel_elem, "http-client"};
+    init_channel_elem,     destroy_channel_elem, grpc_call_next_get_peer,
+    "http-client"};
diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c
index a6cbb5a7f4..0955ae319a 100644
--- a/src/core/channel/http_server_filter.c
+++ b/src/core/channel/http_server_filter.c
@@ -44,6 +44,7 @@ typedef struct call_data {
   gpr_uint8 sent_status;
   gpr_uint8 seen_scheme;
   gpr_uint8 seen_te_trailers;
+  gpr_uint8 seen_authority;
   grpc_linked_mdelem status;
 
   grpc_stream_op_buffer *recv_ops;
@@ -125,6 +126,9 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
     }
     calld->seen_path = 1;
     return md;
+  } else if (md->key == channeld->authority_key) {
+    calld->seen_authority = 1;
+    return md;
   } else if (md->key == channeld->host_key) {
     /* translate host to :authority since :authority may be
        omitted */
@@ -132,6 +136,7 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
         channeld->mdctx, GRPC_MDSTR_REF(channeld->authority_key),
         GRPC_MDSTR_REF(md->value));
     GRPC_MDELEM_UNREF(md);
+    calld->seen_authority = 1;
     return authority;
   } else {
     return md;
@@ -154,12 +159,15 @@ static void hs_on_recv(void *user_data, int success) {
          (:method, :scheme, content-type, with :path and :authority covered
          at the channel level right now) */
       if (calld->seen_post && calld->seen_scheme && calld->seen_te_trailers &&
-          calld->seen_path) {
+          calld->seen_path && calld->seen_authority) {
         /* do nothing */
       } else {
         if (!calld->seen_path) {
           gpr_log(GPR_ERROR, "Missing :path header");
         }
+        if (!calld->seen_authority) {
+          gpr_log(GPR_ERROR, "Missing :authority header");
+        }
         if (!calld->seen_post) {
           gpr_log(GPR_ERROR, "Missing :method header");
         }
@@ -250,9 +258,9 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
   channeld->http_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "http");
   channeld->https_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "https");
   channeld->grpc_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "grpc");
-  channeld->path_key = grpc_mdstr_from_string(mdctx, ":path");
-  channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority");
-  channeld->host_key = grpc_mdstr_from_string(mdctx, "host");
+  channeld->path_key = grpc_mdstr_from_string(mdctx, ":path", 0);
+  channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority", 0);
+  channeld->host_key = grpc_mdstr_from_string(mdctx, "host", 0);
   channeld->content_type =
       grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc");
 
@@ -280,4 +288,5 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
 const grpc_channel_filter grpc_http_server_filter = {
     hs_start_transport_op, grpc_channel_next_op, sizeof(call_data),
     init_call_elem,        destroy_call_elem,    sizeof(channel_data),
-    init_channel_elem,     destroy_channel_elem, "http-server"};
+    init_channel_elem,     destroy_channel_elem, grpc_call_next_get_peer,
+    "http-server"};
diff --git a/src/core/channel/noop_filter.c b/src/core/channel/noop_filter.c
index 5117723617..d631885aaf 100644
--- a/src/core/channel/noop_filter.c
+++ b/src/core/channel/noop_filter.c
@@ -127,4 +127,5 @@ const grpc_channel_filter grpc_no_op_filter = {noop_start_transport_stream_op,
                                                sizeof(channel_data),
                                                init_channel_elem,
                                                destroy_channel_elem,
+                                               grpc_call_next_get_peer,
                                                "no-op"};
diff --git a/src/core/client_config/README.md b/src/core/client_config/README.md
index d0700cfb13..fff7a5af5b 100644
--- a/src/core/client_config/README.md
+++ b/src/core/client_config/README.md
@@ -60,3 +60,7 @@ unix:path        - the unix scheme is used to create and connect to unix domain
                    sockets - the authority must be empty, and the path
                    represents the absolute or relative path to the desired
                    socket
+
+ipv4:host:port   - a pre-resolved ipv4 dotted decimal address/port combination
+
+ipv6:[host]:port - a pre-resolved ipv6 address/port combination
diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c
index 73da624aff..5ae2e0ea52 100644
--- a/src/core/client_config/lb_policies/pick_first.c
+++ b/src/core/client_config/lb_policies/pick_first.c
@@ -62,6 +62,8 @@ typedef struct {
   grpc_subchannel *selected;
   /** have we started picking? */
   int started_picking;
+  /** are we shut down? */
+  int shutdown;
   /** which subchannel are we watching? */
   size_t checking_subchannel;
   /** what is the connectivity of that channel? */
@@ -73,12 +75,30 @@ typedef struct {
   grpc_connectivity_state_tracker state_tracker;
 } pick_first_lb_policy;
 
+static void del_interested_parties_locked(pick_first_lb_policy *p) {
+  pending_pick *pp;
+  for (pp = p->pending_picks; pp; pp = pp->next) {
+    grpc_subchannel_del_interested_party(p->subchannels[p->checking_subchannel],
+                                         pp->pollset);
+  }
+}
+
+static void add_interested_parties_locked(pick_first_lb_policy *p) {
+  pending_pick *pp;
+  for (pp = p->pending_picks; pp; pp = pp->next) {
+    grpc_subchannel_add_interested_party(p->subchannels[p->checking_subchannel],
+                                         pp->pollset);
+  }
+}
+
 void pf_destroy(grpc_lb_policy *pol) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   size_t i;
+  del_interested_parties_locked(p);
   for (i = 0; i < p->num_subchannels; i++) {
     GRPC_SUBCHANNEL_UNREF(p->subchannels[i], "pick_first");
   }
+  grpc_connectivity_state_destroy(&p->state_tracker);
   gpr_free(p->subchannels);
   gpr_mu_destroy(&p->mu);
   gpr_free(p);
@@ -88,12 +108,35 @@ void pf_shutdown(grpc_lb_policy *pol) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   pending_pick *pp;
   gpr_mu_lock(&p->mu);
+  del_interested_parties_locked(p);
+  p->shutdown = 1;
   while ((pp = p->pending_picks)) {
     p->pending_picks = pp->next;
     *pp->target = NULL;
     grpc_iomgr_add_delayed_callback(pp->on_complete, 0);
     gpr_free(pp);
   }
+  grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_FATAL_FAILURE,
+                              "shutdown");
+  gpr_mu_unlock(&p->mu);
+}
+
+static void start_picking(pick_first_lb_policy *p) {
+  p->started_picking = 1;
+  p->checking_subchannel = 0;
+  p->checking_connectivity = GRPC_CHANNEL_IDLE;
+  GRPC_LB_POLICY_REF(&p->base, "pick_first_connectivity");
+  grpc_subchannel_notify_on_state_change(p->subchannels[p->checking_subchannel],
+                                         &p->checking_connectivity,
+                                         &p->connectivity_changed);
+}
+
+void pf_exit_idle(grpc_lb_policy *pol) {
+  pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
+  gpr_mu_lock(&p->mu);
+  if (!p->started_picking) {
+    start_picking(p);
+  }
   gpr_mu_unlock(&p->mu);
 }
 
@@ -109,13 +152,7 @@ void pf_pick(grpc_lb_policy *pol, grpc_pollset *pollset,
     on_complete->cb(on_complete->cb_arg, 1);
   } else {
     if (!p->started_picking) {
-      p->started_picking = 1;
-      p->checking_subchannel = 0;
-      p->checking_connectivity = GRPC_CHANNEL_IDLE;
-      GRPC_LB_POLICY_REF(pol, "pick_first_connectivity");
-      grpc_subchannel_notify_on_state_change(
-          p->subchannels[p->checking_subchannel], &p->checking_connectivity,
-          &p->connectivity_changed);
+      start_picking(p);
     }
     grpc_subchannel_add_interested_party(p->subchannels[p->checking_subchannel],
                                          pollset);
@@ -129,77 +166,97 @@ void pf_pick(grpc_lb_policy *pol, grpc_pollset *pollset,
   }
 }
 
-static void del_interested_parties_locked(pick_first_lb_policy *p) {
-  pending_pick *pp;
-  for (pp = p->pending_picks; pp; pp = pp->next) {
-    grpc_subchannel_del_interested_party(p->subchannels[p->checking_subchannel],
-                                         pp->pollset);
-  }
-}
-
-static void add_interested_parties_locked(pick_first_lb_policy *p) {
-  pending_pick *pp;
-  for (pp = p->pending_picks; pp; pp = pp->next) {
-    grpc_subchannel_add_interested_party(p->subchannels[p->checking_subchannel],
-                                         pp->pollset);
-  }
-}
-
 static void pf_connectivity_changed(void *arg, int iomgr_success) {
   pick_first_lb_policy *p = arg;
   pending_pick *pp;
   int unref = 0;
 
   gpr_mu_lock(&p->mu);
-loop:
-  switch (p->checking_connectivity) {
-    case GRPC_CHANNEL_READY:
-      p->selected = p->subchannels[p->checking_subchannel];
-      while ((pp = p->pending_picks)) {
-        p->pending_picks = pp->next;
-        *pp->target = p->selected;
-        grpc_subchannel_del_interested_party(p->selected, pp->pollset);
-        grpc_iomgr_add_delayed_callback(pp->on_complete, 1);
-        gpr_free(pp);
-      }
-      unref = 1;
-      break;
-    case GRPC_CHANNEL_TRANSIENT_FAILURE:
-      del_interested_parties_locked(p);
-      p->checking_subchannel =
-          (p->checking_subchannel + 1) % p->num_subchannels;
-      p->checking_connectivity = grpc_subchannel_check_connectivity(
-          p->subchannels[p->checking_subchannel]);
-      add_interested_parties_locked(p);
-      goto loop;
-    case GRPC_CHANNEL_CONNECTING:
-    case GRPC_CHANNEL_IDLE:
+
+  if (p->shutdown) {
+    unref = 1;
+  } else if (p->selected != NULL) {
+    grpc_connectivity_state_set(&p->state_tracker, p->checking_connectivity,
+                                "selected_changed");
+    if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) {
       grpc_subchannel_notify_on_state_change(
-          p->subchannels[p->checking_subchannel], &p->checking_connectivity,
-          &p->connectivity_changed);
-      break;
-    case GRPC_CHANNEL_FATAL_FAILURE:
-      del_interested_parties_locked(p);
-      GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel],
-               p->subchannels[p->num_subchannels - 1]);
-      p->num_subchannels--;
-      GRPC_SUBCHANNEL_UNREF(p->subchannels[p->num_subchannels], "pick_first");
-      if (p->num_subchannels == 0) {
+          p->selected, &p->checking_connectivity, &p->connectivity_changed);
+    } else {
+      unref = 1;
+    }
+  } else {
+  loop:
+    switch (p->checking_connectivity) {
+      case GRPC_CHANNEL_READY:
+        grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_READY,
+                                    "connecting_ready");
+        p->selected = p->subchannels[p->checking_subchannel];
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
-          *pp->target = NULL;
+          *pp->target = p->selected;
+          grpc_subchannel_del_interested_party(p->selected, pp->pollset);
           grpc_iomgr_add_delayed_callback(pp->on_complete, 1);
           gpr_free(pp);
         }
-        unref = 1;
-      } else {
-        p->checking_subchannel %= p->num_subchannels;
+        grpc_subchannel_notify_on_state_change(
+            p->selected, &p->checking_connectivity, &p->connectivity_changed);
+        break;
+      case GRPC_CHANNEL_TRANSIENT_FAILURE:
+        grpc_connectivity_state_set(&p->state_tracker,
+                                    GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                    "connecting_transient_failure");
+        del_interested_parties_locked(p);
+        p->checking_subchannel =
+            (p->checking_subchannel + 1) % p->num_subchannels;
         p->checking_connectivity = grpc_subchannel_check_connectivity(
             p->subchannels[p->checking_subchannel]);
         add_interested_parties_locked(p);
-        goto loop;
-      }
+        if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+          grpc_subchannel_notify_on_state_change(
+              p->subchannels[p->checking_subchannel], &p->checking_connectivity,
+              &p->connectivity_changed);
+        } else {
+          goto loop;
+        }
+        break;
+      case GRPC_CHANNEL_CONNECTING:
+      case GRPC_CHANNEL_IDLE:
+        grpc_connectivity_state_set(&p->state_tracker, p->checking_connectivity,
+                                    "connecting_changed");
+        grpc_subchannel_notify_on_state_change(
+            p->subchannels[p->checking_subchannel], &p->checking_connectivity,
+            &p->connectivity_changed);
+        break;
+      case GRPC_CHANNEL_FATAL_FAILURE:
+        del_interested_parties_locked(p);
+        GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel],
+                 p->subchannels[p->num_subchannels - 1]);
+        p->num_subchannels--;
+        GRPC_SUBCHANNEL_UNREF(p->subchannels[p->num_subchannels], "pick_first");
+        if (p->num_subchannels == 0) {
+          grpc_connectivity_state_set(&p->state_tracker,
+                                      GRPC_CHANNEL_FATAL_FAILURE,
+                                      "no_more_channels");
+          while ((pp = p->pending_picks)) {
+            p->pending_picks = pp->next;
+            *pp->target = NULL;
+            grpc_iomgr_add_delayed_callback(pp->on_complete, 1);
+            gpr_free(pp);
+          }
+          unref = 1;
+        } else {
+          grpc_connectivity_state_set(&p->state_tracker,
+                                      GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                      "subchannel_failed");
+          p->checking_subchannel %= p->num_subchannels;
+          p->checking_connectivity = grpc_subchannel_check_connectivity(
+              p->subchannels[p->checking_subchannel]);
+          add_interested_parties_locked(p);
+          goto loop;
+        }
+    }
   }
+
   gpr_mu_unlock(&p->mu);
 
   if (unref) {
@@ -249,8 +306,13 @@ static void pf_notify_on_state_change(grpc_lb_policy *pol,
 }
 
 static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = {
-    pf_destroy,   pf_shutdown,           pf_pick,
-    pf_broadcast, pf_check_connectivity, pf_notify_on_state_change};
+    pf_destroy,
+    pf_shutdown,
+    pf_pick,
+    pf_exit_idle,
+    pf_broadcast,
+    pf_check_connectivity,
+    pf_notify_on_state_change};
 
 grpc_lb_policy *grpc_create_pick_first_lb_policy(grpc_subchannel **subchannels,
                                                  size_t num_subchannels) {
@@ -260,6 +322,8 @@ grpc_lb_policy *grpc_create_pick_first_lb_policy(grpc_subchannel **subchannels,
   grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable);
   p->subchannels = gpr_malloc(sizeof(grpc_subchannel *) * num_subchannels);
   p->num_subchannels = num_subchannels;
+  grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
+                               "pick_first");
   memcpy(p->subchannels, subchannels,
          sizeof(grpc_subchannel *) * num_subchannels);
   grpc_iomgr_closure_init(&p->connectivity_changed, pf_connectivity_changed, p);
diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c
index 6d1c788742..90ec44432f 100644
--- a/src/core/client_config/lb_policy.c
+++ b/src/core/client_config/lb_policy.c
@@ -77,3 +77,18 @@ void grpc_lb_policy_pick(grpc_lb_policy *policy, grpc_pollset *pollset,
 void grpc_lb_policy_broadcast(grpc_lb_policy *policy, grpc_transport_op *op) {
   policy->vtable->broadcast(policy, op);
 }
+
+void grpc_lb_policy_exit_idle(grpc_lb_policy *policy) {
+  policy->vtable->exit_idle(policy);
+}
+
+void grpc_lb_policy_notify_on_state_change(grpc_lb_policy *policy,
+                                           grpc_connectivity_state *state,
+                                           grpc_iomgr_closure *closure) {
+  policy->vtable->notify_on_state_change(policy, state, closure);
+}
+
+grpc_connectivity_state grpc_lb_policy_check_connectivity(
+    grpc_lb_policy *policy) {
+  return policy->vtable->check_connectivity(policy);
+}
diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h
index a468f761cc..3f7ca8f28d 100644
--- a/src/core/client_config/lb_policy.h
+++ b/src/core/client_config/lb_policy.h
@@ -59,6 +59,9 @@ struct grpc_lb_policy_vtable {
                grpc_metadata_batch *initial_metadata, grpc_subchannel **target,
                grpc_iomgr_closure *on_complete);
 
+  /** try to enter a READY connectivity state */
+  void (*exit_idle)(grpc_lb_policy *policy);
+
   /** broadcast a transport op to all subchannels */
   void (*broadcast)(grpc_lb_policy *policy, grpc_transport_op *op);
 
@@ -106,4 +109,13 @@ void grpc_lb_policy_pick(grpc_lb_policy *policy, grpc_pollset *pollset,
 
 void grpc_lb_policy_broadcast(grpc_lb_policy *policy, grpc_transport_op *op);
 
+void grpc_lb_policy_exit_idle(grpc_lb_policy *policy);
+
+void grpc_lb_policy_notify_on_state_change(grpc_lb_policy *policy,
+                                           grpc_connectivity_state *state,
+                                           grpc_iomgr_closure *closure);
+
+grpc_connectivity_state grpc_lb_policy_check_connectivity(
+    grpc_lb_policy *policy);
+
 #endif /* GRPC_INTERNAL_CORE_CONFIG_LB_POLICY_H */
diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/client_config/resolvers/dns_resolver.c
index ac401bc4d3..827b1a2be5 100644
--- a/src/core/client_config/resolvers/dns_resolver.c
+++ b/src/core/client_config/resolvers/dns_resolver.c
@@ -36,9 +36,11 @@
 #include <string.h>
 
 #include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
 #include <grpc/support/string_util.h>
 
 #include "src/core/client_config/lb_policies/pick_first.h"
+#include "src/core/client_config/subchannel_factory_decorators/add_channel_arg.h"
 #include "src/core/iomgr/resolve_address.h"
 #include "src/core/support/string.h"
 
@@ -201,6 +203,9 @@ static grpc_resolver *dns_create(
     grpc_subchannel_factory *subchannel_factory) {
   dns_resolver *r;
   const char *path = uri->path;
+  grpc_arg default_host_arg;
+  char *host;
+  char *port;
 
   if (0 != strcmp(uri->authority, "")) {
     gpr_log(GPR_ERROR, "authority based uri's not supported");
@@ -209,6 +214,16 @@ static grpc_resolver *dns_create(
 
   if (path[0] == '/') ++path;
 
+  gpr_split_host_port(path, &host, &port);
+
+  default_host_arg.type = GRPC_ARG_STRING;
+  default_host_arg.key = GRPC_ARG_DEFAULT_AUTHORITY;
+  default_host_arg.value.string = host;
+  subchannel_factory = grpc_subchannel_factory_add_channel_arg(subchannel_factory, &default_host_arg);
+
+  gpr_free(host);
+  gpr_free(port);
+
   r = gpr_malloc(sizeof(dns_resolver));
   memset(r, 0, sizeof(*r));
   gpr_ref_init(&r->refs, 1);
@@ -218,7 +233,6 @@ static grpc_resolver *dns_create(
   r->default_port = gpr_strdup(default_port);
   r->subchannel_factory = subchannel_factory;
   r->lb_policy_factory = lb_policy_factory;
-  grpc_subchannel_factory_ref(subchannel_factory);
   return &r->base;
 }
 
diff --git a/src/core/client_config/resolvers/sockaddr_resolver.c b/src/core/client_config/resolvers/sockaddr_resolver.c
new file mode 100644
index 0000000000..74584e7e2c
--- /dev/null
+++ b/src/core/client_config/resolvers/sockaddr_resolver.c
@@ -0,0 +1,299 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/client_config/resolvers/sockaddr_resolver.h"
+
+#include <stdio.h>
+#include <string.h>
+#ifdef GPR_POSIX_SOCKET
+#include <sys/un.h>
+#endif
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/client_config/lb_policies/pick_first.h"
+#include "src/core/iomgr/resolve_address.h"
+#include "src/core/support/string.h"
+
+typedef struct {
+  /** base class: must be first */
+  grpc_resolver base;
+  /** refcount */
+  gpr_refcount refs;
+  /** subchannel factory */
+  grpc_subchannel_factory *subchannel_factory;
+  /** load balancing policy factory */
+  grpc_lb_policy *(*lb_policy_factory)(grpc_subchannel **subchannels,
+                                       size_t num_subchannels);
+
+  /** the address that we've 'resolved' */
+  struct sockaddr_storage addr;
+  int addr_len;
+
+  /** mutex guarding the rest of the state */
+  gpr_mu mu;
+  /** have we published? */
+  int published;
+  /** pending next completion, or NULL */
+  grpc_iomgr_closure *next_completion;
+  /** target config address for next completion */
+  grpc_client_config **target_config;
+} sockaddr_resolver;
+
+static void sockaddr_destroy(grpc_resolver *r);
+
+static void sockaddr_maybe_finish_next_locked(sockaddr_resolver *r);
+
+static void sockaddr_shutdown(grpc_resolver *r);
+static void sockaddr_channel_saw_error(grpc_resolver *r,
+                                       struct sockaddr *failing_address,
+                                       int failing_address_len);
+static void sockaddr_next(grpc_resolver *r, grpc_client_config **target_config,
+                          grpc_iomgr_closure *on_complete);
+
+static const grpc_resolver_vtable sockaddr_resolver_vtable = {
+    sockaddr_destroy, sockaddr_shutdown, sockaddr_channel_saw_error,
+    sockaddr_next};
+
+static void sockaddr_shutdown(grpc_resolver *resolver) {
+  sockaddr_resolver *r = (sockaddr_resolver *)resolver;
+  gpr_mu_lock(&r->mu);
+  if (r->next_completion != NULL) {
+    *r->target_config = NULL;
+    /* TODO(ctiller): add delayed callback */
+    grpc_iomgr_add_callback(r->next_completion);
+    r->next_completion = NULL;
+  }
+  gpr_mu_unlock(&r->mu);
+}
+
+static void sockaddr_channel_saw_error(grpc_resolver *resolver,
+                                       struct sockaddr *sa, int len) {}
+
+static void sockaddr_next(grpc_resolver *resolver,
+                          grpc_client_config **target_config,
+                          grpc_iomgr_closure *on_complete) {
+  sockaddr_resolver *r = (sockaddr_resolver *)resolver;
+  gpr_mu_lock(&r->mu);
+  GPR_ASSERT(!r->next_completion);
+  r->next_completion = on_complete;
+  r->target_config = target_config;
+  sockaddr_maybe_finish_next_locked(r);
+  gpr_mu_unlock(&r->mu);
+}
+
+static void sockaddr_maybe_finish_next_locked(sockaddr_resolver *r) {
+  grpc_client_config *cfg;
+  grpc_lb_policy *lb_policy;
+  grpc_subchannel *subchannel;
+  grpc_subchannel_args args;
+
+  if (r->next_completion != NULL && !r->published) {
+    cfg = grpc_client_config_create();
+    memset(&args, 0, sizeof(args));
+    args.addr = (struct sockaddr *)&r->addr;
+    args.addr_len = r->addr_len;
+    subchannel =
+        grpc_subchannel_factory_create_subchannel(r->subchannel_factory, &args);
+    lb_policy = r->lb_policy_factory(&subchannel, 1);
+    grpc_client_config_set_lb_policy(cfg, lb_policy);
+    GRPC_LB_POLICY_UNREF(lb_policy, "unix");
+    r->published = 1;
+    *r->target_config = cfg;
+    grpc_iomgr_add_callback(r->next_completion);
+    r->next_completion = NULL;
+  }
+}
+
+static void sockaddr_destroy(grpc_resolver *gr) {
+  sockaddr_resolver *r = (sockaddr_resolver *)gr;
+  gpr_mu_destroy(&r->mu);
+  grpc_subchannel_factory_unref(r->subchannel_factory);
+  gpr_free(r);
+}
+
+#ifdef GPR_POSIX_SOCKET
+static int parse_unix(grpc_uri *uri, struct sockaddr_storage *addr, int *len) {
+  struct sockaddr_un *un = (struct sockaddr_un *)addr;
+
+  un->sun_family = AF_UNIX;
+  strcpy(un->sun_path, uri->path);
+  *len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
+
+  return 1;
+}
+#endif
+
+static int parse_ipv4(grpc_uri *uri, struct sockaddr_storage *addr, int *len) {
+  const char *host_port = uri->path;
+  char *host;
+  char *port;
+  int port_num;
+  int result = 0;
+  struct sockaddr_in *in = (struct sockaddr_in *)addr;
+
+  if (*host_port == '/') ++host_port;
+  if (!gpr_split_host_port(host_port, &host, &port)) {
+    return 0;
+  }
+
+  memset(in, 0, sizeof(*in));
+  *len = sizeof(*in);
+  in->sin_family = AF_INET;
+  if (inet_pton(AF_INET, host, &in->sin_addr) == 0) {
+    gpr_log(GPR_ERROR, "invalid ipv4 address: '%s'", host);
+    goto done;
+  }
+
+  if (port != NULL) {
+    if (sscanf(port, "%d", &port_num) != 1 || port_num < 0 ||
+        port_num > 65535) {
+      gpr_log(GPR_ERROR, "invalid ipv4 port: '%s'", port);
+      goto done;
+    }
+    in->sin_port = htons(port_num);
+  } else {
+    gpr_log(GPR_ERROR, "no port given for ipv4 scheme");
+    goto done;
+  }
+
+  result = 1;
+done:
+  gpr_free(host);
+  gpr_free(port);
+  return result;
+}
+
+static int parse_ipv6(grpc_uri *uri, struct sockaddr_storage *addr, int *len) {
+  const char *host_port = uri->path;
+  char *host;
+  char *port;
+  int port_num;
+  int result = 0;
+  struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)addr;
+
+  if (*host_port == '/') ++host_port;
+  if (!gpr_split_host_port(host_port, &host, &port)) {
+    return 0;
+  }
+
+  memset(in6, 0, sizeof(*in6));
+  *len = sizeof(*in6);
+  in6->sin6_family = AF_INET6;
+  if (inet_pton(AF_INET6, host, &in6->sin6_addr) == 0) {
+    gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host);
+    goto done;
+  }
+
+  if (port != NULL) {
+    if (sscanf(port, "%d", &port_num) != 1 || port_num < 0 ||
+        port_num > 65535) {
+      gpr_log(GPR_ERROR, "invalid ipv6 port: '%s'", port);
+      goto done;
+    }
+    in6->sin6_port = htons(port_num);
+  } else {
+    gpr_log(GPR_ERROR, "no port given for ipv6 scheme");
+    goto done;
+  }
+
+  result = 1;
+done:
+  gpr_free(host);
+  gpr_free(port);
+  return result;
+}
+
+static grpc_resolver *sockaddr_create(
+    grpc_uri *uri,
+    grpc_lb_policy *(*lb_policy_factory)(grpc_subchannel **subchannels,
+                                         size_t num_subchannels),
+    grpc_subchannel_factory *subchannel_factory,
+    int parse(grpc_uri *uri, struct sockaddr_storage *dst, int *len)) {
+  sockaddr_resolver *r;
+
+  if (0 != strcmp(uri->authority, "")) {
+    gpr_log(GPR_ERROR, "authority based uri's not supported");
+    return NULL;
+  }
+
+  r = gpr_malloc(sizeof(sockaddr_resolver));
+  memset(r, 0, sizeof(*r));
+  if (!parse(uri, &r->addr, &r->addr_len)) {
+    gpr_free(r);
+    return NULL;
+  }
+
+  gpr_ref_init(&r->refs, 1);
+  gpr_mu_init(&r->mu);
+  grpc_resolver_init(&r->base, &sockaddr_resolver_vtable);
+  r->subchannel_factory = subchannel_factory;
+  r->lb_policy_factory = lb_policy_factory;
+
+  grpc_subchannel_factory_ref(subchannel_factory);
+  return &r->base;
+}
+
+/*
+ * FACTORY
+ */
+
+static void sockaddr_factory_ref(grpc_resolver_factory *factory) {}
+
+static void sockaddr_factory_unref(grpc_resolver_factory *factory) {}
+
+#define DECL_FACTORY(name)                                            \
+  static grpc_resolver *name##_factory_create_resolver(               \
+      grpc_resolver_factory *factory, grpc_uri *uri,                  \
+      grpc_subchannel_factory *subchannel_factory) {                  \
+    return sockaddr_create(uri, grpc_create_pick_first_lb_policy,     \
+                           subchannel_factory, parse_##name);         \
+  }                                                                   \
+  static const grpc_resolver_factory_vtable name##_factory_vtable = { \
+      sockaddr_factory_ref, sockaddr_factory_unref,                   \
+      name##_factory_create_resolver};                                \
+  static grpc_resolver_factory name##_resolver_factory = {            \
+      &name##_factory_vtable};                                        \
+  grpc_resolver_factory *grpc_##name##_resolver_factory_create() {    \
+    return &name##_resolver_factory;                                  \
+  }
+
+#ifdef GPR_POSIX_SOCKET
+DECL_FACTORY(unix)
+#endif
+DECL_FACTORY(ipv4)
+DECL_FACTORY(ipv6)
diff --git a/src/core/client_config/resolvers/unix_resolver_posix.h b/src/core/client_config/resolvers/sockaddr_resolver.h
index 57ace59e21..1b7a18f9c2 100644
--- a/src/core/client_config/resolvers/unix_resolver_posix.h
+++ b/src/core/client_config/resolvers/sockaddr_resolver.h
@@ -38,7 +38,13 @@
 
 #include "src/core/client_config/resolver_factory.h"
 
+grpc_resolver_factory *grpc_ipv4_resolver_factory_create(void);
+
+grpc_resolver_factory *grpc_ipv6_resolver_factory_create(void);
+
+#ifdef GPR_POSIX_SOCKET
 /** Create a unix resolver factory */
 grpc_resolver_factory *grpc_unix_resolver_factory_create(void);
+#endif
 
 #endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_UNIX_RESOLVER_H */
diff --git a/src/core/client_config/resolvers/unix_resolver_posix.c b/src/core/client_config/resolvers/unix_resolver_posix.c
deleted file mode 100644
index be515d2689..0000000000
--- a/src/core/client_config/resolvers/unix_resolver_posix.c
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <grpc/support/port_platform.h>
-#ifdef GPR_POSIX_SOCKET
-
-#include "src/core/client_config/resolvers/unix_resolver_posix.h"
-
-#include <string.h>
-#include <sys/un.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/string_util.h>
-
-#include "src/core/client_config/lb_policies/pick_first.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/support/string.h"
-
-typedef struct {
-  /** base class: must be first */
-  grpc_resolver base;
-  /** refcount */
-  gpr_refcount refs;
-  /** subchannel factory */
-  grpc_subchannel_factory *subchannel_factory;
-  /** load balancing policy factory */
-  grpc_lb_policy *(*lb_policy_factory)(grpc_subchannel **subchannels,
-                                       size_t num_subchannels);
-
-  /** the address that we've 'resolved' */
-  struct sockaddr_un addr;
-  int addr_len;
-
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
-  /** have we published? */
-  int published;
-  /** pending next completion, or NULL */
-  grpc_iomgr_closure *next_completion;
-  /** target config address for next completion */
-  grpc_client_config **target_config;
-} unix_resolver;
-
-static void unix_destroy(grpc_resolver *r);
-
-static void unix_maybe_finish_next_locked(unix_resolver *r);
-
-static void unix_shutdown(grpc_resolver *r);
-static void unix_channel_saw_error(grpc_resolver *r,
-                                   struct sockaddr *failing_address,
-                                   int failing_address_len);
-static void unix_next(grpc_resolver *r, grpc_client_config **target_config,
-                      grpc_iomgr_closure *on_complete);
-
-static const grpc_resolver_vtable unix_resolver_vtable = {
-    unix_destroy, unix_shutdown, unix_channel_saw_error, unix_next};
-
-static void unix_shutdown(grpc_resolver *resolver) {
-  unix_resolver *r = (unix_resolver *)resolver;
-  gpr_mu_lock(&r->mu);
-  if (r->next_completion != NULL) {
-    *r->target_config = NULL;
-    /* TODO(ctiller): add delayed callback */
-    grpc_iomgr_add_callback(r->next_completion);
-    r->next_completion = NULL;
-  }
-  gpr_mu_unlock(&r->mu);
-}
-
-static void unix_channel_saw_error(grpc_resolver *resolver, struct sockaddr *sa,
-                                   int len) {}
-
-static void unix_next(grpc_resolver *resolver,
-                      grpc_client_config **target_config,
-                      grpc_iomgr_closure *on_complete) {
-  unix_resolver *r = (unix_resolver *)resolver;
-  gpr_mu_lock(&r->mu);
-  GPR_ASSERT(!r->next_completion);
-  r->next_completion = on_complete;
-  r->target_config = target_config;
-  unix_maybe_finish_next_locked(r);
-  gpr_mu_unlock(&r->mu);
-}
-
-static void unix_maybe_finish_next_locked(unix_resolver *r) {
-  grpc_client_config *cfg;
-  grpc_lb_policy *lb_policy;
-  grpc_subchannel *subchannel;
-  grpc_subchannel_args args;
-
-  if (r->next_completion != NULL && !r->published) {
-    cfg = grpc_client_config_create();
-    memset(&args, 0, sizeof(args));
-    args.addr = (struct sockaddr *)&r->addr;
-    args.addr_len = r->addr_len;
-    subchannel =
-        grpc_subchannel_factory_create_subchannel(r->subchannel_factory, &args);
-    lb_policy = r->lb_policy_factory(&subchannel, 1);
-    grpc_client_config_set_lb_policy(cfg, lb_policy);
-    GRPC_LB_POLICY_UNREF(lb_policy, "unix");
-    r->published = 1;
-    *r->target_config = cfg;
-    grpc_iomgr_add_callback(r->next_completion);
-    r->next_completion = NULL;
-  }
-}
-
-static void unix_destroy(grpc_resolver *gr) {
-  unix_resolver *r = (unix_resolver *)gr;
-  gpr_mu_destroy(&r->mu);
-  grpc_subchannel_factory_unref(r->subchannel_factory);
-  gpr_free(r);
-}
-
-static grpc_resolver *unix_create(
-    grpc_uri *uri,
-    grpc_lb_policy *(*lb_policy_factory)(grpc_subchannel **subchannels,
-                                         size_t num_subchannels),
-    grpc_subchannel_factory *subchannel_factory) {
-  unix_resolver *r;
-
-  if (0 != strcmp(uri->authority, "")) {
-    gpr_log(GPR_ERROR, "authority based uri's not supported");
-    return NULL;
-  }
-
-  r = gpr_malloc(sizeof(unix_resolver));
-  memset(r, 0, sizeof(*r));
-  gpr_ref_init(&r->refs, 1);
-  gpr_mu_init(&r->mu);
-  grpc_resolver_init(&r->base, &unix_resolver_vtable);
-  r->subchannel_factory = subchannel_factory;
-  r->lb_policy_factory = lb_policy_factory;
-
-  r->addr.sun_family = AF_UNIX;
-  strcpy(r->addr.sun_path, uri->path);
-  r->addr_len = strlen(r->addr.sun_path) + sizeof(r->addr.sun_family) + 1;
-
-  grpc_subchannel_factory_ref(subchannel_factory);
-  return &r->base;
-}
-
-/*
- * FACTORY
- */
-
-static void unix_factory_ref(grpc_resolver_factory *factory) {}
-
-static void unix_factory_unref(grpc_resolver_factory *factory) {}
-
-static grpc_resolver *unix_factory_create_resolver(
-    grpc_resolver_factory *factory, grpc_uri *uri,
-    grpc_subchannel_factory *subchannel_factory) {
-  return unix_create(uri, grpc_create_pick_first_lb_policy, subchannel_factory);
-}
-
-static const grpc_resolver_factory_vtable unix_factory_vtable = {
-    unix_factory_ref, unix_factory_unref, unix_factory_create_resolver};
-static grpc_resolver_factory unix_resolver_factory = {&unix_factory_vtable};
-
-grpc_resolver_factory *grpc_unix_resolver_factory_create() {
-  return &unix_resolver_factory;
-}
-
-#endif
diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c
index 487f5afb35..ca52c75beb 100644
--- a/src/core/client_config/subchannel.c
+++ b/src/core/client_config/subchannel.c
@@ -38,9 +38,17 @@
 #include <grpc/support/alloc.h>
 
 #include "src/core/channel/channel_args.h"
+#include "src/core/channel/client_channel.h"
 #include "src/core/channel/connected_channel.h"
 #include "src/core/iomgr/alarm.h"
 #include "src/core/transport/connectivity_state.h"
+#include "src/core/surface/channel.h"
+
+#define GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS 20
+#define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 1
+#define GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER 1.6
+#define GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS 120
+#define GRPC_SUBCHANNEL_RECONNECT_JITTER 0.2
 
 typedef struct {
   /* all fields protected by subchannel->mu */
@@ -94,8 +102,10 @@ struct grpc_subchannel {
   grpc_iomgr_closure connected;
 
   /** pollset_set tracking who's interested in a connection
-      being setup */
-  grpc_pollset_set pollset_set;
+      being setup - owned by the master channel (in particular the
+     client_channel
+      filter there-in) */
+  grpc_pollset_set *pollset_set;
 
   /** mutex protecting remaining elements */
   gpr_mu mu;
@@ -121,6 +131,8 @@ struct grpc_subchannel {
   int have_alarm;
   /** our alarm */
   grpc_alarm alarm;
+  /** current random value */
+  gpr_uint32 random;
 };
 
 struct grpc_subchannel_call {
@@ -132,7 +144,8 @@ struct grpc_subchannel_call {
 #define CHANNEL_STACK_FROM_CONNECTION(con) ((grpc_channel_stack *)((con) + 1))
 
 static grpc_subchannel_call *create_call(connection *con);
-static void connectivity_state_changed_locked(grpc_subchannel *c);
+static void connectivity_state_changed_locked(grpc_subchannel *c,
+                                              const char *reason);
 static grpc_connectivity_state compute_connectivity_locked(grpc_subchannel *c);
 static gpr_timespec compute_connect_deadline(grpc_subchannel *c);
 static void subchannel_connected(void *subchannel, int iomgr_success);
@@ -244,7 +257,6 @@ static void subchannel_destroy(grpc_subchannel *c) {
   grpc_channel_args_destroy(c->args);
   gpr_free(c->addr);
   grpc_mdctx_unref(c->mdctx);
-  grpc_pollset_set_destroy(&c->pollset_set);
   grpc_connectivity_state_destroy(&c->state_tracker);
   grpc_connector_unref(c->connector);
   gpr_free(c);
@@ -252,17 +264,23 @@ static void subchannel_destroy(grpc_subchannel *c) {
 
 void grpc_subchannel_add_interested_party(grpc_subchannel *c,
                                           grpc_pollset *pollset) {
-  grpc_pollset_set_add_pollset(&c->pollset_set, pollset);
+  grpc_pollset_set_add_pollset(c->pollset_set, pollset);
 }
 
 void grpc_subchannel_del_interested_party(grpc_subchannel *c,
                                           grpc_pollset *pollset) {
-  grpc_pollset_set_del_pollset(&c->pollset_set, pollset);
+  grpc_pollset_set_del_pollset(c->pollset_set, pollset);
+}
+
+static gpr_uint32 random_seed() {
+  return (gpr_uint32)(gpr_time_to_millis(gpr_now(GPR_CLOCK_MONOTONIC)));
 }
 
 grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
                                         grpc_subchannel_args *args) {
   grpc_subchannel *c = gpr_malloc(sizeof(*c));
+  grpc_channel_element *parent_elem = grpc_channel_stack_last_element(
+      grpc_channel_get_channel_stack(args->master));
   memset(c, 0, sizeof(*c));
   c->refs = 1;
   c->connector = connector;
@@ -277,10 +295,12 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
   c->args = grpc_channel_args_copy(args->args);
   c->mdctx = args->mdctx;
   c->master = args->master;
+  c->pollset_set = grpc_client_channel_get_connecting_pollset_set(parent_elem);
+  c->random = random_seed();
   grpc_mdctx_ref(c->mdctx);
-  grpc_pollset_set_init(&c->pollset_set);
   grpc_iomgr_closure_init(&c->connected, subchannel_connected, c);
-  grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE);
+  grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE,
+                               "subchannel");
   gpr_mu_init(&c->mu);
   return c;
 }
@@ -288,7 +308,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
 static void continue_connect(grpc_subchannel *c) {
   grpc_connect_in_args args;
 
-  args.interested_parties = &c->pollset_set;
+  args.interested_parties = c->pollset_set;
   args.addr = c->addr;
   args.addr_len = c->addr_len;
   args.deadline = compute_connect_deadline(c);
@@ -300,15 +320,16 @@ static void continue_connect(grpc_subchannel *c) {
 }
 
 static void start_connect(grpc_subchannel *c) {
-  gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
-  c->next_attempt = now;
-  c->backoff_delta = gpr_time_from_seconds(1, GPR_TIMESPAN);
-
+  c->backoff_delta = gpr_time_from_seconds(
+      GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS, GPR_TIMESPAN);
+  c->next_attempt =
+      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), c->backoff_delta);
   continue_connect(c);
 }
 
 static void continue_creating_call(void *arg, int iomgr_success) {
   waiting_for_connect *w4c = arg;
+  grpc_subchannel_del_interested_party(w4c->subchannel, w4c->pollset);
   grpc_subchannel_create_call(w4c->subchannel, w4c->pollset, w4c->target,
                               w4c->notify);
   GRPC_SUBCHANNEL_UNREF(w4c->subchannel, "waiting_for_connect");
@@ -341,9 +362,10 @@ void grpc_subchannel_create_call(grpc_subchannel *c, grpc_pollset *pollset,
     grpc_subchannel_add_interested_party(c, pollset);
     if (!c->connecting) {
       c->connecting = 1;
-      connectivity_state_changed_locked(c);
+      connectivity_state_changed_locked(c, "create_call");
       /* released by connection */
       SUBCHANNEL_REF_LOCKED(c, "connecting");
+      GRPC_CHANNEL_INTERNAL_REF(c->master, "connecting");
       gpr_mu_unlock(&c->mu);
 
       start_connect(c);
@@ -372,7 +394,8 @@ void grpc_subchannel_notify_on_state_change(grpc_subchannel *c,
     c->connecting = 1;
     /* released by connection */
     SUBCHANNEL_REF_LOCKED(c, "connecting");
-    connectivity_state_changed_locked(c);
+    GRPC_CHANNEL_INTERNAL_REF(c->master, "connecting");
+    connectivity_state_changed_locked(c, "state_change");
   }
   gpr_mu_unlock(&c->mu);
   if (do_connect) {
@@ -388,7 +411,7 @@ void grpc_subchannel_process_transport_op(grpc_subchannel *c,
   gpr_mu_lock(&c->mu);
   if (op->disconnect) {
     c->disconnected = 1;
-    connectivity_state_changed_locked(c);
+    connectivity_state_changed_locked(c, "disconnect");
     if (c->have_alarm) {
       cancel_alarm = 1;
     }
@@ -456,13 +479,15 @@ static void on_state_changed(void *p, int iomgr_success) {
         destroy_connection = sw->subchannel->active;
       }
       sw->subchannel->active = NULL;
-      grpc_connectivity_state_set(&c->state_tracker,
-                                  GRPC_CHANNEL_TRANSIENT_FAILURE);
+      grpc_connectivity_state_set(
+          &c->state_tracker, c->disconnected ? GRPC_CHANNEL_FATAL_FAILURE
+                                             : GRPC_CHANNEL_TRANSIENT_FAILURE,
+          "connection_failed");
       break;
   }
 
 done:
-  connectivity_state_changed_locked(c);
+  connectivity_state_changed_locked(c, "transport_state_changed");
   destroy = SUBCHANNEL_UNREF_LOCKED(c, "state_watcher");
   gpr_free(sw);
   gpr_mu_unlock(mu);
@@ -519,6 +544,8 @@ static void publish_transport(grpc_subchannel *c) {
     gpr_free(sw);
     gpr_free(filters);
     grpc_channel_stack_destroy(stk);
+    GRPC_CHANNEL_INTERNAL_UNREF(c->master, "connecting");
+    GRPC_SUBCHANNEL_UNREF(c, "connecting");
     return;
   }
 
@@ -536,14 +563,16 @@ static void publish_transport(grpc_subchannel *c) {
   memset(&op, 0, sizeof(op));
   op.connectivity_state = &sw->connectivity_state;
   op.on_connectivity_state_change = &sw->closure;
+  op.bind_pollset_set = c->pollset_set;
   SUBCHANNEL_REF_LOCKED(c, "state_watcher");
+  GRPC_CHANNEL_INTERNAL_UNREF(c->master, "connecting");
   GPR_ASSERT(!SUBCHANNEL_UNREF_LOCKED(c, "connecting"));
   elem =
       grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(c->active), 0);
   elem->filter->start_transport_op(elem, &op);
 
   /* signal completion */
-  connectivity_state_changed_locked(c);
+  connectivity_state_changed_locked(c, "connected");
   while ((w4c = c->waiting)) {
     c->waiting = w4c->next;
     grpc_iomgr_add_callback(&w4c->continuation);
@@ -558,6 +587,35 @@ static void publish_transport(grpc_subchannel *c) {
   }
 }
 
+/* Generate a random number between 0 and 1. */
+static double generate_uniform_random_number(grpc_subchannel *c) {
+  c->random = (1103515245 * c->random + 12345) % ((gpr_uint32)1 << 31);
+  return c->random / (double)((gpr_uint32)1 << 31);
+}
+
+/* Update backoff_delta and next_attempt in subchannel */
+static void update_reconnect_parameters(grpc_subchannel *c) {
+  gpr_int32 backoff_delta_millis, jitter;
+  gpr_int32 max_backoff_millis =
+      GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000;
+  double jitter_range;
+  backoff_delta_millis =
+      (gpr_int32)(gpr_time_to_millis(c->backoff_delta) *
+                  GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER);
+  if (backoff_delta_millis > max_backoff_millis) {
+    backoff_delta_millis = max_backoff_millis;
+  }
+  c->backoff_delta = gpr_time_from_millis(backoff_delta_millis, GPR_TIMESPAN);
+  c->next_attempt =
+      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), c->backoff_delta);
+
+  jitter_range = GRPC_SUBCHANNEL_RECONNECT_JITTER * backoff_delta_millis;
+  jitter =
+      (gpr_int32)((2 * generate_uniform_random_number(c) - 1) * jitter_range);
+  c->next_attempt =
+      gpr_time_add(c->next_attempt, gpr_time_from_millis(jitter, GPR_TIMESPAN));
+}
+
 static void on_alarm(void *arg, int iomgr_success) {
   grpc_subchannel *c = arg;
   gpr_mu_lock(&c->mu);
@@ -565,11 +623,13 @@ static void on_alarm(void *arg, int iomgr_success) {
   if (c->disconnected) {
     iomgr_success = 0;
   }
-  connectivity_state_changed_locked(c);
+  connectivity_state_changed_locked(c, "alarm");
   gpr_mu_unlock(&c->mu);
   if (iomgr_success) {
+    update_reconnect_parameters(c);
     continue_connect(c);
   } else {
+    GRPC_CHANNEL_INTERNAL_UNREF(c->master, "connecting");
     GRPC_SUBCHANNEL_UNREF(c, "connecting");
   }
 }
@@ -579,19 +639,25 @@ static void subchannel_connected(void *arg, int iomgr_success) {
   if (c->connecting_result.transport != NULL) {
     publish_transport(c);
   } else {
+    gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
     gpr_mu_lock(&c->mu);
-    connectivity_state_changed_locked(c);
     GPR_ASSERT(!c->have_alarm);
     c->have_alarm = 1;
-    c->next_attempt = gpr_time_add(c->next_attempt, c->backoff_delta);
-    c->backoff_delta = gpr_time_add(c->backoff_delta, c->backoff_delta);
-    grpc_alarm_init(&c->alarm, c->next_attempt, on_alarm, c, gpr_now(GPR_CLOCK_MONOTONIC));
+    connectivity_state_changed_locked(c, "connect_failed");
+    grpc_alarm_init(&c->alarm, c->next_attempt, on_alarm, c, now);
     gpr_mu_unlock(&c->mu);
   }
 }
 
 static gpr_timespec compute_connect_deadline(grpc_subchannel *c) {
-  return gpr_time_add(c->next_attempt, c->backoff_delta);
+  gpr_timespec current_deadline =
+      gpr_time_add(c->next_attempt, c->backoff_delta);
+  gpr_timespec min_deadline = gpr_time_add(
+      gpr_now(GPR_CLOCK_MONOTONIC),
+      gpr_time_from_seconds(GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS,
+                            GPR_TIMESPAN));
+  return gpr_time_cmp(current_deadline, min_deadline) > 0 ? current_deadline
+                                                          : min_deadline;
 }
 
 static grpc_connectivity_state compute_connectivity_locked(grpc_subchannel *c) {
@@ -610,9 +676,10 @@ static grpc_connectivity_state compute_connectivity_locked(grpc_subchannel *c) {
   return GRPC_CHANNEL_IDLE;
 }
 
-static void connectivity_state_changed_locked(grpc_subchannel *c) {
+static void connectivity_state_changed_locked(grpc_subchannel *c,
+                                              const char *reason) {
   grpc_connectivity_state current = compute_connectivity_locked(c);
-  grpc_connectivity_state_set(&c->state_tracker, current);
+  grpc_connectivity_state_set(&c->state_tracker, current, reason);
 }
 
 /*
@@ -640,6 +707,12 @@ void grpc_subchannel_call_unref(
   }
 }
 
+char *grpc_subchannel_call_get_peer(grpc_subchannel_call *call) {
+  grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
+  grpc_call_element *top_elem = grpc_call_stack_element(call_stack, 0);
+  return top_elem->filter->get_peer(top_elem);
+}
+
 void grpc_subchannel_call_process_op(grpc_subchannel_call *call,
                                      grpc_transport_stream_op *op) {
   grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
diff --git a/src/core/client_config/subchannel.h b/src/core/client_config/subchannel.h
index a23a623277..d1cd33b2af 100644
--- a/src/core/client_config/subchannel.h
+++ b/src/core/client_config/subchannel.h
@@ -100,6 +100,9 @@ void grpc_subchannel_del_interested_party(grpc_subchannel *channel,
 void grpc_subchannel_call_process_op(grpc_subchannel_call *subchannel_call,
                                      grpc_transport_stream_op *op);
 
+/** continue querying for peer */
+char *grpc_subchannel_call_get_peer(grpc_subchannel_call *subchannel_call);
+
 struct grpc_subchannel_args {
   /** Channel filters for this channel - wrapped factories will likely
       want to mutate this */
diff --git a/src/core/census/grpc_context.h b/src/core/client_config/subchannel_factory_decorators/add_channel_arg.c
index 4637e7218e..7dc6d99ebe 100644
--- a/src/core/census/grpc_context.h
+++ b/src/core/client_config/subchannel_factory_decorators/add_channel_arg.c
@@ -31,27 +31,13 @@
  *
  */
 
-/* GRPC <--> CENSUS context interface */
-
-#ifndef CENSUS_GRPC_CONTEXT_H
-#define CENSUS_GRPC_CONTEXT_H
-
-#include <grpc/census.h>
-#include "src/core/surface/call.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Set census context for the call; Must be called before first call to
-   grpc_call_start_batch(). */
-void grpc_census_call_set_context(grpc_call *call, census_context *context);
-
-/* Retrieve the calls current census context. */
-census_context *grpc_census_call_get_context(grpc_call *call);
-
-#ifdef __cplusplus
+#include "src/core/client_config/subchannel_factory_decorators/add_channel_arg.h"
+#include "src/core/client_config/subchannel_factory_decorators/merge_channel_args.h"
+
+grpc_subchannel_factory *grpc_subchannel_factory_add_channel_arg(
+		grpc_subchannel_factory *input, const grpc_arg *arg) {
+	grpc_channel_args args;
+	args.num_args = 1;
+	args.args = (grpc_arg *)arg;
+	return grpc_subchannel_factory_merge_channel_args(input, &args);
 }
-#endif
-
-#endif /* CENSUS_GRPC_CONTEXT_H */
diff --git a/src/core/client_config/subchannel_factory_decorators/add_channel_arg.h b/src/core/client_config/subchannel_factory_decorators/add_channel_arg.h
new file mode 100644
index 0000000000..1937623374
--- /dev/null
+++ b/src/core/client_config/subchannel_factory_decorators/add_channel_arg.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_DECORATORS_ADD_CHANNEL_ARG_H
+#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_DECORATORS_ADD_CHANNEL_ARG_H
+
+#include "src/core/client_config/subchannel_factory.h"
+
+/** Takes a subchannel factory, returns a new one that mutates incoming
+    channel_args by adding a new argument; ownership of input, arg is retained
+    by the caller. */
+grpc_subchannel_factory *grpc_subchannel_factory_add_channel_arg(
+		grpc_subchannel_factory *input, const grpc_arg *arg);
+
+#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_DECORATORS_ADD_CHANNEL_ARG_H */
diff --git a/src/core/client_config/subchannel_factory_decorators/merge_channel_args.c b/src/core/client_config/subchannel_factory_decorators/merge_channel_args.c
new file mode 100644
index 0000000000..7e028857ac
--- /dev/null
+++ b/src/core/client_config/subchannel_factory_decorators/merge_channel_args.c
@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/client_config/subchannel_factory_decorators/merge_channel_args.h"
+#include <grpc/support/alloc.h>
+#include "src/core/channel/channel_args.h"
+
+typedef struct {
+  grpc_subchannel_factory base;
+  gpr_refcount refs;
+  grpc_subchannel_factory *wrapped;
+  grpc_channel_args *merge_args;
+} merge_args_factory;
+
+static void merge_args_factory_ref(grpc_subchannel_factory *scf) {
+  merge_args_factory *f = (merge_args_factory *)scf;
+  gpr_ref(&f->refs);
+}
+
+static void merge_args_factory_unref(grpc_subchannel_factory *scf) {
+  merge_args_factory *f = (merge_args_factory *)scf;
+  if (gpr_unref(&f->refs)) {
+  	grpc_subchannel_factory_unref(f->wrapped);
+    grpc_channel_args_destroy(f->merge_args);
+    gpr_free(f);
+  }
+}
+
+static grpc_subchannel *merge_args_factory_create_subchannel(
+    grpc_subchannel_factory *scf, grpc_subchannel_args *args) {
+  merge_args_factory *f = (merge_args_factory *)scf;
+  grpc_channel_args *final_args =
+      grpc_channel_args_merge(args->args, f->merge_args);
+  grpc_subchannel *s;
+  args->args = final_args;
+  s = grpc_subchannel_factory_create_subchannel(f->wrapped, args);
+  grpc_channel_args_destroy(final_args);
+  return s;
+}
+
+static const grpc_subchannel_factory_vtable merge_args_factory_vtable = {
+    merge_args_factory_ref, merge_args_factory_unref,
+    merge_args_factory_create_subchannel};
+
+grpc_subchannel_factory *grpc_subchannel_factory_merge_channel_args(
+		grpc_subchannel_factory *input, const grpc_channel_args *args) {
+  merge_args_factory *f = gpr_malloc(sizeof(*f));
+  f->base.vtable = &merge_args_factory_vtable;
+  gpr_ref_init(&f->refs, 1);
+  grpc_subchannel_factory_ref(input);
+  f->wrapped = input;
+  f->merge_args = grpc_channel_args_copy(args);
+  return &f->base;
+}
diff --git a/src/core/client_config/subchannel_factory_decorators/merge_channel_args.h b/src/core/client_config/subchannel_factory_decorators/merge_channel_args.h
new file mode 100644
index 0000000000..73a03b752f
--- /dev/null
+++ b/src/core/client_config/subchannel_factory_decorators/merge_channel_args.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_DECORATORS_MERGE_CHANNEL_ARGS_H
+#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_DECORATORS_MERGE_CHANNEL_ARGS_H
+
+#include "src/core/client_config/subchannel_factory.h"
+
+/** Takes a subchannel factory, returns a new one that mutates incoming
+    channel_args by adding a new argument; ownership of input, args is retained
+    by the caller. */
+grpc_subchannel_factory *grpc_subchannel_factory_merge_channel_args(
+		grpc_subchannel_factory *input, const grpc_channel_args *args);
+
+#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_DECORATORS_MERGE_CHANNEL_ARGS_H */
diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c
index 65997d5f44..9012070e8e 100644
--- a/src/core/httpcli/httpcli.c
+++ b/src/core/httpcli/httpcli.c
@@ -40,9 +40,7 @@
 #include "src/core/iomgr/resolve_address.h"
 #include "src/core/iomgr/tcp_client.h"
 #include "src/core/httpcli/format_request.h"
-#include "src/core/httpcli/httpcli_security_connector.h"
 #include "src/core/httpcli/parser.h"
-#include "src/core/security/secure_transport_setup.h"
 #include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -57,7 +55,7 @@ typedef struct {
   char *host;
   gpr_timespec deadline;
   int have_read_byte;
-  int use_ssl;
+  const grpc_httpcli_handshaker *handshaker;
   grpc_httpcli_response_cb on_response;
   void *user_data;
   grpc_httpcli_context *context;
@@ -68,6 +66,16 @@ typedef struct {
 static grpc_httpcli_get_override g_get_override = NULL;
 static grpc_httpcli_post_override g_post_override = NULL;
 
+static void plaintext_handshake(void *arg, grpc_endpoint *endpoint,
+                                const char *host,
+                                void (*on_done)(void *arg,
+                                                grpc_endpoint *endpoint)) {
+  on_done(arg, endpoint);
+}
+
+const grpc_httpcli_handshaker grpc_httpcli_plaintext = {"http",
+                                                        plaintext_handshake};
+
 void grpc_httpcli_context_init(grpc_httpcli_context *context) {
   grpc_pollset_set_init(&context->pollset_set);
 }
@@ -163,18 +171,16 @@ static void start_write(internal_request *req) {
   }
 }
 
-static void on_secure_transport_setup_done(void *rp,
-                                           grpc_security_status status,
-                                           grpc_endpoint *wrapped_endpoint,
-                                           grpc_endpoint *secure_endpoint) {
-  internal_request *req = rp;
-  if (status != GRPC_SECURITY_OK) {
-    gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status);
-    finish(req, 0);
-  } else {
-    req->ep = secure_endpoint;
-    start_write(req);
+static void on_handshake_done(void *arg, grpc_endpoint *ep) {
+  internal_request *req = arg;
+
+  if (!ep) {
+    next_address(req);
+    return;
   }
+
+  req->ep = ep;
+  start_write(req);
 }
 
 static void on_connected(void *arg, grpc_endpoint *tcp) {
@@ -184,25 +190,7 @@ static void on_connected(void *arg, grpc_endpoint *tcp) {
     next_address(req);
     return;
   }
-  req->ep = tcp;
-  if (req->use_ssl) {
-    grpc_channel_security_connector *sc = NULL;
-    const unsigned char *pem_root_certs = NULL;
-    size_t pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs);
-    if (pem_root_certs == NULL || pem_root_certs_size == 0) {
-      gpr_log(GPR_ERROR, "Could not get default pem root certs.");
-      finish(req, 0);
-      return;
-    }
-    GPR_ASSERT(grpc_httpcli_ssl_channel_security_connector_create(
-                   pem_root_certs, pem_root_certs_size, req->host, &sc) ==
-               GRPC_SECURITY_OK);
-    grpc_setup_secure_transport(&sc->base, tcp, on_secure_transport_setup_done,
-                                req);
-    GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
-  } else {
-    start_write(req);
-  }
+  req->handshaker->handshake(req, tcp, req->host, on_handshake_done);
 }
 
 static void next_address(internal_request *req) {
@@ -245,18 +233,17 @@ void grpc_httpcli_get(grpc_httpcli_context *context, grpc_pollset *pollset,
   req->on_response = on_response;
   req->user_data = user_data;
   req->deadline = deadline;
-  req->use_ssl = request->use_ssl;
+  req->handshaker =
+      request->handshaker ? request->handshaker : &grpc_httpcli_plaintext;
   req->context = context;
   req->pollset = pollset;
   gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->path);
   grpc_iomgr_register_object(&req->iomgr_obj, name);
   gpr_free(name);
-  if (req->use_ssl) {
-    req->host = gpr_strdup(request->host);
-  }
+  req->host = gpr_strdup(request->host);
 
   grpc_pollset_set_add_pollset(&req->context->pollset_set, req->pollset);
-  grpc_resolve_address(request->host, req->use_ssl ? "https" : "http",
+  grpc_resolve_address(request->host, req->handshaker->default_port,
                        on_resolved, req);
 }
 
@@ -279,18 +266,17 @@ void grpc_httpcli_post(grpc_httpcli_context *context, grpc_pollset *pollset,
   req->on_response = on_response;
   req->user_data = user_data;
   req->deadline = deadline;
-  req->use_ssl = request->use_ssl;
+  req->handshaker =
+      request->handshaker ? request->handshaker : &grpc_httpcli_plaintext;
   req->context = context;
   req->pollset = pollset;
   gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->path);
   grpc_iomgr_register_object(&req->iomgr_obj, name);
   gpr_free(name);
-  if (req->use_ssl) {
-    req->host = gpr_strdup(request->host);
-  }
+  req->host = gpr_strdup(request->host);
 
   grpc_pollset_set_add_pollset(&req->context->pollset_set, req->pollset);
-  grpc_resolve_address(request->host, req->use_ssl ? "https" : "http",
+  grpc_resolve_address(request->host, req->handshaker->default_port,
                        on_resolved, req);
 }
 
diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h
index ab98178f8a..c45966714c 100644
--- a/src/core/httpcli/httpcli.h
+++ b/src/core/httpcli/httpcli.h
@@ -38,6 +38,7 @@
 
 #include <grpc/support/time.h>
 
+#include "src/core/iomgr/endpoint.h"
 #include "src/core/iomgr/pollset_set.h"
 
 /* User agent this library reports */
@@ -58,6 +59,15 @@ typedef struct grpc_httpcli_context {
   grpc_pollset_set pollset_set;
 } grpc_httpcli_context;
 
+typedef struct {
+  const char *default_port;
+  void (*handshake)(void *arg, grpc_endpoint *endpoint, const char *host,
+                    void (*on_done)(void *arg, grpc_endpoint *endpoint));
+} grpc_httpcli_handshaker;
+
+extern const grpc_httpcli_handshaker grpc_httpcli_plaintext;
+extern const grpc_httpcli_handshaker grpc_httpcli_ssl;
+
 /* A request */
 typedef struct grpc_httpcli_request {
   /* The host name to connect to */
@@ -69,8 +79,8 @@ typedef struct grpc_httpcli_request {
        Host, Connection, User-Agent */
   size_t hdr_count;
   grpc_httpcli_header *hdrs;
-  /* whether to use ssl for the request */
-  int use_ssl;
+  /* handshaker to use ssl for the request */
+  const grpc_httpcli_handshaker *handshaker;
 } grpc_httpcli_request;
 
 /* A response */
diff --git a/src/core/httpcli/httpcli_security_connector.c b/src/core/httpcli/httpcli_security_connector.c
index ce0d3d5a70..7887f9d530 100644
--- a/src/core/httpcli/httpcli_security_connector.c
+++ b/src/core/httpcli/httpcli_security_connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/httpcli/httpcli_security_connector.h"
+#include "src/core/httpcli/httpcli.h"
 
 #include <string.h>
 
@@ -96,7 +96,7 @@ static grpc_security_status httpcli_ssl_check_peer(grpc_security_connector *sc,
 static grpc_security_connector_vtable httpcli_ssl_vtable = {
     httpcli_ssl_destroy, httpcli_ssl_create_handshaker, httpcli_ssl_check_peer};
 
-grpc_security_status grpc_httpcli_ssl_channel_security_connector_create(
+static grpc_security_status httpcli_ssl_channel_security_connector_create(
     const unsigned char *pem_root_certs, size_t pem_root_certs_size,
     const char *secure_peer_name, grpc_channel_security_connector **sc) {
   tsi_result result = TSI_OK;
@@ -130,3 +130,48 @@ grpc_security_status grpc_httpcli_ssl_channel_security_connector_create(
   *sc = &c->base;
   return GRPC_SECURITY_OK;
 }
+
+/* handshaker */
+
+typedef struct {
+  void (*func)(void *arg, grpc_endpoint *endpoint);
+  void *arg;
+} on_done_closure;
+
+static void on_secure_transport_setup_done(void *rp,
+                                           grpc_security_status status,
+                                           grpc_endpoint *wrapped_endpoint,
+                                           grpc_endpoint *secure_endpoint) {
+  on_done_closure *c = rp;
+  if (status != GRPC_SECURITY_OK) {
+    gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status);
+    c->func(c->arg, NULL);
+  } else {
+    c->func(c->arg, secure_endpoint);
+  }
+  gpr_free(c);
+}
+
+static void ssl_handshake(void *arg, grpc_endpoint *tcp, const char *host,
+                          void (*on_done)(void *arg, grpc_endpoint *endpoint)) {
+  grpc_channel_security_connector *sc = NULL;
+  const unsigned char *pem_root_certs = NULL;
+  on_done_closure *c = gpr_malloc(sizeof(*c));
+  size_t pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs);
+  if (pem_root_certs == NULL || pem_root_certs_size == 0) {
+    gpr_log(GPR_ERROR, "Could not get default pem root certs.");
+    on_done(arg, NULL);
+    gpr_free(c);
+    return;
+  }
+  c->func = on_done;
+  c->arg = arg;
+  GPR_ASSERT(httpcli_ssl_channel_security_connector_create(
+                 pem_root_certs, pem_root_certs_size, host, &sc) ==
+             GRPC_SECURITY_OK);
+  grpc_setup_secure_transport(&sc->base, tcp, on_secure_transport_setup_done,
+                              c);
+  GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
+}
+
+const grpc_httpcli_handshaker grpc_httpcli_ssl = {"https", ssl_handshake};
diff --git a/src/core/iomgr/alarm.c b/src/core/iomgr/alarm.c
index 931f746f75..68d33b9cf6 100644
--- a/src/core/iomgr/alarm.c
+++ b/src/core/iomgr/alarm.c
@@ -361,7 +361,9 @@ static int run_some_expired_alarms(gpr_mu *drop_mu, gpr_timespec now,
 
 int grpc_alarm_check(gpr_mu *drop_mu, gpr_timespec now, gpr_timespec *next) {
   GPR_ASSERT(now.clock_type == g_clock_type);
-  return run_some_expired_alarms(drop_mu, now, next, 1);
+  return run_some_expired_alarms(
+      drop_mu, now, next, 
+      gpr_time_cmp(now, gpr_inf_future(now.clock_type)) != 0);
 }
 
 gpr_timespec grpc_alarm_list_next_timeout(void) {
diff --git a/src/core/iomgr/endpoint.c b/src/core/iomgr/endpoint.c
index 96487958a7..744fe7656c 100644
--- a/src/core/iomgr/endpoint.c
+++ b/src/core/iomgr/endpoint.c
@@ -50,6 +50,14 @@ void grpc_endpoint_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) {
   ep->vtable->add_to_pollset(ep, pollset);
 }
 
+void grpc_endpoint_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pollset_set) {
+  ep->vtable->add_to_pollset_set(ep, pollset_set);
+}
+
 void grpc_endpoint_shutdown(grpc_endpoint *ep) { ep->vtable->shutdown(ep); }
 
 void grpc_endpoint_destroy(grpc_endpoint *ep) { ep->vtable->destroy(ep); }
+
+char *grpc_endpoint_get_peer(grpc_endpoint *ep) {
+  return ep->vtable->get_peer(ep);
+}
diff --git a/src/core/iomgr/endpoint.h b/src/core/iomgr/endpoint.h
index 881e851800..a2216925f9 100644
--- a/src/core/iomgr/endpoint.h
+++ b/src/core/iomgr/endpoint.h
@@ -35,6 +35,7 @@
 #define GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_H
 
 #include "src/core/iomgr/pollset.h"
+#include "src/core/iomgr/pollset_set.h"
 #include <grpc/support/slice.h>
 #include <grpc/support/time.h>
 
@@ -70,14 +71,18 @@ struct grpc_endpoint_vtable {
                                       size_t nslices, grpc_endpoint_write_cb cb,
                                       void *user_data);
   void (*add_to_pollset)(grpc_endpoint *ep, grpc_pollset *pollset);
+  void (*add_to_pollset_set)(grpc_endpoint *ep, grpc_pollset_set *pollset);
   void (*shutdown)(grpc_endpoint *ep);
   void (*destroy)(grpc_endpoint *ep);
+  char *(*get_peer)(grpc_endpoint *ep);
 };
 
 /* When data is available on the connection, calls the callback with slices. */
 void grpc_endpoint_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb,
                                   void *user_data);
 
+char *grpc_endpoint_get_peer(grpc_endpoint *ep);
+
 /* Write slices out to the socket.
 
    If the connection is ready for more data after the end of the call, it
@@ -98,6 +103,7 @@ void grpc_endpoint_destroy(grpc_endpoint *ep);
 /* Add an endpoint to a pollset, so that when the pollset is polled, events from
    this endpoint are considered */
 void grpc_endpoint_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset);
+void grpc_endpoint_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pollset_set);
 
 struct grpc_endpoint {
   const grpc_endpoint_vtable *vtable;
diff --git a/src/core/iomgr/endpoint_pair_posix.c b/src/core/iomgr/endpoint_pair_posix.c
index fa2d2555d6..deae9c6875 100644
--- a/src/core/iomgr/endpoint_pair_posix.c
+++ b/src/core/iomgr/endpoint_pair_posix.c
@@ -66,12 +66,12 @@ grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
   create_sockets(sv);
 
   gpr_asprintf(&final_name, "%s:client", name);
-  p.client =
-      grpc_tcp_create(grpc_fd_create(sv[1], final_name), read_slice_size);
+  p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), read_slice_size,
+                             "socketpair-server");
   gpr_free(final_name);
   gpr_asprintf(&final_name, "%s:server", name);
-  p.server =
-      grpc_tcp_create(grpc_fd_create(sv[0], final_name), read_slice_size);
+  p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), read_slice_size,
+                             "socketpair-client");
   gpr_free(final_name);
   return p;
 }
diff --git a/src/core/iomgr/endpoint_pair_windows.c b/src/core/iomgr/endpoint_pair_windows.c
index c6790b2937..e8295df8b3 100644
--- a/src/core/iomgr/endpoint_pair_windows.c
+++ b/src/core/iomgr/endpoint_pair_windows.c
@@ -81,8 +81,10 @@ grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, size_t read
   SOCKET sv[2];
   grpc_endpoint_pair p;
   create_sockets(sv);
-  p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"));
-  p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"));
+  p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"),
+                             "endpoint:server");
+  p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"),
+                             "endpoint:client");
   return p;
 }
 
diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c
index 6ad377ce1c..2d08a77a70 100644
--- a/src/core/iomgr/fd_posix.c
+++ b/src/core/iomgr/fd_posix.c
@@ -102,6 +102,7 @@ static grpc_fd *alloc_fd(int fd) {
   r->freelist_next = NULL;
   r->read_watcher = r->write_watcher = NULL;
   r->on_done_closure = NULL;
+  r->closed = 0;
   return r;
 }
 
@@ -167,13 +168,19 @@ int grpc_fd_is_orphaned(grpc_fd *fd) {
   return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
 }
 
+static void pollset_kick_locked(grpc_pollset *pollset) {
+  gpr_mu_lock(GRPC_POLLSET_MU(pollset));
+  grpc_pollset_kick(pollset, NULL);
+  gpr_mu_unlock(GRPC_POLLSET_MU(pollset));
+}
+
 static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
   if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) {
-    grpc_pollset_force_kick(fd->inactive_watcher_root.next->pollset);
+    pollset_kick_locked(fd->inactive_watcher_root.next->pollset);
   } else if (fd->read_watcher) {
-    grpc_pollset_force_kick(fd->read_watcher->pollset);
+    pollset_kick_locked(fd->read_watcher->pollset);
   } else if (fd->write_watcher) {
-    grpc_pollset_force_kick(fd->write_watcher->pollset);
+    pollset_kick_locked(fd->write_watcher->pollset);
   }
 }
 
@@ -187,13 +194,13 @@ static void wake_all_watchers_locked(grpc_fd *fd) {
   grpc_fd_watcher *watcher;
   for (watcher = fd->inactive_watcher_root.next;
        watcher != &fd->inactive_watcher_root; watcher = watcher->next) {
-    grpc_pollset_force_kick(watcher->pollset);
+    pollset_kick_locked(watcher->pollset);
   }
   if (fd->read_watcher) {
-    grpc_pollset_force_kick(fd->read_watcher->pollset);
+    pollset_kick_locked(fd->read_watcher->pollset);
   }
   if (fd->write_watcher && fd->write_watcher != fd->read_watcher) {
-    grpc_pollset_force_kick(fd->write_watcher->pollset);
+    pollset_kick_locked(fd->write_watcher->pollset);
   }
 }
 
@@ -209,6 +216,8 @@ void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done,
   REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
   gpr_mu_lock(&fd->watcher_mu);
   if (!has_watchers(fd)) {
+    GPR_ASSERT(!fd->closed);
+    fd->closed = 1;
     close(fd->fd);
     if (fd->on_done_closure) {
       grpc_iomgr_add_callback(fd->on_done_closure);
@@ -373,13 +382,15 @@ gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
     return 0;
   }
   /* if there is nobody polling for read, but we need to, then start doing so */
-  if (read_mask && !fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) {
+  if (read_mask && !fd->read_watcher &&
+      (gpr_uintptr)gpr_atm_acq_load(&fd->readst) > READY) {
     fd->read_watcher = watcher;
     mask |= read_mask;
   }
   /* if there is nobody polling for write, but we need to, then start doing so
    */
-  if (write_mask && !fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) {
+  if (write_mask && !fd->write_watcher &&
+      (gpr_uintptr)gpr_atm_acq_load(&fd->writest) > READY) {
     fd->write_watcher = watcher;
     mask |= write_mask;
   }
@@ -426,7 +437,8 @@ void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) {
   if (kick) {
     maybe_wake_one_watcher_locked(fd);
   }
-  if (grpc_fd_is_orphaned(fd) && !has_watchers(fd)) {
+  if (grpc_fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) {
+    fd->closed = 1;
     close(fd->fd);
     if (fd->on_done_closure != NULL) {
       grpc_iomgr_add_callback(fd->on_done_closure);
diff --git a/src/core/iomgr/fd_posix.h b/src/core/iomgr/fd_posix.h
index 94d0019fa4..835e9b339a 100644
--- a/src/core/iomgr/fd_posix.h
+++ b/src/core/iomgr/fd_posix.h
@@ -60,6 +60,7 @@ struct grpc_fd {
 
   gpr_mu set_state_mu;
   gpr_atm shutdown;
+  int closed;
 
   /* The watcher list.
 
@@ -108,7 +109,8 @@ grpc_fd *grpc_fd_create(int fd, const char *name);
    on_done is called when the underlying file descriptor is definitely close()d.
    If on_done is NULL, no callback will be made.
    Requires: *fd initialized; no outstanding notify_on_read or
-   notify_on_write. */
+   notify_on_write.
+   MUST NOT be called with a pollset lock taken */
 void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done,
                     const char *reason);
 
@@ -121,11 +123,13 @@ void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done,
    i.e. a combination of read_mask and write_mask determined by the fd's current
    interest in said events.
    Polling strategies that do not need to alter their behavior depending on the
-   fd's current interest (such as epoll) do not need to call this function. */
+   fd's current interest (such as epoll) do not need to call this function.
+   MUST NOT be called with a pollset lock taken */
 gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
                               gpr_uint32 read_mask, gpr_uint32 write_mask,
                               grpc_fd_watcher *rec);
-/* Complete polling previously started with grpc_fd_begin_poll */
+/* Complete polling previously started with grpc_fd_begin_poll
+   MUST NOT be called with a pollset lock taken */
 void grpc_fd_end_poll(grpc_fd_watcher *rec, int got_read, int got_write);
 
 /* Return 1 if this fd is orphaned, 0 otherwise */
diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c
index a18c176b30..fdc9adf4af 100644
--- a/src/core/iomgr/iomgr.c
+++ b/src/core/iomgr/iomgr.c
@@ -88,6 +88,7 @@ void grpc_kick_poller(void) {
 
 void grpc_iomgr_init(void) {
   gpr_thd_id id;
+  g_shutdown = 0;
   gpr_mu_init(&g_mu);
   gpr_cv_init(&g_rcv);
   grpc_alarm_list_init(gpr_now(GPR_CLOCK_MONOTONIC));
@@ -146,7 +147,6 @@ void grpc_iomgr_shutdown(void) {
       continue;
     }
     if (grpc_alarm_check(&g_mu, gpr_inf_future(GPR_CLOCK_MONOTONIC), NULL)) {
-      gpr_log(GPR_DEBUG, "got late alarm");
       continue;
     }
     if (g_root_object.next != &g_root_object) {
diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h
index c40188b3c9..c474e4dbf1 100644
--- a/src/core/iomgr/pollset.h
+++ b/src/core/iomgr/pollset.h
@@ -37,6 +37,8 @@
 #include <grpc/support/port_platform.h>
 #include <grpc/support/time.h>
 
+#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
+
 /* A grpc_pollset is a set of file descriptors that a higher level item is
    interested in. For example:
     - a server will typically keep a pollset containing all connected channels,
@@ -63,13 +65,24 @@ void grpc_pollset_destroy(grpc_pollset *pollset);
    descriptors.
    Requires GRPC_POLLSET_MU(pollset) locked.
    May unlock GRPC_POLLSET_MU(pollset) during its execution.
-   
+
+   worker is a (platform-specific) handle that can be used to wake up
+   from grpc_pollset_work before any events are received and before the timeout
+   has expired. It is both initialized and destroyed by grpc_pollset_work.
+   Initialization of worker is guaranteed to occur BEFORE the
+   GRPC_POLLSET_MU(pollset) is released for the first time by
+   grpc_pollset_work, and it is guaranteed that GRPC_POLLSET_MU(pollset) will
+   not be released by grpc_pollset_work AFTER worker has been destroyed.
+
    Returns true if some work has been done, and false if the deadline
-   got attained. */
-int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline);
+   expired. */
+int grpc_pollset_work(grpc_pollset *pollset, grpc_pollset_worker *worker,
+                      gpr_timespec deadline);
 
 /* Break one polling thread out of polling work for this pollset.
-   Requires GRPC_POLLSET_MU(pollset) locked. */
-void grpc_pollset_kick(grpc_pollset *pollset);
+   If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers.
+   Otherwise, if specific_worker is non-NULL, then kick that worker. */
+void grpc_pollset_kick(grpc_pollset *pollset,
+                       grpc_pollset_worker *specific_worker);
 
 #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */
diff --git a/src/core/iomgr/pollset_kick_posix.c b/src/core/iomgr/pollset_kick_posix.c
deleted file mode 100644
index 51021784f2..0000000000
--- a/src/core/iomgr/pollset_kick_posix.c
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <grpc/support/port_platform.h>
-
-#ifdef GPR_POSIX_SOCKET
-#include "src/core/iomgr/pollset_kick_posix.h"
-
-#include <errno.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "src/core/iomgr/socket_utils_posix.h"
-#include "src/core/iomgr/wakeup_fd_posix.h"
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-
-/* This implementation is based on a freelist of wakeup fds, with extra logic to
- * handle kicks while there is no attached fd. */
-
-/* TODO(klempner): Autosize this, and consider providing a way to disable the
- * cap entirely on systems with large fd limits */
-#define GRPC_MAX_CACHED_WFDS 50
-
-static grpc_kick_fd_info *fd_freelist = NULL;
-static int fd_freelist_count = 0;
-static gpr_mu fd_freelist_mu;
-
-static grpc_kick_fd_info *allocate_wfd(void) {
-  grpc_kick_fd_info *info = NULL;
-  gpr_mu_lock(&fd_freelist_mu);
-  if (fd_freelist != NULL) {
-    info = fd_freelist;
-    fd_freelist = fd_freelist->next;
-    --fd_freelist_count;
-  }
-  gpr_mu_unlock(&fd_freelist_mu);
-  if (info == NULL) {
-    info = gpr_malloc(sizeof(*info));
-    grpc_wakeup_fd_create(&info->wakeup_fd);
-    info->next = NULL;
-  }
-  return info;
-}
-
-static void destroy_wfd(grpc_kick_fd_info *wfd) {
-  grpc_wakeup_fd_destroy(&wfd->wakeup_fd);
-  gpr_free(wfd);
-}
-
-static void free_wfd(grpc_kick_fd_info *fd_info) {
-  gpr_mu_lock(&fd_freelist_mu);
-  if (fd_freelist_count < GRPC_MAX_CACHED_WFDS) {
-    fd_info->next = fd_freelist;
-    fd_freelist = fd_info;
-    fd_freelist_count++;
-    fd_info = NULL;
-  }
-  gpr_mu_unlock(&fd_freelist_mu);
-
-  if (fd_info) {
-    destroy_wfd(fd_info);
-  }
-}
-
-void grpc_pollset_kick_init(grpc_pollset_kick_state *kick_state) {
-  gpr_mu_init(&kick_state->mu);
-  kick_state->kicked = 0;
-  kick_state->fd_list.next = kick_state->fd_list.prev = &kick_state->fd_list;
-}
-
-void grpc_pollset_kick_destroy(grpc_pollset_kick_state *kick_state) {
-  gpr_mu_destroy(&kick_state->mu);
-  GPR_ASSERT(kick_state->fd_list.next == &kick_state->fd_list);
-}
-
-grpc_kick_fd_info *grpc_pollset_kick_pre_poll(
-    grpc_pollset_kick_state *kick_state) {
-  grpc_kick_fd_info *fd_info;
-  gpr_mu_lock(&kick_state->mu);
-  if (kick_state->kicked) {
-    kick_state->kicked = 0;
-    gpr_mu_unlock(&kick_state->mu);
-    return NULL;
-  }
-  fd_info = allocate_wfd();
-  fd_info->next = &kick_state->fd_list;
-  fd_info->prev = fd_info->next->prev;
-  fd_info->next->prev = fd_info->prev->next = fd_info;
-  gpr_mu_unlock(&kick_state->mu);
-  return fd_info;
-}
-
-void grpc_pollset_kick_consume(grpc_pollset_kick_state *kick_state,
-                               grpc_kick_fd_info *fd_info) {
-  grpc_wakeup_fd_consume_wakeup(&fd_info->wakeup_fd);
-}
-
-void grpc_pollset_kick_post_poll(grpc_pollset_kick_state *kick_state,
-                                 grpc_kick_fd_info *fd_info) {
-  gpr_mu_lock(&kick_state->mu);
-  fd_info->next->prev = fd_info->prev;
-  fd_info->prev->next = fd_info->next;
-  free_wfd(fd_info);
-  gpr_mu_unlock(&kick_state->mu);
-}
-
-void grpc_pollset_kick_kick(grpc_pollset_kick_state *kick_state) {
-  gpr_mu_lock(&kick_state->mu);
-  if (kick_state->fd_list.next != &kick_state->fd_list) {
-    grpc_wakeup_fd_wakeup(&kick_state->fd_list.next->wakeup_fd);
-  } else {
-    kick_state->kicked = 1;
-  }
-  gpr_mu_unlock(&kick_state->mu);
-}
-
-void grpc_pollset_kick_global_init_fallback_fd(void) {
-  gpr_mu_init(&fd_freelist_mu);
-  grpc_wakeup_fd_global_init_force_fallback();
-}
-
-void grpc_pollset_kick_global_init(void) {
-  gpr_mu_init(&fd_freelist_mu);
-  grpc_wakeup_fd_global_init();
-}
-
-void grpc_pollset_kick_global_destroy(void) {
-  while (fd_freelist != NULL) {
-    grpc_kick_fd_info *current = fd_freelist;
-    fd_freelist = fd_freelist->next;
-    destroy_wfd(current);
-  }
-  grpc_wakeup_fd_global_destroy();
-  gpr_mu_destroy(&fd_freelist_mu);
-}
-
-#endif /* GPR_POSIX_SOCKET */
diff --git a/src/core/iomgr/pollset_kick_posix.h b/src/core/iomgr/pollset_kick_posix.h
deleted file mode 100644
index 77e32a8d51..0000000000
--- a/src/core/iomgr/pollset_kick_posix.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_POSIX_H
-
-#include "src/core/iomgr/wakeup_fd_posix.h"
-#include <grpc/support/sync.h>
-
-/* pollset kicking allows breaking a thread out of polling work for
-   a given pollset.
-   writing a byte to a pipe is used as a posix-ly portable base
-   mechanism, and eventfds are utilized on Linux for better performance. */
-
-typedef struct grpc_kick_fd_info {
-  grpc_wakeup_fd_info wakeup_fd;
-  /* used for polling list and free list */
-  struct grpc_kick_fd_info *next;
-  /* only used when polling */
-  struct grpc_kick_fd_info *prev;
-} grpc_kick_fd_info;
-
-typedef struct grpc_pollset_kick_state {
-  gpr_mu mu;
-  int kicked;
-  struct grpc_kick_fd_info fd_list;
-} grpc_pollset_kick_state;
-
-#define GRPC_POLLSET_KICK_GET_FD(kick_fd_info) \
-  GRPC_WAKEUP_FD_GET_READ_FD(&(kick_fd_info)->wakeup_fd)
-
-/* This is an abstraction around the typical pipe mechanism for waking up a
-   thread sitting in a poll() style call. */
-
-void grpc_pollset_kick_global_init(void);
-void grpc_pollset_kick_global_destroy(void);
-
-void grpc_pollset_kick_init(grpc_pollset_kick_state *kick_state);
-void grpc_pollset_kick_destroy(grpc_pollset_kick_state *kick_state);
-
-/* Guarantees a pure posix implementation rather than a specialized one, if
- * applicable. Intended for testing. */
-void grpc_pollset_kick_global_init_fallback_fd(void);
-
-/* Must be called before entering poll(). If return value is NULL, this consumed
-   an existing kick. Otherwise the return value is an FD to add to the poll set.
- */
-grpc_kick_fd_info *grpc_pollset_kick_pre_poll(
-    grpc_pollset_kick_state *kick_state);
-
-/* Consume an existing kick. Must be called after poll returns that the fd was
-   readable, and before calling kick_post_poll. */
-void grpc_pollset_kick_consume(grpc_pollset_kick_state *kick_state,
-                               grpc_kick_fd_info *fd_info);
-
-/* Must be called after pre_poll, and after consume if applicable */
-void grpc_pollset_kick_post_poll(grpc_pollset_kick_state *kick_state,
-                                 grpc_kick_fd_info *fd_info);
-
-/* Actually kick */
-void grpc_pollset_kick_kick(grpc_pollset_kick_state *kick_state);
-
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_POSIX_H */
diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c
index d697b59e4c..1320c64579 100644
--- a/src/core/iomgr/pollset_multipoller_with_epoll.c
+++ b/src/core/iomgr/pollset_multipoller_with_epoll.c
@@ -36,6 +36,7 @@
 #ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
 
 #include <errno.h>
+#include <poll.h>
 #include <string.h>
 #include <sys/epoll.h>
 #include <unistd.h>
@@ -44,23 +45,28 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+typedef struct wakeup_fd_hdl {
+  grpc_wakeup_fd wakeup_fd;
+  struct wakeup_fd_hdl *next;
+} wakeup_fd_hdl;
+
+typedef struct {
+  grpc_pollset *pollset;
+  grpc_fd *fd;
+  grpc_iomgr_closure closure;
+} delayed_add;
+
 typedef struct {
   int epoll_fd;
-  grpc_wakeup_fd_info wakeup_fd;
+  wakeup_fd_hdl *free_wakeup_fds;
 } pollset_hdr;
 
-static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset,
-                                                grpc_fd *fd,
-                                                int and_unlock_pollset) {
+static void finally_add_fd(grpc_pollset *pollset, grpc_fd *fd) {
   pollset_hdr *h = pollset->data.ptr;
   struct epoll_event ev;
   int err;
   grpc_fd_watcher watcher;
 
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-
   /* We pretend to be polling whilst adding an fd to keep the fd from being
      closed during the add. This may result in a spurious wakeup being assigned
      to this pollset whilst adding, but that should be benign. */
@@ -80,6 +86,52 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset,
   grpc_fd_end_poll(&watcher, 0, 0);
 }
 
+static void perform_delayed_add(void *arg, int iomgr_status) {
+  delayed_add *da = arg;
+  int do_shutdown_cb = 0;
+
+  if (!grpc_fd_is_orphaned(da->fd)) {
+    finally_add_fd(da->pollset, da->fd);
+  }
+
+  gpr_mu_lock(&da->pollset->mu);
+  da->pollset->in_flight_cbs--;
+  if (da->pollset->shutting_down) {
+    /* We don't care about this pollset anymore. */
+    if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) {
+      GPR_ASSERT(!grpc_pollset_has_workers(da->pollset));
+      da->pollset->called_shutdown = 1;
+      do_shutdown_cb = 1;
+    }
+  }
+  gpr_mu_unlock(&da->pollset->mu);
+
+  GRPC_FD_UNREF(da->fd, "delayed_add");
+
+  if (do_shutdown_cb) {
+    da->pollset->shutdown_done_cb(da->pollset->shutdown_done_arg);
+  }
+
+  gpr_free(da);
+}
+
+static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset,
+                                                grpc_fd *fd,
+                                                int and_unlock_pollset) {
+  if (and_unlock_pollset) {
+    gpr_mu_unlock(&pollset->mu);
+    finally_add_fd(pollset, fd);
+  } else {
+    delayed_add *da = gpr_malloc(sizeof(*da));
+    da->pollset = pollset;
+    da->fd = fd;
+    GRPC_FD_REF(fd, "delayed_add");
+    grpc_iomgr_closure_init(&da->closure, perform_delayed_add, da);
+    pollset->in_flight_cbs++;
+    grpc_iomgr_add_callback(&da->closure);
+  }
+}
+
 static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset,
                                                 grpc_fd *fd,
                                                 int and_unlock_pollset) {
@@ -103,12 +155,14 @@ static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset,
 #define GRPC_EPOLL_MAX_EVENTS 1000
 
 static void multipoll_with_epoll_pollset_maybe_work(
-    grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now,
-    int allow_synchronous_callback) {
+    grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec deadline,
+    gpr_timespec now, int allow_synchronous_callback) {
   struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
   int ep_rv;
+  int poll_rv;
   pollset_hdr *h = pollset->data.ptr;
   int timeout_ms;
+  struct pollfd pfds[2];
 
   /* If you want to ignore epoll's ability to sanely handle parallel pollers,
    * for a more apples-to-apples performance comparison with poll, add a
@@ -116,43 +170,58 @@ static void multipoll_with_epoll_pollset_maybe_work(
    * here.
    */
 
-  pollset->counter += 1;
   gpr_mu_unlock(&pollset->mu);
 
   timeout_ms = grpc_poll_deadline_to_millis_timeout(deadline, now);
 
-  do {
-    ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms);
-    if (ep_rv < 0) {
-      if (errno != EINTR) {
-        gpr_log(GPR_ERROR, "epoll_wait() failed: %s", strerror(errno));
-      }
-    } else {
-      int i;
-      for (i = 0; i < ep_rv; ++i) {
-        if (ep_ev[i].data.ptr == 0) {
-          grpc_wakeup_fd_consume_wakeup(&h->wakeup_fd);
-        } else {
-          grpc_fd *fd = ep_ev[i].data.ptr;
-          /* TODO(klempner): We might want to consider making err and pri
-           * separate events */
-          int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
-          int read = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
-          int write = ep_ev[i].events & EPOLLOUT;
-          if (read || cancel) {
-            grpc_fd_become_readable(fd, allow_synchronous_callback);
+  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd);
+  pfds[0].events = POLLIN;
+  pfds[0].revents = 0;
+  pfds[1].fd = h->epoll_fd;
+  pfds[1].events = POLLIN;
+  pfds[1].revents = 0;
+
+  poll_rv = poll(pfds, 2, timeout_ms);
+
+  if (poll_rv < 0) {
+    if (errno != EINTR) {
+      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
+    }
+  } else if (poll_rv == 0) {
+    /* do nothing */
+  } else {
+    if (pfds[0].revents) {
+      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd);
+    }
+    if (pfds[1].revents) {
+      do {
+        ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
+        if (ep_rv < 0) {
+          if (errno != EINTR) {
+            gpr_log(GPR_ERROR, "epoll_wait() failed: %s", strerror(errno));
           }
-          if (write || cancel) {
-            grpc_fd_become_writable(fd, allow_synchronous_callback);
+        } else {
+          int i;
+          for (i = 0; i < ep_rv; ++i) {
+            grpc_fd *fd = ep_ev[i].data.ptr;
+            /* TODO(klempner): We might want to consider making err and pri
+             * separate events */
+            int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
+            int read = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
+            int write = ep_ev[i].events & EPOLLOUT;
+            if (read || cancel) {
+              grpc_fd_become_readable(fd, allow_synchronous_callback);
+            }
+            if (write || cancel) {
+              grpc_fd_become_writable(fd, allow_synchronous_callback);
+            }
           }
         }
-      }
+      } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
     }
-    timeout_ms = 0;
-  } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
+  }
 
   gpr_mu_lock(&pollset->mu);
-  pollset->counter -= 1;
 }
 
 static void multipoll_with_epoll_pollset_finish_shutdown(
@@ -160,21 +229,14 @@ static void multipoll_with_epoll_pollset_finish_shutdown(
 
 static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) {
   pollset_hdr *h = pollset->data.ptr;
-  grpc_wakeup_fd_destroy(&h->wakeup_fd);
   close(h->epoll_fd);
   gpr_free(h);
 }
 
-static void epoll_kick(grpc_pollset *pollset) {
-  pollset_hdr *h = pollset->data.ptr;
-  grpc_wakeup_fd_wakeup(&h->wakeup_fd);
-}
-
 static const grpc_pollset_vtable multipoll_with_epoll_pollset = {
     multipoll_with_epoll_pollset_add_fd,
     multipoll_with_epoll_pollset_del_fd,
     multipoll_with_epoll_pollset_maybe_work,
-    epoll_kick,
     multipoll_with_epoll_pollset_finish_shutdown,
     multipoll_with_epoll_pollset_destroy};
 
@@ -182,8 +244,6 @@ static void epoll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds,
                                      size_t nfds) {
   size_t i;
   pollset_hdr *h = gpr_malloc(sizeof(pollset_hdr));
-  struct epoll_event ev;
-  int err;
 
   pollset->vtable = &multipoll_with_epoll_pollset;
   pollset->data.ptr = h;
@@ -196,16 +256,6 @@ static void epoll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds,
   for (i = 0; i < nfds; i++) {
     multipoll_with_epoll_pollset_add_fd(pollset, fds[i], 0);
   }
-
-  grpc_wakeup_fd_create(&h->wakeup_fd);
-  ev.events = EPOLLIN;
-  ev.data.ptr = 0;
-  err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD,
-                  GRPC_WAKEUP_FD_GET_READ_FD(&h->wakeup_fd), &ev);
-  if (err < 0) {
-    gpr_log(GPR_ERROR, "Wakeup fd epoll_ctl failed: %s", strerror(errno));
-    abort();
-  }
 }
 
 grpc_platform_become_multipoller_type grpc_platform_become_multipoller =
diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
index 0084e83953..b5b2d7534d 100644
--- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c
+++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
@@ -53,12 +53,6 @@ typedef struct {
   size_t fd_count;
   size_t fd_capacity;
   grpc_fd **fds;
-  /* fds being polled by the current poller: parallel arrays of pollfd, and
-     a grpc_fd_watcher */
-  size_t pfd_count;
-  size_t pfd_capacity;
-  grpc_fd_watcher *watchers;
-  struct pollfd *pfds;
   /* fds that have been removed from the pollset explicitly */
   size_t del_count;
   size_t del_capacity;
@@ -102,80 +96,60 @@ static void multipoll_with_poll_pollset_del_fd(grpc_pollset *pollset,
   }
 }
 
-static void end_polling(grpc_pollset *pollset) {
-  size_t i;
-  pollset_hdr *h;
-  h = pollset->data.ptr;
-  for (i = 1; i < h->pfd_count; i++) {
-    grpc_fd_end_poll(&h->watchers[i], h->pfds[i].revents & POLLIN,
-                     h->pfds[i].revents & POLLOUT);
-  }
-}
-
 static void multipoll_with_poll_pollset_maybe_work(
-    grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now,
-    int allow_synchronous_callback) {
+    grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec deadline,
+    gpr_timespec now, int allow_synchronous_callback) {
   int timeout;
   int r;
-  size_t i, np, nf, nd;
+  size_t i, j, pfd_count, fd_count;
   pollset_hdr *h;
-  grpc_kick_fd_info *kfd;
+  /* TODO(ctiller): inline some elements to avoid an allocation */
+  grpc_fd_watcher *watchers;
+  struct pollfd *pfds;
 
   h = pollset->data.ptr;
   timeout = grpc_poll_deadline_to_millis_timeout(deadline, now);
-  if (h->pfd_capacity < h->fd_count + 1) {
-    h->pfd_capacity = GPR_MAX(h->pfd_capacity * 3 / 2, h->fd_count + 1);
-    gpr_free(h->pfds);
-    gpr_free(h->watchers);
-    h->pfds = gpr_malloc(sizeof(struct pollfd) * h->pfd_capacity);
-    h->watchers = gpr_malloc(sizeof(grpc_fd_watcher) * h->pfd_capacity);
-  }
-  nf = 0;
-  np = 1;
-  kfd = grpc_pollset_kick_pre_poll(&pollset->kick_state);
-  if (kfd == NULL) {
-    /* Already kicked */
-    return;
-  }
-  h->pfds[0].fd = GRPC_POLLSET_KICK_GET_FD(kfd);
-  h->pfds[0].events = POLLIN;
-  h->pfds[0].revents = POLLOUT;
+  /* TODO(ctiller): perform just one malloc here if we exceed the inline case */
+  pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 1));
+  watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 1));
+  fd_count = 0;
+  pfd_count = 1;
+  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd);
+  pfds[0].events = POLLIN;
+  pfds[0].revents = POLLOUT;
   for (i = 0; i < h->fd_count; i++) {
     int remove = grpc_fd_is_orphaned(h->fds[i]);
-    for (nd = 0; nd < h->del_count; nd++) {
-      if (h->fds[i] == h->dels[nd]) remove = 1;
+    for (j = 0; !remove && j < h->del_count; j++) {
+      if (h->fds[i] == h->dels[j]) remove = 1;
     }
     if (remove) {
       GRPC_FD_UNREF(h->fds[i], "multipoller");
     } else {
-      h->fds[nf++] = h->fds[i];
-      h->watchers[np].fd = h->fds[i];
-      h->pfds[np].fd = h->fds[i]->fd;
-      h->pfds[np].revents = 0;
-      np++;
+      h->fds[fd_count++] = h->fds[i];
+      watchers[pfd_count].fd = h->fds[i];
+      pfds[pfd_count].fd = h->fds[i]->fd;
+      pfds[pfd_count].revents = 0;
+      pfd_count++;
     }
   }
-  h->pfd_count = np;
-  h->fd_count = nf;
-  for (nd = 0; nd < h->del_count; nd++) {
-    GRPC_FD_UNREF(h->dels[nd], "multipoller_del");
+  for (j = 0; j < h->del_count; j++) {
+    GRPC_FD_UNREF(h->dels[j], "multipoller_del");
   }
   h->del_count = 0;
-  if (h->pfd_count == 0) {
-    end_polling(pollset);
-    return;
-  }
-  pollset->counter++;
+  h->fd_count = fd_count;
   gpr_mu_unlock(&pollset->mu);
 
-  for (i = 1; i < np; i++) {
-    h->pfds[i].events = grpc_fd_begin_poll(h->watchers[i].fd, pollset, POLLIN,
-                                           POLLOUT, &h->watchers[i]);
+  for (i = 1; i < pfd_count; i++) {
+    pfds[i].events = grpc_fd_begin_poll(watchers[i].fd, pollset, POLLIN,
+                                        POLLOUT, &watchers[i]);
   }
 
-  r = poll(h->pfds, h->pfd_count, timeout);
+  r = poll(pfds, pfd_count, timeout);
 
-  end_polling(pollset);
+  for (i = 1; i < pfd_count; i++) {
+    grpc_fd_end_poll(&watchers[i], pfds[i].revents & POLLIN,
+                     pfds[i].revents & POLLOUT);
+  }
 
   if (r < 0) {
     if (errno != EINTR) {
@@ -184,35 +158,31 @@ static void multipoll_with_poll_pollset_maybe_work(
   } else if (r == 0) {
     /* do nothing */
   } else {
-    if (h->pfds[0].revents & POLLIN) {
-      grpc_pollset_kick_consume(&pollset->kick_state, kfd);
+    if (pfds[0].revents & POLLIN) {
+      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd);
     }
-    for (i = 1; i < np; i++) {
-      if (h->watchers[i].fd == NULL) {
+    for (i = 1; i < pfd_count; i++) {
+      if (watchers[i].fd == NULL) {
         continue;
       }
-      if (h->pfds[i].revents & (POLLIN | POLLHUP | POLLERR)) {
-        grpc_fd_become_readable(h->watchers[i].fd, allow_synchronous_callback);
+      if (pfds[i].revents & (POLLIN | POLLHUP | POLLERR)) {
+        grpc_fd_become_readable(watchers[i].fd, allow_synchronous_callback);
       }
-      if (h->pfds[i].revents & (POLLOUT | POLLHUP | POLLERR)) {
-        grpc_fd_become_writable(h->watchers[i].fd, allow_synchronous_callback);
+      if (pfds[i].revents & (POLLOUT | POLLHUP | POLLERR)) {
+        grpc_fd_become_writable(watchers[i].fd, allow_synchronous_callback);
       }
     }
   }
-  grpc_pollset_kick_post_poll(&pollset->kick_state, kfd);
 
-  gpr_mu_lock(&pollset->mu);
-  pollset->counter--;
-}
+  gpr_free(pfds);
+  gpr_free(watchers);
 
-static void multipoll_with_poll_pollset_kick(grpc_pollset *p) {
-  grpc_pollset_force_kick(p);
+  gpr_mu_lock(&pollset->mu);
 }
 
 static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) {
   size_t i;
   pollset_hdr *h = pollset->data.ptr;
-  GPR_ASSERT(pollset->counter == 0);
   for (i = 0; i < h->fd_count; i++) {
     GRPC_FD_UNREF(h->fds[i], "multipoller");
   }
@@ -226,8 +196,6 @@ static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) {
 static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
   pollset_hdr *h = pollset->data.ptr;
   multipoll_with_poll_pollset_finish_shutdown(pollset);
-  gpr_free(h->pfds);
-  gpr_free(h->watchers);
   gpr_free(h->fds);
   gpr_free(h->dels);
   gpr_free(h);
@@ -237,7 +205,6 @@ static const grpc_pollset_vtable multipoll_with_poll_pollset = {
     multipoll_with_poll_pollset_add_fd,
     multipoll_with_poll_pollset_del_fd,
     multipoll_with_poll_pollset_maybe_work,
-    multipoll_with_poll_pollset_kick,
     multipoll_with_poll_pollset_finish_shutdown,
     multipoll_with_poll_pollset_destroy};
 
@@ -250,10 +217,6 @@ void grpc_poll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds,
   h->fd_count = nfds;
   h->fd_capacity = nfds;
   h->fds = gpr_malloc(nfds * sizeof(grpc_fd *));
-  h->pfd_count = 0;
-  h->pfd_capacity = 0;
-  h->pfds = NULL;
-  h->watchers = NULL;
   h->del_count = 0;
   h->del_capacity = 0;
   h->dels = NULL;
diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c
index c8646af615..d3a9193af1 100644
--- a/src/core/iomgr/pollset_posix.c
+++ b/src/core/iomgr/pollset_posix.c
@@ -55,22 +55,60 @@
 #include <grpc/support/useful.h>
 
 GPR_TLS_DECL(g_current_thread_poller);
+GPR_TLS_DECL(g_current_thread_worker);
 
-void grpc_pollset_kick(grpc_pollset *p) {
-  if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p && p->counter) {
-    p->vtable->kick(p);
-  }
+static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev->next = worker->next;
+  worker->next->prev = worker->prev;
+}
+
+int grpc_pollset_has_workers(grpc_pollset *p) {
+  return p->root_worker.next != &p->root_worker;
 }
 
-void grpc_pollset_force_kick(grpc_pollset *p) {
-  if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) {
-    grpc_pollset_kick_kick(&p->kick_state);
+static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
+  if (grpc_pollset_has_workers(p)) {
+    grpc_pollset_worker *w = p->root_worker.next;
+    remove_worker(p, w);
+    return w;
+  } else {
+    return NULL;
   }
 }
 
-static void kick_using_pollset_kick(grpc_pollset *p) {
-  if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) {
-    grpc_pollset_kick_kick(&p->kick_state);
+static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->next = &p->root_worker;
+  worker->prev = worker->next->prev;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev = &p->root_worker;
+  worker->next = worker->prev->next;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
+  if (specific_worker != NULL) {
+    if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
+      for (specific_worker = p->root_worker.next;
+           specific_worker != &p->root_worker;
+           specific_worker = specific_worker->next) {
+        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd);
+      }
+      p->kicked_without_pollers = 1;
+    } else if (gpr_tls_get(&g_current_thread_worker) !=
+               (gpr_intptr)specific_worker) {
+      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd);
+    }
+  } else if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) {
+    specific_worker = pop_front_worker(p);
+    if (specific_worker != NULL) {
+      push_back_worker(p, specific_worker);
+      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd);
+    } else {
+      p->kicked_without_pollers = 1;
+    }
   }
 }
 
@@ -78,16 +116,12 @@ static void kick_using_pollset_kick(grpc_pollset *p) {
 
 void grpc_pollset_global_init(void) {
   gpr_tls_init(&g_current_thread_poller);
-
-  /* Initialize kick fd state */
-  grpc_pollset_kick_global_init();
+  grpc_wakeup_fd_global_init();
 }
 
 void grpc_pollset_global_shutdown(void) {
-  /* destroy the kick pipes */
-  grpc_pollset_kick_global_destroy();
-
   gpr_tls_destroy(&g_current_thread_poller);
+  grpc_wakeup_fd_global_destroy();
 }
 
 /* main interface */
@@ -96,7 +130,7 @@ static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null);
 
 void grpc_pollset_init(grpc_pollset *pollset) {
   gpr_mu_init(&pollset->mu);
-  grpc_pollset_kick_init(&pollset->kick_state);
+  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
   pollset->in_flight_cbs = 0;
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
@@ -134,27 +168,44 @@ static void finish_shutdown(grpc_pollset *pollset) {
   pollset->shutdown_done_cb(pollset->shutdown_done_arg);
 }
 
-int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) {
+int grpc_pollset_work(grpc_pollset *pollset, grpc_pollset_worker *worker,
+                      gpr_timespec deadline) {
   /* pollset->mu already held */
   gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+  int added_worker = 0;
   if (gpr_time_cmp(now, deadline) > 0) {
     return 0;
   }
+  /* this must happen before we (potentially) drop pollset->mu */
+  worker->next = worker->prev = NULL;
+  /* TODO(ctiller): pool these */
+  grpc_wakeup_fd_init(&worker->wakeup_fd);
   if (grpc_maybe_call_delayed_callbacks(&pollset->mu, 1)) {
-    return 1;
+    goto done;
   }
   if (grpc_alarm_check(&pollset->mu, now, &deadline)) {
-    return 1;
+    goto done;
   }
   if (pollset->shutting_down) {
-    return 1;
+    goto done;
+  }
+  if (!pollset->kicked_without_pollers) {
+    push_front_worker(pollset, worker);
+    added_worker = 1;
+    gpr_tls_set(&g_current_thread_poller, (gpr_intptr)pollset);
+    pollset->vtable->maybe_work(pollset, worker, deadline, now, 1);
+    gpr_tls_set(&g_current_thread_poller, 0);
+  } else {
+    pollset->kicked_without_pollers = 0;
+  }
+done:
+  grpc_wakeup_fd_destroy(&worker->wakeup_fd);
+  if (added_worker) {
+    remove_worker(pollset, worker);
   }
-  gpr_tls_set(&g_current_thread_poller, (gpr_intptr)pollset);
-  pollset->vtable->maybe_work(pollset, deadline, now, 1);
-  gpr_tls_set(&g_current_thread_poller, 0);
   if (pollset->shutting_down) {
-    if (pollset->counter > 0) {
-      grpc_pollset_kick(pollset);
+    if (grpc_pollset_has_workers(pollset)) {
+      grpc_pollset_kick(pollset, NULL);
     } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
       pollset->called_shutdown = 1;
       gpr_mu_unlock(&pollset->mu);
@@ -177,15 +228,13 @@ void grpc_pollset_shutdown(grpc_pollset *pollset,
   GPR_ASSERT(!pollset->shutting_down);
   pollset->shutting_down = 1;
   if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
-      pollset->counter == 0) {
+      !grpc_pollset_has_workers(pollset)) {
     pollset->called_shutdown = 1;
     call_shutdown = 1;
   }
   pollset->shutdown_done_cb = shutdown_done;
   pollset->shutdown_done_arg = shutdown_done_arg;
-  if (pollset->counter > 0) {
-    grpc_pollset_kick(pollset);
-  }
+  grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
   gpr_mu_unlock(&pollset->mu);
 
   if (call_shutdown) {
@@ -196,8 +245,8 @@ void grpc_pollset_shutdown(grpc_pollset *pollset,
 void grpc_pollset_destroy(grpc_pollset *pollset) {
   GPR_ASSERT(pollset->shutting_down);
   GPR_ASSERT(pollset->in_flight_cbs == 0);
+  GPR_ASSERT(!grpc_pollset_has_workers(pollset));
   pollset->vtable->destroy(pollset);
-  grpc_pollset_kick_destroy(&pollset->kick_state);
   gpr_mu_destroy(&pollset->mu);
 }
 
@@ -248,8 +297,8 @@ static void basic_do_promote(void *args, int success) {
 
   gpr_mu_lock(&pollset->mu);
   /* First we need to ensure that nobody is polling concurrently */
-  if (pollset->counter != 0) {
-    grpc_pollset_kick(pollset);
+  if (grpc_pollset_has_workers(pollset)) {
+    grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
     grpc_iomgr_add_callback(&up_args->promotion_closure);
     gpr_mu_unlock(&pollset->mu);
     return;
@@ -264,7 +313,8 @@ static void basic_do_promote(void *args, int success) {
   pollset->in_flight_cbs--;
   if (pollset->shutting_down) {
     /* We don't care about this pollset anymore. */
-    if (pollset->in_flight_cbs == 0 && pollset->counter == 0 && !pollset->called_shutdown) {
+    if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) {
+      GPR_ASSERT(!grpc_pollset_has_workers(pollset));
       pollset->called_shutdown = 1;
       do_shutdown_cb = 1;
     }
@@ -307,7 +357,7 @@ static void basic_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd,
   GPR_ASSERT(fd);
   if (fd == pollset->data.ptr) goto exit;
 
-  if (!pollset->counter) {
+  if (!grpc_pollset_has_workers(pollset)) {
     /* Fast path -- no in flight cbs */
     /* TODO(klempner): Comment this out and fix any test failures or establish
      * they are due to timing issues */
@@ -343,7 +393,7 @@ static void basic_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd,
   up_args->promotion_closure.cb_arg = up_args;
   grpc_iomgr_add_callback(&up_args->promotion_closure);
 
-  grpc_pollset_kick(pollset);
+  grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
 
 exit:
   if (and_unlock_pollset) {
@@ -365,12 +415,12 @@ static void basic_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd,
 }
 
 static void basic_pollset_maybe_work(grpc_pollset *pollset,
+                                     grpc_pollset_worker *worker,
                                      gpr_timespec deadline, gpr_timespec now,
                                      int allow_synchronous_callback) {
   struct pollfd pfd[2];
   grpc_fd *fd;
   grpc_fd_watcher fd_watcher;
-  grpc_kick_fd_info *kfd;
   int timeout;
   int r;
   int nfds;
@@ -387,16 +437,10 @@ static void basic_pollset_maybe_work(grpc_pollset *pollset,
     fd = pollset->data.ptr = NULL;
   }
   timeout = grpc_poll_deadline_to_millis_timeout(deadline, now);
-  kfd = grpc_pollset_kick_pre_poll(&pollset->kick_state);
-  if (kfd == NULL) {
-    /* Already kicked */
-    return;
-  }
-  pfd[0].fd = GRPC_POLLSET_KICK_GET_FD(kfd);
+  pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd);
   pfd[0].events = POLLIN;
   pfd[0].revents = 0;
   nfds = 1;
-  pollset->counter++;
   if (fd) {
     pfd[1].fd = fd->fd;
     pfd[1].revents = 0;
@@ -428,7 +472,7 @@ static void basic_pollset_maybe_work(grpc_pollset *pollset,
     /* do nothing */
   } else {
     if (pfd[0].revents & POLLIN) {
-      grpc_pollset_kick_consume(&pollset->kick_state, kfd);
+      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd);
     }
     if (nfds > 1) {
       if (pfd[1].revents & (POLLIN | POLLHUP | POLLERR)) {
@@ -440,14 +484,10 @@ static void basic_pollset_maybe_work(grpc_pollset *pollset,
     }
   }
 
-  grpc_pollset_kick_post_poll(&pollset->kick_state, kfd);
-
   gpr_mu_lock(&pollset->mu);
-  pollset->counter--;
 }
 
 static void basic_pollset_destroy(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->counter == 0);
   if (pollset->data.ptr != NULL) {
     GRPC_FD_UNREF(pollset->data.ptr, "basicpoll");
     pollset->data.ptr = NULL;
@@ -455,14 +495,13 @@ static void basic_pollset_destroy(grpc_pollset *pollset) {
 }
 
 static const grpc_pollset_vtable basic_pollset = {
-    basic_pollset_add_fd,    basic_pollset_del_fd,  basic_pollset_maybe_work,
-    kick_using_pollset_kick, basic_pollset_destroy, basic_pollset_destroy};
+    basic_pollset_add_fd, basic_pollset_del_fd, basic_pollset_maybe_work,
+    basic_pollset_destroy, basic_pollset_destroy};
 
 static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) {
   pollset->vtable = &basic_pollset;
-  pollset->counter = 0;
   pollset->data.ptr = fd_or_null;
-  if (fd_or_null) {
+  if (fd_or_null != NULL) {
     GRPC_FD_REF(fd_or_null, "basicpoll");
   }
 }
diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h
index 37de1276d1..1c1b736193 100644
--- a/src/core/iomgr/pollset_posix.h
+++ b/src/core/iomgr/pollset_posix.h
@@ -35,8 +35,7 @@
 #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_POSIX_H
 
 #include <grpc/support/sync.h>
-
-#include "src/core/iomgr/pollset_kick_posix.h"
+#include "src/core/iomgr/wakeup_fd_posix.h"
 
 typedef struct grpc_pollset_vtable grpc_pollset_vtable;
 
@@ -45,6 +44,12 @@ typedef struct grpc_pollset_vtable grpc_pollset_vtable;
    use the struct tag */
 struct grpc_fd;
 
+typedef struct grpc_pollset_worker {
+  grpc_wakeup_fd wakeup_fd;
+  struct grpc_pollset_worker *next;
+  struct grpc_pollset_worker *prev;
+} grpc_pollset_worker;
+
 typedef struct grpc_pollset {
   /* pollsets under posix can mutate representation as fds are added and
      removed.
@@ -52,11 +57,11 @@ typedef struct grpc_pollset {
      few fds, and an epoll() based implementation for many fds */
   const grpc_pollset_vtable *vtable;
   gpr_mu mu;
-  grpc_pollset_kick_state kick_state;
-  int counter;
+  grpc_pollset_worker root_worker;
   int in_flight_cbs;
   int shutting_down;
   int called_shutdown;
+  int kicked_without_pollers;
   void (*shutdown_done_cb)(void *arg);
   void *shutdown_done_arg;
   union {
@@ -70,9 +75,9 @@ struct grpc_pollset_vtable {
                  int and_unlock_pollset);
   void (*del_fd)(grpc_pollset *pollset, struct grpc_fd *fd,
                  int and_unlock_pollset);
-  void (*maybe_work)(grpc_pollset *pollset, gpr_timespec deadline,
-                     gpr_timespec now, int allow_synchronous_callback);
-  void (*kick)(grpc_pollset *pollset);
+  void (*maybe_work)(grpc_pollset *pollset, grpc_pollset_worker *worker,
+                     gpr_timespec deadline, gpr_timespec now,
+                     int allow_synchronous_callback);
   void (*finish_shutdown)(grpc_pollset *pollset);
   void (*destroy)(grpc_pollset *pollset);
 };
@@ -85,22 +90,16 @@ void grpc_pollset_add_fd(grpc_pollset *pollset, struct grpc_fd *fd);
    poll after an fd is orphaned) */
 void grpc_pollset_del_fd(grpc_pollset *pollset, struct grpc_fd *fd);
 
-/* Force any current pollers to break polling: it's the callers responsibility
-   to ensure that the pollset indeed needs to be kicked - no verification that
-   the pollset is actually performing polling work is done. At worst this will
-   result in spurious wakeups if performed at the wrong moment.
-   Does not touch pollset->mu. */
-void grpc_pollset_force_kick(grpc_pollset *pollset);
 /* Returns the fd to listen on for kicks */
 int grpc_kick_read_fd(grpc_pollset *p);
 /* Call after polling has been kicked to leave the kicked state */
 void grpc_kick_drain(grpc_pollset *p);
 
 /* Convert a timespec to milliseconds:
-   - very small or negative poll times are clamped to zero to do a 
+   - very small or negative poll times are clamped to zero to do a
      non-blocking poll (which becomes spin polling)
    - other small values are rounded up to one millisecond
-   - longer than a millisecond polls are rounded up to the next nearest 
+   - longer than a millisecond polls are rounded up to the next nearest
      millisecond to avoid spinning
    - infinite timeouts are converted to -1 */
 int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now);
@@ -114,4 +113,8 @@ extern grpc_platform_become_multipoller_type grpc_platform_become_multipoller;
 void grpc_poll_become_multipoller(grpc_pollset *pollset, struct grpc_fd **fds,
                                   size_t fd_count);
 
+/* Return 1 if the pollset has active threads in grpc_pollset_work (pollset must
+ * be locked) */
+int grpc_pollset_has_workers(grpc_pollset *pollset);
+
 #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_POSIX_H */
diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c
index 5ff7df1dcd..2076ac70ef 100644
--- a/src/core/iomgr/pollset_set_posix.c
+++ b/src/core/iomgr/pollset_set_posix.c
@@ -60,7 +60,7 @@ void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) {
 
 void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set,
                                   grpc_pollset *pollset) {
-  size_t i;
+  size_t i, j;
   gpr_mu_lock(&pollset_set->mu);
   if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
     pollset_set->pollset_capacity =
@@ -70,9 +70,15 @@ void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set,
                                                sizeof(*pollset_set->pollsets));
   }
   pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    grpc_pollset_add_fd(pollset, pollset_set->fds[i]);
+  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
+    if (grpc_fd_is_orphaned(pollset_set->fds[i])) {
+      GRPC_FD_UNREF(pollset_set->fds[i], "pollset");
+    } else {
+      grpc_pollset_add_fd(pollset, pollset_set->fds[i]);
+      pollset_set->fds[j++] = pollset_set->fds[i];
+    }
   }
+  pollset_set->fd_count = j;
   gpr_mu_unlock(&pollset_set->mu);
 }
 
diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c
index a9c4739c7c..22dc5891c3 100644
--- a/src/core/iomgr/pollset_windows.c
+++ b/src/core/iomgr/pollset_windows.c
@@ -42,6 +42,38 @@
 #include "src/core/iomgr/pollset.h"
 #include "src/core/iomgr/pollset_windows.h"
 
+static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev->next = worker->next;
+  worker->next->prev = worker->prev;
+}
+
+static int has_workers(grpc_pollset *p) {
+  return p->root_worker.next != &p->root_worker;
+}
+
+static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
+  if (has_workers(p)) {
+    grpc_pollset_worker *w = p->root_worker.next;
+    remove_worker(p, w);
+    return w;
+  }
+  else {
+    return NULL;
+  }
+}
+
+static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->next = &p->root_worker;
+  worker->prev = worker->next->prev;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev = &p->root_worker;
+  worker->next = worker->prev->next;
+  worker->prev->next = worker->next->prev = worker;
+}
+
 /* There isn't really any such thing as a pollset under Windows, due to the
    nature of the IO completion ports. We're still going to provide a minimal
    set of features for the sake of the rest of grpc. But grpc_pollset_work
@@ -50,7 +82,8 @@
 void grpc_pollset_init(grpc_pollset *pollset) {
   memset(pollset, 0, sizeof(*pollset));
   gpr_mu_init(&pollset->mu);
-  gpr_cv_init(&pollset->cv);
+  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
+  pollset->kicked_without_pollers = 0;
 }
 
 void grpc_pollset_shutdown(grpc_pollset *pollset,
@@ -58,34 +91,66 @@ void grpc_pollset_shutdown(grpc_pollset *pollset,
                            void *shutdown_done_arg) {
   gpr_mu_lock(&pollset->mu);
   pollset->shutting_down = 1;
-  gpr_cv_broadcast(&pollset->cv);
+  grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
   gpr_mu_unlock(&pollset->mu);
   shutdown_done(shutdown_done_arg);
 }
 
 void grpc_pollset_destroy(grpc_pollset *pollset) {
   gpr_mu_destroy(&pollset->mu);
-  gpr_cv_destroy(&pollset->cv);
 }
 
-int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) {
+int grpc_pollset_work(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec deadline) {
   gpr_timespec now;
+  int added_worker = 0;
   now = gpr_now(GPR_CLOCK_MONOTONIC);
   if (gpr_time_cmp(now, deadline) > 0) {
     return 0 /* GPR_FALSE */;
   }
+  worker->next = worker->prev = NULL;
+  gpr_cv_init(&worker->cv);
   if (grpc_maybe_call_delayed_callbacks(&pollset->mu, 1 /* GPR_TRUE */)) {
-    return 1 /* GPR_TRUE */;
+    goto done;
   }
   if (grpc_alarm_check(&pollset->mu, now, &deadline)) {
-    return 1 /* GPR_TRUE */;
+    goto done;
   }
-  if (!pollset->shutting_down) {
-    gpr_cv_wait(&pollset->cv, &pollset->mu, deadline);
+  if (!pollset->kicked_without_pollers && !pollset->shutting_down) {
+    push_front_worker(pollset, worker);
+    added_worker = 1;
+    gpr_cv_wait(&worker->cv, &pollset->mu, deadline);
+  } else {
+    pollset->kicked_without_pollers = 0;
+  }
+done:
+  gpr_cv_destroy(&worker->cv);
+  if (added_worker) {
+    remove_worker(pollset, worker);
   }
   return 1 /* GPR_TRUE */;
 }
 
-void grpc_pollset_kick(grpc_pollset *p) { gpr_cv_signal(&p->cv); }
+void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
+  if (specific_worker != NULL) {
+    if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
+      for (specific_worker = p->root_worker.next;
+        specific_worker != &p->root_worker;
+        specific_worker = specific_worker->next) {
+        gpr_cv_signal(&specific_worker->cv);
+      }
+      p->kicked_without_pollers = 1;
+    } else {
+      gpr_cv_signal(&specific_worker->cv);
+    }
+  } else {
+    specific_worker = pop_front_worker(p);
+    if (specific_worker != NULL) {
+      push_back_worker(p, specific_worker);
+      gpr_cv_signal(&specific_worker->cv);
+    } else {
+      p->kicked_without_pollers = 1;
+    }
+  }
+}
 
 #endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h
index c9b8d3f374..4efa5a1717 100644
--- a/src/core/iomgr/pollset_windows.h
+++ b/src/core/iomgr/pollset_windows.h
@@ -40,12 +40,20 @@
 
 /* There isn't really any such thing as a pollset under Windows, due to the
    nature of the IO completion ports. A Windows "pollset" is merely a mutex
-   and a condition variable, used to synchronize with the IOCP. */
+   used to synchronize with the IOCP, and workers are condition variables
+   used to block threads until work is ready. */
+
+typedef struct grpc_pollset_worker {
+  gpr_cv cv;
+  struct grpc_pollset_worker *next;
+  struct grpc_pollset_worker *prev;
+} grpc_pollset_worker;
 
 typedef struct grpc_pollset {
   gpr_mu mu;
-  gpr_cv cv;
   int shutting_down;
+  int kicked_without_pollers;
+  grpc_pollset_worker root_worker;
 } grpc_pollset;
 
 #define GRPC_POLLSET_MU(pollset) (&(pollset)->mu)
diff --git a/src/core/iomgr/sockaddr_utils.c b/src/core/iomgr/sockaddr_utils.c
index e91b94f8c8..65ec1f94ac 100644
--- a/src/core/iomgr/sockaddr_utils.c
+++ b/src/core/iomgr/sockaddr_utils.c
@@ -36,12 +36,18 @@
 #include <errno.h>
 #include <string.h>
 
-#include "src/core/support/string.h"
+#ifdef GPR_POSIX_SOCKET
+#include <sys/un.h>
+#endif
+
+#include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/support/string.h"
+
 static const gpr_uint8 kV4MappedPrefix[] = {0, 0, 0, 0, 0,    0,
                                             0, 0, 0, 0, 0xff, 0xff};
 
@@ -161,6 +167,36 @@ int grpc_sockaddr_to_string(char **out, const struct sockaddr *addr,
   return ret;
 }
 
+char *grpc_sockaddr_to_uri(const struct sockaddr *addr) {
+  char *temp;
+  char *result;
+  struct sockaddr_in addr_normalized;
+
+  if (grpc_sockaddr_is_v4mapped(addr, &addr_normalized)) {
+    addr = (const struct sockaddr *)&addr_normalized;
+  }
+
+  switch (addr->sa_family) {
+    case AF_INET:
+      grpc_sockaddr_to_string(&temp, addr, 0);
+      gpr_asprintf(&result, "ipv4:%s", temp);
+      gpr_free(temp);
+      return result;
+    case AF_INET6:
+      grpc_sockaddr_to_string(&temp, addr, 0);
+      gpr_asprintf(&result, "ipv6:%s", temp);
+      gpr_free(temp);
+      return result;
+#ifdef GPR_POSIX_SOCKET
+    case AF_UNIX:
+      gpr_asprintf(&result, "unix:%s", ((struct sockaddr_un *)addr)->sun_path);
+      return result;
+#endif
+  }
+
+  return NULL;
+}
+
 int grpc_sockaddr_get_port(const struct sockaddr *addr) {
   switch (addr->sa_family) {
     case AF_INET:
diff --git a/src/core/iomgr/sockaddr_utils.h b/src/core/iomgr/sockaddr_utils.h
index bdfb83479b..99f1ed54da 100644
--- a/src/core/iomgr/sockaddr_utils.h
+++ b/src/core/iomgr/sockaddr_utils.h
@@ -84,4 +84,6 @@ int grpc_sockaddr_set_port(const struct sockaddr *addr, int port);
 int grpc_sockaddr_to_string(char **out, const struct sockaddr *addr,
                             int normalize);
 
+char *grpc_sockaddr_to_uri(const struct sockaddr *addr);
+
 #endif  /* GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_UTILS_H */
diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/iomgr/tcp_client_posix.c
index 41d8b169e0..9572ce5980 100644
--- a/src/core/iomgr/tcp_client_posix.c
+++ b/src/core/iomgr/tcp_client_posix.c
@@ -64,6 +64,7 @@ typedef struct {
   int refs;
   grpc_iomgr_closure write_closure;
   grpc_pollset_set *interested_parties;
+  char *addr_str;
 } async_connect;
 
 static int prepare_socket(const struct sockaddr *addr, int fd) {
@@ -88,17 +89,18 @@ error:
   return 0;
 }
 
-static void on_alarm(void *acp, int success) {
+static void tc_on_alarm(void *acp, int success) {
   int done;
   async_connect *ac = acp;
   gpr_mu_lock(&ac->mu);
-  if (ac->fd != NULL && success) {
+  if (ac->fd != NULL) {
     grpc_fd_shutdown(ac->fd);
   }
   done = (--ac->refs == 0);
   gpr_mu_unlock(&ac->mu);
   if (done) {
     gpr_mu_destroy(&ac->mu);
+    gpr_free(ac->addr_str);
     gpr_free(ac);
   }
 }
@@ -108,11 +110,17 @@ static void on_writable(void *acp, int success) {
   int so_error = 0;
   socklen_t so_error_size;
   int err;
-  int fd = ac->fd->fd;
   int done;
   grpc_endpoint *ep = NULL;
   void (*cb)(void *arg, grpc_endpoint *tcp) = ac->cb;
   void *cb_arg = ac->cb_arg;
+  grpc_fd *fd;
+
+  gpr_mu_lock(&ac->mu);
+  GPR_ASSERT(ac->fd);
+  fd = ac->fd;
+  ac->fd = NULL;
+  gpr_mu_unlock(&ac->mu);
 
   grpc_alarm_cancel(&ac->alarm);
 
@@ -120,7 +128,7 @@ static void on_writable(void *acp, int success) {
   if (success) {
     do {
       so_error_size = sizeof(so_error);
-      err = getsockopt(fd, SOL_SOCKET, SO_ERROR, &so_error, &so_error_size);
+      err = getsockopt(fd->fd, SOL_SOCKET, SO_ERROR, &so_error, &so_error_size);
     } while (err < 0 && errno == EINTR);
     if (err < 0) {
       gpr_log(GPR_ERROR, "getsockopt(ERROR): %s", strerror(errno));
@@ -143,7 +151,7 @@ static void on_writable(void *acp, int success) {
            don't do that! */
         gpr_log(GPR_ERROR, "kernel out of buffers");
         gpr_mu_unlock(&ac->mu);
-        grpc_fd_notify_on_write(ac->fd, &ac->write_closure);
+        grpc_fd_notify_on_write(fd, &ac->write_closure);
         return;
       } else {
         switch (so_error) {
@@ -157,8 +165,9 @@ static void on_writable(void *acp, int success) {
         goto finish;
       }
     } else {
-      grpc_pollset_set_del_fd(ac->interested_parties, ac->fd);
-      ep = grpc_tcp_create(ac->fd, GRPC_TCP_DEFAULT_READ_SLICE_SIZE);
+      grpc_pollset_set_del_fd(ac->interested_parties, fd);
+      ep = grpc_tcp_create(fd, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, ac->addr_str);
+      fd = NULL;
       goto finish;
     }
   } else {
@@ -169,16 +178,16 @@ static void on_writable(void *acp, int success) {
   abort();
 
 finish:
-  if (ep == NULL) {
-    grpc_pollset_set_del_fd(ac->interested_parties, ac->fd);
-    grpc_fd_orphan(ac->fd, NULL, "tcp_client_orphan");
-  } else {
-    ac->fd = NULL;
+  if (fd != NULL) {
+    grpc_pollset_set_del_fd(ac->interested_parties, fd);
+    grpc_fd_orphan(fd, NULL, "tcp_client_orphan");
+    fd = NULL;
   }
   done = (--ac->refs == 0);
   gpr_mu_unlock(&ac->mu);
   if (done) {
     gpr_mu_destroy(&ac->mu);
+    gpr_free(ac->addr_str);
     gpr_free(ac);
   }
   cb(cb_arg, ep);
@@ -223,13 +232,13 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep),
     err = connect(fd, addr, addr_len);
   } while (err < 0 && errno == EINTR);
 
-  grpc_sockaddr_to_string(&addr_str, addr, 1);
+  addr_str = grpc_sockaddr_to_uri(addr);
   gpr_asprintf(&name, "tcp-client:%s", addr_str);
 
   fdobj = grpc_fd_create(fd, name);
 
   if (err >= 0) {
-    cb(arg, grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE));
+    cb(arg, grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str));
     goto done;
   }
 
@@ -247,6 +256,8 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep),
   ac->cb_arg = arg;
   ac->fd = fdobj;
   ac->interested_parties = interested_parties;
+  ac->addr_str = addr_str;
+  addr_str = NULL;
   gpr_mu_init(&ac->mu);
   ac->refs = 2;
   ac->write_closure.cb = on_writable;
@@ -254,7 +265,7 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep),
 
   gpr_mu_lock(&ac->mu);
   grpc_alarm_init(&ac->alarm, gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), 
-                  on_alarm, ac, gpr_now(GPR_CLOCK_MONOTONIC));
+                  tc_on_alarm, ac, gpr_now(GPR_CLOCK_MONOTONIC));
   grpc_fd_notify_on_write(ac->fd, &ac->write_closure);
   gpr_mu_unlock(&ac->mu);
 
diff --git a/src/core/iomgr/tcp_client_windows.c b/src/core/iomgr/tcp_client_windows.c
index 39fd43130b..79a58fe2af 100644
--- a/src/core/iomgr/tcp_client_windows.c
+++ b/src/core/iomgr/tcp_client_windows.c
@@ -58,6 +58,7 @@ typedef struct {
   grpc_winsocket *socket;
   gpr_timespec deadline;
   grpc_alarm alarm;
+  char *addr_name;
   int refs;
   int aborted;
 } async_connect;
@@ -67,6 +68,7 @@ static void async_connect_cleanup(async_connect *ac) {
   gpr_mu_unlock(&ac->mu);
   if (done) {
     gpr_mu_destroy(&ac->mu);
+    gpr_free(ac->addr_name);
     gpr_free(ac);
   }
 }
@@ -107,7 +109,7 @@ static void on_connect(void *acp, int from_iocp) {
       gpr_log(GPR_ERROR, "on_connect error: %s", utf8_message);
       gpr_free(utf8_message);
     } else if (!aborted) {
-      ep = grpc_tcp_create(ac->socket);
+      ep = grpc_tcp_create(ac->socket, ac->addr_name);
     }
   } else {
     gpr_log(GPR_ERROR, "on_connect is shutting down");
@@ -213,6 +215,7 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp),
   ac->socket = socket;
   gpr_mu_init(&ac->mu);
   ac->refs = 2;
+  ac->addr_name = grpc_sockaddr_to_uri(addr);
   ac->aborted = 0;
 
   grpc_alarm_init(&ac->alarm, deadline, on_alarm, ac,
diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c
index b6d6efc9fb..24fee0596f 100644
--- a/src/core/iomgr/tcp_posix.c
+++ b/src/core/iomgr/tcp_posix.c
@@ -44,15 +44,17 @@
 #include <sys/socket.h>
 #include <unistd.h>
 
-#include "src/core/support/string.h"
-#include "src/core/debug/trace.h"
-#include "src/core/profiling/timers.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 
+#include "src/core/support/string.h"
+#include "src/core/debug/trace.h"
+#include "src/core/profiling/timers.h"
+
 #ifdef GPR_HAVE_MSG_NOSIGNAL
 #define SENDMSG_FLAGS MSG_NOSIGNAL
 #else
@@ -282,6 +284,8 @@ typedef struct {
   grpc_iomgr_closure write_closure;
 
   grpc_iomgr_closure handle_read_closure;
+
+  char *peer_string;
 } grpc_tcp;
 
 static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success);
@@ -296,6 +300,7 @@ static void grpc_tcp_unref(grpc_tcp *tcp) {
   int refcount_zero = gpr_unref(&tcp->refcount);
   if (refcount_zero) {
     grpc_fd_orphan(tcp->em_fd, NULL, "tcp_unref_orphan");
+    gpr_free(tcp->peer_string);
     gpr_free(tcp);
   }
 }
@@ -314,7 +319,7 @@ static void call_read_cb(grpc_tcp *tcp, gpr_slice *slices, size_t nslices,
     gpr_log(GPR_DEBUG, "read: status=%d", status);
     for (i = 0; i < nslices; i++) {
       char *dump = gpr_dump_slice(slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
-      gpr_log(GPR_DEBUG, "READ: %s", dump);
+      gpr_log(GPR_DEBUG, "READ %p: %s", tcp, dump);
       gpr_free(dump);
     }
   }
@@ -443,7 +448,7 @@ static void grpc_tcp_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb,
     grpc_fd_notify_on_read(tcp->em_fd, &tcp->read_closure);
   } else {
     tcp->handle_read_closure.cb_arg = tcp;
-    grpc_iomgr_add_callback(&tcp->handle_read_closure);
+    grpc_iomgr_add_delayed_callback(&tcp->handle_read_closure, 1);
   }
 }
 
@@ -567,13 +572,27 @@ static void grpc_tcp_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) {
   grpc_pollset_add_fd(pollset, tcp->em_fd);
 }
 
+static void grpc_tcp_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pollset_set) {
+  grpc_tcp *tcp = (grpc_tcp *)ep;
+  grpc_pollset_set_add_fd(pollset_set, tcp->em_fd);
+}
+
+static char *grpc_tcp_get_peer(grpc_endpoint *ep) {
+  grpc_tcp *tcp = (grpc_tcp *)ep;
+  return gpr_strdup(tcp->peer_string);
+}
+
 static const grpc_endpoint_vtable vtable = {
-    grpc_tcp_notify_on_read, grpc_tcp_write, grpc_tcp_add_to_pollset,
-    grpc_tcp_shutdown, grpc_tcp_destroy};
+    grpc_tcp_notify_on_read, grpc_tcp_write,
+    grpc_tcp_add_to_pollset, grpc_tcp_add_to_pollset_set,
+    grpc_tcp_shutdown,       grpc_tcp_destroy,
+    grpc_tcp_get_peer};
 
-grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size) {
+grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
+                               const char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp));
   tcp->base.vtable = &vtable;
+  tcp->peer_string = gpr_strdup(peer_string);
   tcp->fd = em_fd->fd;
   tcp->read_cb = NULL;
   tcp->write_cb = NULL;
diff --git a/src/core/iomgr/tcp_posix.h b/src/core/iomgr/tcp_posix.h
index 44279d5a26..d752feaeea 100644
--- a/src/core/iomgr/tcp_posix.h
+++ b/src/core/iomgr/tcp_posix.h
@@ -53,6 +53,7 @@ extern int grpc_tcp_trace;
 
 /* Create a tcp endpoint given a file desciptor and a read slice size.
    Takes ownership of fd. */
-grpc_endpoint *grpc_tcp_create(grpc_fd *fd, size_t read_slice_size);
+grpc_endpoint *grpc_tcp_create(grpc_fd *fd, size_t read_slice_size,
+                               const char *peer_string);
 
 #endif  /* GRPC_INTERNAL_CORE_IOMGR_TCP_POSIX_H */
diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c
index 5854031c9b..6399aaadb9 100644
--- a/src/core/iomgr/tcp_server_posix.c
+++ b/src/core/iomgr/tcp_server_posix.c
@@ -142,6 +142,7 @@ grpc_tcp_server *grpc_tcp_server_create(void) {
 
 static void finish_shutdown(grpc_tcp_server *s) {
   s->shutdown_complete(s->shutdown_complete_arg);
+  s->shutdown_complete = NULL;
 
   gpr_mu_destroy(&s->mu);
 
@@ -157,6 +158,7 @@ static void destroyed_port(void *server, int success) {
     gpr_mu_unlock(&s->mu);
     finish_shutdown(s);
   } else {
+    GPR_ASSERT(s->destroyed_ports < s->nports);
     gpr_mu_unlock(&s->mu);
   }
 }
@@ -332,7 +334,7 @@ static void on_read(void *arg, int success) {
 
     grpc_set_socket_no_sigpipe_if_possible(fd);
 
-    grpc_sockaddr_to_string(&addr_str, (struct sockaddr *)&addr, 1);
+    addr_str = grpc_sockaddr_to_uri((struct sockaddr *)&addr);
     gpr_asprintf(&name, "tcp-server-connection:%s", addr_str);
 
     fdobj = grpc_fd_create(fd, name);
@@ -342,8 +344,9 @@ static void on_read(void *arg, int success) {
     for (i = 0; i < sp->server->pollset_count; i++) {
       grpc_pollset_add_fd(sp->server->pollsets[i], fdobj);
     }
-    sp->server->cb(sp->server->cb_arg,
-                   grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE));
+    sp->server->cb(
+        sp->server->cb_arg,
+        grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str));
 
     gpr_free(name);
     gpr_free(addr_str);
diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c
index 187009b2c8..0adbe9507c 100644
--- a/src/core/iomgr/tcp_server_windows.c
+++ b/src/core/iomgr/tcp_server_windows.c
@@ -79,6 +79,8 @@ struct grpc_tcp_server {
 
   /* active port count: how many ports are actually still listening */
   int active_ports;
+  /* number of iomgr callbacks that have been explicitly scheduled during shutdown */
+  int iomgr_callbacks_pending;
 
   /* all listening ports */
   server_port *ports;
@@ -93,6 +95,7 @@ grpc_tcp_server *grpc_tcp_server_create(void) {
   gpr_mu_init(&s->mu);
   gpr_cv_init(&s->cv);
   s->active_ports = 0;
+  s->iomgr_callbacks_pending = 0;
   s->cb = NULL;
   s->cb_arg = NULL;
   s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP);
@@ -112,10 +115,10 @@ void grpc_tcp_server_destroy(grpc_tcp_server *s,
   for (i = 0; i < s->nports; i++) {
     server_port *sp = &s->ports[i];
     sp->shutting_down = 1;
-    grpc_winsocket_shutdown(sp->socket);
+    s->iomgr_callbacks_pending += grpc_winsocket_shutdown(sp->socket);
   }
   /* This happens asynchronously. Wait while that happens. */
-  while (s->active_ports) {
+  while (s->active_ports || s->iomgr_callbacks_pending) {
     gpr_cv_wait(&s->cv, &s->mu, gpr_inf_future(GPR_CLOCK_REALTIME));
   }
   gpr_mu_unlock(&s->mu);
@@ -183,6 +186,17 @@ error:
   return -1;
 }
 
+static void decrement_active_ports_and_notify(server_port *sp) {
+  sp->shutting_down = 0;
+  sp->socket->read_info.outstanding = 0;
+  gpr_mu_lock(&sp->server->mu);
+  GPR_ASSERT(sp->server->active_ports > 0);
+  if (0 == --sp->server->active_ports) {
+    gpr_cv_broadcast(&sp->server->cv);
+  }
+  gpr_mu_unlock(&sp->server->mu);
+}
+
 /* start_accept will reference that for the IOCP notification request. */
 static void on_accept(void *arg, int from_iocp);
 
@@ -231,6 +245,15 @@ static void start_accept(server_port *port) {
   return;
 
 failure:
+  if (port->shutting_down) {
+    /* We are abandoning the listener port, take that into account to prevent
+       occasional hangs on shutdown. The hang happens when sp->shutting_down
+       change is not seen by on_accept and we proceed to trying new accept,
+       but we fail there because the listening port has been closed in the
+       meantime. */
+    decrement_active_ports_and_notify(port);
+    return;
+  }
   utf8_message = gpr_format_message(WSAGetLastError());
   gpr_log(GPR_ERROR, message, utf8_message);
   gpr_free(utf8_message);
@@ -243,14 +266,27 @@ static void on_accept(void *arg, int from_iocp) {
   SOCKET sock = sp->new_socket;
   grpc_winsocket_callback_info *info = &sp->socket->read_info;
   grpc_endpoint *ep = NULL;
+  struct sockaddr_storage peer_name;
+  char *peer_name_string;
+  char *fd_name;
+  int peer_name_len = sizeof(peer_name);
   DWORD transfered_bytes;
   DWORD flags;
   BOOL wsa_success;
+  int err;
 
   /* The general mechanism for shutting down is to queue abortion calls. While
      this is necessary in the read/write case, it's useless for the accept
-     case. Let's do nothing. */
-  if (!from_iocp) return;
+     case. We only need to adjust the pending callback count */
+  if (!from_iocp) {
+    gpr_mu_lock(&sp->server->mu);
+    GPR_ASSERT(sp->server->iomgr_callbacks_pending > 0);
+    if (0 == --sp->server->iomgr_callbacks_pending) {
+      gpr_cv_broadcast(&sp->server->cv);
+    }
+    gpr_mu_unlock(&sp->server->mu);
+    return;
+  }
 
   /* The IOCP notified us of a completed operation. Let's grab the results,
       and act accordingly. */
@@ -259,15 +295,9 @@ static void on_accept(void *arg, int from_iocp) {
                                             &transfered_bytes, FALSE, &flags);
   if (!wsa_success) {
     if (sp->shutting_down) {
-      /* During the shutdown case, we ARE expecting an error. So that's swell,
+      /* During the shutdown case, we ARE expecting an error. So that's well,
          and we can wake up the shutdown thread. */
-      sp->shutting_down = 0;
-      sp->socket->read_info.outstanding = 0;
-      gpr_mu_lock(&sp->server->mu);
-      if (0 == --sp->server->active_ports) {
-        gpr_cv_broadcast(&sp->server->cv);
-      }
-      gpr_mu_unlock(&sp->server->mu);
+      decrement_active_ports_and_notify(sp);
       return;
     } else {
       char *utf8_message = gpr_format_message(WSAGetLastError());
@@ -277,8 +307,28 @@ static void on_accept(void *arg, int from_iocp) {
     }
   } else {
     if (!sp->shutting_down) {
-      /* TODO(ctiller): add sockaddr address to label */
-      ep = grpc_tcp_create(grpc_winsocket_create(sock, "server"));
+      peer_name_string = NULL;
+      err = setsockopt(sock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT,
+                       (char *)&sp->socket->socket,
+                       sizeof(sp->socket->socket));
+      if (err) {
+        char *utf8_message = gpr_format_message(WSAGetLastError());
+        gpr_log(GPR_ERROR, "setsockopt error: %s", utf8_message);
+        gpr_free(utf8_message);
+      }
+      err = getpeername(sock, (struct sockaddr*)&peer_name, &peer_name_len);
+      if (!err) {
+        peer_name_string = grpc_sockaddr_to_uri((struct sockaddr*)&peer_name);
+      } else {
+        char *utf8_message = gpr_format_message(WSAGetLastError());
+        gpr_log(GPR_ERROR, "getpeername error: %s", utf8_message);
+        gpr_free(utf8_message);
+      }
+      gpr_asprintf(&fd_name, "tcp_server:%s", peer_name_string);
+      ep = grpc_tcp_create(grpc_winsocket_create(sock, fd_name),
+                           peer_name_string);
+      gpr_free(fd_name);
+      gpr_free(peer_name_string);
     }
   }
 
diff --git a/src/core/iomgr/tcp_windows.c b/src/core/iomgr/tcp_windows.c
index 1bf81a73e0..89aa741470 100644
--- a/src/core/iomgr/tcp_windows.c
+++ b/src/core/iomgr/tcp_windows.c
@@ -96,6 +96,8 @@ typedef struct grpc_tcp {
      to protect ourselves when requesting a shutdown. */
   gpr_mu mu;
   int shutting_down;
+
+  char *peer_string;
 } grpc_tcp;
 
 static void tcp_ref(grpc_tcp *tcp) {
@@ -107,6 +109,7 @@ static void tcp_unref(grpc_tcp *tcp) {
     gpr_slice_buffer_destroy(&tcp->write_slices);
     grpc_winsocket_orphan(tcp->socket);
     gpr_mu_destroy(&tcp->mu);
+    gpr_free(tcp->peer_string);
     gpr_free(tcp);
   }
 }
@@ -365,8 +368,17 @@ static grpc_endpoint_write_status win_write(grpc_endpoint *ep,
   return GRPC_ENDPOINT_WRITE_PENDING;
 }
 
-static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) {
-  grpc_tcp *tcp = (grpc_tcp *) ep;
+static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *ps) {
+  grpc_tcp *tcp;
+  (void) ps;
+  tcp = (grpc_tcp *) ep;
+  grpc_iocp_add_socket(tcp->socket);
+}
+
+static void win_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pss) {
+  grpc_tcp *tcp;
+  (void) pss;
+  tcp = (grpc_tcp *) ep;
   grpc_iocp_add_socket(tcp->socket);
 }
 
@@ -393,11 +405,17 @@ static void win_destroy(grpc_endpoint *ep) {
   tcp_unref(tcp);
 }
 
-static grpc_endpoint_vtable vtable = {
-  win_notify_on_read, win_write, win_add_to_pollset, win_shutdown, win_destroy
-};
+static char *win_get_peer(grpc_endpoint *ep) {
+  grpc_tcp *tcp = (grpc_tcp *)ep;
+  return gpr_strdup(tcp->peer_string);
+}
+
+static grpc_endpoint_vtable vtable = {win_notify_on_read, win_write,
+                                      win_add_to_pollset, win_add_to_pollset_set,
+                                      win_shutdown,       win_destroy,
+                                      win_get_peer};
 
-grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket) {
+grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *) gpr_malloc(sizeof(grpc_tcp));
   memset(tcp, 0, sizeof(grpc_tcp));
   tcp->base.vtable = &vtable;
@@ -405,6 +423,7 @@ grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket) {
   gpr_mu_init(&tcp->mu);
   gpr_slice_buffer_init(&tcp->write_slices);
   gpr_ref_init(&tcp->refcount, 1);
+  tcp->peer_string = gpr_strdup(peer_string);
   return &tcp->base;
 }
 
diff --git a/src/core/iomgr/tcp_windows.h b/src/core/iomgr/tcp_windows.h
index 4cbc12c53a..7e301db250 100644
--- a/src/core/iomgr/tcp_windows.h
+++ b/src/core/iomgr/tcp_windows.h
@@ -50,7 +50,7 @@
 /* Create a tcp endpoint given a winsock handle.
  * Takes ownership of the handle.
  */
-grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket);
+grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string);
 
 int grpc_tcp_prepare_socket(SOCKET sock);
 
diff --git a/src/core/iomgr/wakeup_fd_eventfd.c b/src/core/iomgr/wakeup_fd_eventfd.c
index 99c32bb9db..52912235f8 100644
--- a/src/core/iomgr/wakeup_fd_eventfd.c
+++ b/src/core/iomgr/wakeup_fd_eventfd.c
@@ -42,7 +42,7 @@
 #include "src/core/iomgr/wakeup_fd_posix.h"
 #include <grpc/support/log.h>
 
-static void eventfd_create(grpc_wakeup_fd_info *fd_info) {
+static void eventfd_create(grpc_wakeup_fd *fd_info) {
   int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
   /* TODO(klempner): Handle failure more gracefully */
   GPR_ASSERT(efd >= 0);
@@ -50,7 +50,7 @@ static void eventfd_create(grpc_wakeup_fd_info *fd_info) {
   fd_info->write_fd = -1;
 }
 
-static void eventfd_consume(grpc_wakeup_fd_info *fd_info) {
+static void eventfd_consume(grpc_wakeup_fd *fd_info) {
   eventfd_t value;
   int err;
   do {
@@ -58,14 +58,14 @@ static void eventfd_consume(grpc_wakeup_fd_info *fd_info) {
   } while (err < 0 && errno == EINTR);
 }
 
-static void eventfd_wakeup(grpc_wakeup_fd_info *fd_info) {
+static void eventfd_wakeup(grpc_wakeup_fd *fd_info) {
   int err;
   do {
     err = eventfd_write(fd_info->read_fd, 1);
   } while (err < 0 && errno == EINTR);
 }
 
-static void eventfd_destroy(grpc_wakeup_fd_info *fd_info) {
+static void eventfd_destroy(grpc_wakeup_fd *fd_info) {
   close(fd_info->read_fd);
 }
 
diff --git a/src/core/iomgr/wakeup_fd_pipe.c b/src/core/iomgr/wakeup_fd_pipe.c
index f895478990..9fc4ee2388 100644
--- a/src/core/iomgr/wakeup_fd_pipe.c
+++ b/src/core/iomgr/wakeup_fd_pipe.c
@@ -44,7 +44,7 @@
 #include "src/core/iomgr/socket_utils_posix.h"
 #include <grpc/support/log.h>
 
-static void pipe_create(grpc_wakeup_fd_info *fd_info) {
+static void pipe_init(grpc_wakeup_fd *fd_info) {
   int pipefd[2];
   /* TODO(klempner): Make this nonfatal */
   GPR_ASSERT(0 == pipe(pipefd));
@@ -54,7 +54,7 @@ static void pipe_create(grpc_wakeup_fd_info *fd_info) {
   fd_info->write_fd = pipefd[1];
 }
 
-static void pipe_consume(grpc_wakeup_fd_info *fd_info) {
+static void pipe_consume(grpc_wakeup_fd *fd_info) {
   char buf[128];
   int r;
 
@@ -74,13 +74,13 @@ static void pipe_consume(grpc_wakeup_fd_info *fd_info) {
   }
 }
 
-static void pipe_wakeup(grpc_wakeup_fd_info *fd_info) {
+static void pipe_wakeup(grpc_wakeup_fd *fd_info) {
   char c = 0;
   while (write(fd_info->write_fd, &c, 1) != 1 && errno == EINTR)
     ;
 }
 
-static void pipe_destroy(grpc_wakeup_fd_info *fd_info) {
+static void pipe_destroy(grpc_wakeup_fd *fd_info) {
   close(fd_info->read_fd);
   close(fd_info->write_fd);
 }
@@ -91,7 +91,7 @@ static int pipe_check_availability(void) {
 }
 
 const grpc_wakeup_fd_vtable grpc_pipe_wakeup_fd_vtable = {
-  pipe_create, pipe_consume, pipe_wakeup, pipe_destroy, pipe_check_availability
-};
+    pipe_init, pipe_consume, pipe_wakeup, pipe_destroy,
+    pipe_check_availability};
 
 #endif  /* GPR_POSIX_WAKUP_FD */
diff --git a/src/core/iomgr/wakeup_fd_posix.c b/src/core/iomgr/wakeup_fd_posix.c
index d3cc3ec570..e48f5223fa 100644
--- a/src/core/iomgr/wakeup_fd_posix.c
+++ b/src/core/iomgr/wakeup_fd_posix.c
@@ -57,19 +57,19 @@ void grpc_wakeup_fd_global_destroy(void) {
   wakeup_fd_vtable = NULL;
 }
 
-void grpc_wakeup_fd_create(grpc_wakeup_fd_info *fd_info) {
-  wakeup_fd_vtable->create(fd_info);
+void grpc_wakeup_fd_init(grpc_wakeup_fd *fd_info) {
+  wakeup_fd_vtable->init(fd_info);
 }
 
-void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd_info *fd_info) {
+void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd *fd_info) {
   wakeup_fd_vtable->consume(fd_info);
 }
 
-void grpc_wakeup_fd_wakeup(grpc_wakeup_fd_info *fd_info) {
+void grpc_wakeup_fd_wakeup(grpc_wakeup_fd *fd_info) {
   wakeup_fd_vtable->wakeup(fd_info);
 }
 
-void grpc_wakeup_fd_destroy(grpc_wakeup_fd_info *fd_info) {
+void grpc_wakeup_fd_destroy(grpc_wakeup_fd *fd_info) {
   wakeup_fd_vtable->destroy(fd_info);
 }
 
diff --git a/src/core/iomgr/wakeup_fd_posix.h b/src/core/iomgr/wakeup_fd_posix.h
index 1b0ff70c7f..a4da4df51f 100644
--- a/src/core/iomgr/wakeup_fd_posix.h
+++ b/src/core/iomgr/wakeup_fd_posix.h
@@ -69,28 +69,28 @@ void grpc_wakeup_fd_global_destroy(void);
  * purposes only.*/
 void grpc_wakeup_fd_global_init_force_fallback(void);
 
-typedef struct grpc_wakeup_fd_info grpc_wakeup_fd_info;
+typedef struct grpc_wakeup_fd grpc_wakeup_fd;
 
 typedef struct grpc_wakeup_fd_vtable {
-  void (*create)(grpc_wakeup_fd_info *fd_info);
-  void (*consume)(grpc_wakeup_fd_info *fd_info);
-  void (*wakeup)(grpc_wakeup_fd_info *fd_info);
-  void (*destroy)(grpc_wakeup_fd_info *fd_info);
+  void (*init)(grpc_wakeup_fd *fd_info);
+  void (*consume)(grpc_wakeup_fd *fd_info);
+  void (*wakeup)(grpc_wakeup_fd *fd_info);
+  void (*destroy)(grpc_wakeup_fd *fd_info);
   /* Must be called before calling any other functions */
   int (*check_availability)(void);
 } grpc_wakeup_fd_vtable;
 
-struct grpc_wakeup_fd_info {
+struct grpc_wakeup_fd {
   int read_fd;
   int write_fd;
 };
 
 #define GRPC_WAKEUP_FD_GET_READ_FD(fd_info) ((fd_info)->read_fd)
 
-void grpc_wakeup_fd_create(grpc_wakeup_fd_info *fd_info);
-void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd_info *fd_info);
-void grpc_wakeup_fd_wakeup(grpc_wakeup_fd_info *fd_info);
-void grpc_wakeup_fd_destroy(grpc_wakeup_fd_info *fd_info);
+void grpc_wakeup_fd_init(grpc_wakeup_fd *fd_info);
+void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd *fd_info);
+void grpc_wakeup_fd_wakeup(grpc_wakeup_fd *fd_info);
+void grpc_wakeup_fd_destroy(grpc_wakeup_fd *fd_info);
 
 /* Defined in some specialized implementation's .c file, or by
  * wakeup_fd_nospecial.c if no such implementation exists. */
diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c
index 9e49a807f1..0e699874bc 100644
--- a/src/core/security/client_auth_filter.c
+++ b/src/core/security/client_auth_filter.c
@@ -77,10 +77,9 @@ typedef struct {
 
 static void bubble_up_error(grpc_call_element *elem, const char *error_msg) {
   call_data *calld = elem->call_data;
-  channel_data *chand = elem->channel_data;
-  grpc_transport_stream_op_add_cancellation(
-      &calld->op, GRPC_STATUS_UNAUTHENTICATED,
-      grpc_mdstr_from_string(chand->md_ctx, error_msg));
+  gpr_log(GPR_ERROR, "Client side authentication failure: %s", error_msg);
+  grpc_transport_stream_op_add_cancellation(&calld->op,
+                                            GRPC_STATUS_UNAUTHENTICATED);
   grpc_call_next_op(elem, &calld->op);
 }
 
@@ -316,10 +315,10 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
       (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF(
           sc, "client_auth_filter");
   chand->md_ctx = metadata_context;
-  chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority");
-  chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path");
-  chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message");
-  chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status");
+  chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority", 0);
+  chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path", 0);
+  chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message", 0);
+  chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status", 0);
 }
 
 /* Destructor for channel data */
@@ -344,6 +343,8 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
 }
 
 const grpc_channel_filter grpc_client_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem,          destroy_call_elem,    sizeof(channel_data),
-    init_channel_elem,       destroy_channel_elem, "client-auth"};
+    auth_start_transport_op, grpc_channel_next_op,
+    sizeof(call_data),       init_call_elem,
+    destroy_call_elem,       sizeof(channel_data),
+    init_channel_elem,       destroy_channel_elem,
+    grpc_call_next_get_peer, "client-auth"};
diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c
index fb59fa4b0e..6421ce673d 100644
--- a/src/core/security/credentials.c
+++ b/src/core/security/credentials.c
@@ -149,6 +149,12 @@ grpc_security_status grpc_server_credentials_create_security_connector(
   return creds->vtable->create_security_connector(creds, sc);
 }
 
+void grpc_server_credentials_set_auth_metadata_processor(
+    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
+  if (creds == NULL) return;
+  creds->processor = processor;
+}
+
 /* -- Ssl credentials. -- */
 
 static void ssl_destroy(grpc_credentials *creds) {
@@ -259,8 +265,10 @@ static void ssl_build_config(const char *pem_root_certs,
 
 static void ssl_build_server_config(
     const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs, grpc_ssl_server_config *config) {
+    size_t num_key_cert_pairs, int force_client_auth,
+    grpc_ssl_server_config *config) {
   size_t i;
+  config->force_client_auth = force_client_auth;
   if (pem_root_certs != NULL) {
     ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
                           &config->pem_root_certs_size);
@@ -302,20 +310,20 @@ grpc_credentials *grpc_ssl_credentials_create(
 
 grpc_server_credentials *grpc_ssl_server_credentials_create(
     const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs) {
+    size_t num_key_cert_pairs, int force_client_auth) {
   grpc_ssl_server_credentials *c =
       gpr_malloc(sizeof(grpc_ssl_server_credentials));
   memset(c, 0, sizeof(grpc_ssl_server_credentials));
   c->base.type = GRPC_CREDENTIALS_TYPE_SSL;
   c->base.vtable = &ssl_server_vtable;
   ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
-                          num_key_cert_pairs, &c->config);
+                          num_key_cert_pairs, force_client_auth, &c->config);
   return &c->base;
 }
 
 /* -- Jwt credentials -- */
 
-static void jwt_reset_cache(grpc_jwt_credentials *c) {
+static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
   if (c->cached.jwt_md != NULL) {
     grpc_credentials_md_store_unref(c->cached.jwt_md);
     c->cached.jwt_md = NULL;
@@ -328,7 +336,8 @@ static void jwt_reset_cache(grpc_jwt_credentials *c) {
 }
 
 static void jwt_destroy(grpc_credentials *creds) {
-  grpc_jwt_credentials *c = (grpc_jwt_credentials *)creds;
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
   grpc_auth_json_key_destruct(&c->key);
   jwt_reset_cache(c);
   gpr_mu_destroy(&c->cache_mu);
@@ -346,7 +355,8 @@ static void jwt_get_request_metadata(grpc_credentials *creds,
                                      const char *service_url,
                                      grpc_credentials_metadata_cb cb,
                                      void *user_data) {
-  grpc_jwt_credentials *c = (grpc_jwt_credentials *)creds;
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
 
@@ -399,15 +409,16 @@ static grpc_credentials_vtable jwt_vtable = {
     jwt_destroy, jwt_has_request_metadata, jwt_has_request_metadata_only,
     jwt_get_request_metadata, NULL};
 
-grpc_credentials *grpc_jwt_credentials_create_from_auth_json_key(
+grpc_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
     grpc_auth_json_key key, gpr_timespec token_lifetime) {
-  grpc_jwt_credentials *c;
+  grpc_service_account_jwt_access_credentials *c;
   if (!grpc_auth_json_key_is_valid(&key)) {
     gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
     return NULL;
   }
-  c = gpr_malloc(sizeof(grpc_jwt_credentials));
-  memset(c, 0, sizeof(grpc_jwt_credentials));
+  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
+  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
   c->base.type = GRPC_CREDENTIALS_TYPE_JWT;
   gpr_ref_init(&c->base.refcount, 1);
   c->base.vtable = &jwt_vtable;
@@ -418,9 +429,9 @@ grpc_credentials *grpc_jwt_credentials_create_from_auth_json_key(
   return &c->base;
 }
 
-grpc_credentials *grpc_jwt_credentials_create(const char *json_key,
-                                              gpr_timespec token_lifetime) {
-  return grpc_jwt_credentials_create_from_auth_json_key(
+grpc_credentials *grpc_service_account_jwt_access_credentials_create(
+    const char *json_key, gpr_timespec token_lifetime) {
+  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
       grpc_auth_json_key_create_from_string(json_key), token_lifetime);
 }
 
@@ -674,7 +685,7 @@ static void service_account_fetch_oauth2(
   request.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
   request.hdr_count = 1;
   request.hdrs = &header;
-  request.use_ssl = 1;
+  request.handshaker = &grpc_httpcli_ssl;
   grpc_httpcli_post(httpcli_context, pollset, &request, body, strlen(body),
                     deadline, response_cb, metadata_req);
   gpr_free(body);
@@ -733,7 +744,7 @@ static void refresh_token_fetch_oauth2(
   request.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
   request.hdr_count = 1;
   request.hdrs = &header;
-  request.use_ssl = 1;
+  request.handshaker = &grpc_httpcli_ssl;
   grpc_httpcli_post(httpcli_context, pollset, &request, body, strlen(body),
                     deadline, response_cb, metadata_req);
   gpr_free(body);
@@ -760,19 +771,19 @@ grpc_credentials *grpc_refresh_token_credentials_create(
       grpc_auth_refresh_token_create_from_string(json_refresh_token));
 }
 
-/* -- Fake Oauth2 credentials. -- */
+/* -- Metadata-only credentials. -- */
 
-static void fake_oauth2_destroy(grpc_credentials *creds) {
-  grpc_fake_oauth2_credentials *c = (grpc_fake_oauth2_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
+static void md_only_test_destroy(grpc_credentials *creds) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+  grpc_credentials_md_store_unref(c->md_store);
   gpr_free(c);
 }
 
-static int fake_oauth2_has_request_metadata(const grpc_credentials *creds) {
+static int md_only_test_has_request_metadata(const grpc_credentials *creds) {
   return 1;
 }
 
-static int fake_oauth2_has_request_metadata_only(
+static int md_only_test_has_request_metadata_only(
     const grpc_credentials *creds) {
   return 1;
 }
@@ -780,19 +791,19 @@ static int fake_oauth2_has_request_metadata_only(
 void on_simulated_token_fetch_done(void *user_data, int success) {
   grpc_credentials_metadata_request *r =
       (grpc_credentials_metadata_request *)user_data;
-  grpc_fake_oauth2_credentials *c = (grpc_fake_oauth2_credentials *)r->creds;
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
   GPR_ASSERT(success);
-  r->cb(r->user_data, c->access_token_md->entries,
-        c->access_token_md->num_entries, GRPC_CREDENTIALS_OK);
+  r->cb(r->user_data, c->md_store->entries,
+        c->md_store->num_entries, GRPC_CREDENTIALS_OK);
   grpc_credentials_metadata_request_destroy(r);
 }
 
-static void fake_oauth2_get_request_metadata(grpc_credentials *creds,
+static void md_only_test_get_request_metadata(grpc_credentials *creds,
                                              grpc_pollset *pollset,
                                              const char *service_url,
                                              grpc_credentials_metadata_cb cb,
                                              void *user_data) {
-  grpc_fake_oauth2_credentials *c = (grpc_fake_oauth2_credentials *)creds;
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
 
   if (c->is_async) {
     grpc_credentials_metadata_request *cb_arg =
@@ -801,26 +812,26 @@ static void fake_oauth2_get_request_metadata(grpc_credentials *creds,
                             on_simulated_token_fetch_done, cb_arg);
     grpc_iomgr_add_callback(cb_arg->on_simulated_token_fetch_done_closure);
   } else {
-    cb(user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
+    cb(user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
   }
 }
 
-static grpc_credentials_vtable fake_oauth2_vtable = {
-    fake_oauth2_destroy, fake_oauth2_has_request_metadata,
-    fake_oauth2_has_request_metadata_only, fake_oauth2_get_request_metadata,
+static grpc_credentials_vtable md_only_test_vtable = {
+    md_only_test_destroy, md_only_test_has_request_metadata,
+    md_only_test_has_request_metadata_only, md_only_test_get_request_metadata,
     NULL};
 
-grpc_credentials *grpc_fake_oauth2_credentials_create(
-    const char *token_md_value, int is_async) {
-  grpc_fake_oauth2_credentials *c =
-      gpr_malloc(sizeof(grpc_fake_oauth2_credentials));
-  memset(c, 0, sizeof(grpc_fake_oauth2_credentials));
+grpc_credentials *grpc_md_only_test_credentials_create(const char *md_key,
+                                                       const char *md_value,
+                                                       int is_async) {
+  grpc_md_only_test_credentials *c =
+      gpr_malloc(sizeof(grpc_md_only_test_credentials));
+  memset(c, 0, sizeof(grpc_md_only_test_credentials));
   c->base.type = GRPC_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &fake_oauth2_vtable;
+  c->base.vtable = &md_only_test_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  c->md_store = grpc_credentials_md_store_create(1);
+  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
   c->is_async = is_async;
   return &c->base;
 }
diff --git a/src/core/security/credentials.h b/src/core/security/credentials.h
index d988901cf7..04736525dc 100644
--- a/src/core/security/credentials.h
+++ b/src/core/security/credentials.h
@@ -52,6 +52,8 @@ typedef enum {
   GRPC_CREDENTIALS_ERROR
 } grpc_credentials_status;
 
+#define GRPC_FAKE_TRANSPORT_SECURITY_TYPE "fake"
+
 #define GRPC_CREDENTIALS_TYPE_SSL "Ssl"
 #define GRPC_CREDENTIALS_TYPE_OAUTH2 "Oauth2"
 #define GRPC_CREDENTIALS_TYPE_JWT "Jwt"
@@ -112,6 +114,12 @@ void grpc_credentials_md_store_unref(grpc_credentials_md_store *store);
 
 /* --- grpc_credentials. --- */
 
+/* Creates a fake transport security credentials object for testing. */
+grpc_credentials *grpc_fake_transport_security_credentials_create(void);
+/* Creates a fake server transport security credentials object for testing. */
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void);
+
 /* It is the caller's responsibility to gpr_free the result if not NULL. */
 char *grpc_get_well_known_google_credentials_file_path(void);
 
@@ -182,13 +190,15 @@ grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
 void grpc_flush_cached_google_default_credentials(void);
 
-/* Simulates an oauth2 token fetch with the specified value for testing. */
-grpc_credentials *grpc_fake_oauth2_credentials_create(
-    const char *token_md_value, int is_async);
+/* Metadata-only credentials with the specified key and value where
+   asynchronicity can be simulated for testing. */
+grpc_credentials *grpc_md_only_test_credentials_create(
+    const char *md_key, const char *md_value, int is_async);
 
 /* Private constructor for jwt credentials from an already parsed json key.
    Takes ownership of the key. */
-grpc_credentials *grpc_jwt_credentials_create_from_auth_json_key(
+grpc_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
     grpc_auth_json_key key, gpr_timespec token_lifetime);
 
 /* Private constructor for refresh token credentials from an already parsed
@@ -207,6 +217,7 @@ typedef struct {
 struct grpc_server_credentials {
   const grpc_server_credentials_vtable *vtable;
   const char *type;
+  grpc_auth_metadata_processor processor;
 };
 
 grpc_security_status grpc_server_credentials_create_security_connector(
@@ -240,7 +251,7 @@ typedef struct {
 
   grpc_auth_json_key key;
   gpr_timespec jwt_lifetime;
-} grpc_jwt_credentials;
+} grpc_service_account_jwt_access_credentials;
 
 /* -- Oauth2TokenFetcher credentials --
 
@@ -288,13 +299,13 @@ typedef struct {
   grpc_credentials_md_store *access_token_md;
 } grpc_access_token_credentials;
 
-/* -- Fake Oauth2 credentials. -- */
+/* --  Metadata-only Test credentials. -- */
 
 typedef struct {
   grpc_credentials base;
-  grpc_credentials_md_store *access_token_md;
+  grpc_credentials_md_store *md_store;
   int is_async;
-} grpc_fake_oauth2_credentials;
+} grpc_md_only_test_credentials;
 
 /* -- IAM credentials. -- */
 
diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c
index 833484310f..f368819597 100644
--- a/src/core/security/google_default_credentials.c
+++ b/src/core/security/google_default_credentials.c
@@ -80,7 +80,7 @@ static void on_compute_engine_detection_http_response(
   }
   gpr_mu_lock(GRPC_POLLSET_MU(&detector->pollset));
   detector->is_done = 1;
-  grpc_pollset_kick(&detector->pollset);
+  grpc_pollset_kick(&detector->pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&detector->pollset));
 }
 
@@ -112,7 +112,9 @@ static int is_stack_running_on_compute_engine(void) {
      called once for the lifetime of the process by the default credentials. */
   gpr_mu_lock(GRPC_POLLSET_MU(&detector.pollset));
   while (!detector.is_done) {
-    grpc_pollset_work(&detector.pollset, gpr_inf_future(GPR_CLOCK_REALTIME));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&detector.pollset, &worker,
+                      gpr_inf_future(GPR_CLOCK_REALTIME));
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(&detector.pollset));
 
@@ -140,8 +142,9 @@ static grpc_credentials *create_default_creds_from_path(char *creds_path) {
   /* First, try an auth json key. */
   key = grpc_auth_json_key_create_from_json(json);
   if (grpc_auth_json_key_is_valid(&key)) {
-    result = grpc_jwt_credentials_create_from_auth_json_key(
-        key, grpc_max_auth_token_lifetime);
+    result =
+        grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+            key, grpc_max_auth_token_lifetime);
     goto end;
   }
 
diff --git a/src/core/security/jwt_verifier.c b/src/core/security/jwt_verifier.c
index 1276693da7..38ad134a6a 100644
--- a/src/core/security/jwt_verifier.c
+++ b/src/core/security/jwt_verifier.c
@@ -628,7 +628,7 @@ static void on_openid_config_retrieved(void *user_data,
     goto error;
   }
   jwks_uri += 8;
-  req.use_ssl = 1;
+  req.handshaker = &grpc_httpcli_ssl;
   req.host = gpr_strdup(jwks_uri);
   req.path = strchr(jwks_uri, '/');
   if (req.path == NULL) {
@@ -685,7 +685,7 @@ static void retrieve_key_and_verify(verifier_cb_ctx *ctx) {
   const char *iss;
   grpc_httpcli_request req;
   memset(&req, 0, sizeof(grpc_httpcli_request));
-  req.use_ssl = 1;
+  req.handshaker = &grpc_httpcli_ssl;
 
   GPR_ASSERT(ctx != NULL && ctx->header != NULL && ctx->claims != NULL);
   iss = ctx->claims->iss;
diff --git a/src/core/security/secure_endpoint.c b/src/core/security/secure_endpoint.c
index 3548198046..95fbf71f3d 100644
--- a/src/core/security/secure_endpoint.c
+++ b/src/core/security/secure_endpoint.c
@@ -331,9 +331,22 @@ static void endpoint_add_to_pollset(grpc_endpoint *secure_ep,
   grpc_endpoint_add_to_pollset(ep->wrapped_ep, pollset);
 }
 
+static void endpoint_add_to_pollset_set(grpc_endpoint *secure_ep,
+                                    grpc_pollset_set *pollset_set) {
+  secure_endpoint *ep = (secure_endpoint *)secure_ep;
+  grpc_endpoint_add_to_pollset_set(ep->wrapped_ep, pollset_set);
+}
+
+static char *endpoint_get_peer(grpc_endpoint *secure_ep) {
+  secure_endpoint *ep = (secure_endpoint *)secure_ep;
+  return grpc_endpoint_get_peer(ep->wrapped_ep);
+}
+
 static const grpc_endpoint_vtable vtable = {
-    endpoint_notify_on_read, endpoint_write, endpoint_add_to_pollset,
-    endpoint_shutdown, endpoint_unref};
+    endpoint_notify_on_read, endpoint_write,
+    endpoint_add_to_pollset, endpoint_add_to_pollset_set,
+    endpoint_shutdown,       endpoint_unref,
+    endpoint_get_peer};
 
 grpc_endpoint *grpc_secure_endpoint_create(
     struct tsi_frame_protector *protector, grpc_endpoint *transport,
diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c
index f6e423eb27..a354536dcd 100644
--- a/src/core/security/security_connector.c
+++ b/src/core/security/security_connector.c
@@ -263,9 +263,9 @@ static grpc_security_status fake_check_peer(grpc_security_connector *sc,
     goto end;
   }
   GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
-  sc->auth_context = grpc_auth_context_create(NULL, 1);
-  sc->auth_context->properties[0] = grpc_auth_property_init_from_cstring(
-      GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
+  sc->auth_context = grpc_auth_context_create(NULL);
+  grpc_auth_context_add_cstring_property(
+      sc->auth_context, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
       GRPC_FAKE_TRANSPORT_SECURITY_TYPE);
 
 end:
@@ -409,31 +409,35 @@ static int ssl_host_matches_name(const tsi_peer *peer, const char *peer_name) {
 grpc_auth_context *tsi_ssl_peer_to_auth_context(const tsi_peer *peer) {
   size_t i;
   grpc_auth_context *ctx = NULL;
+  const char *peer_identity_property_name = NULL;
 
   /* The caller has checked the certificate type property. */
   GPR_ASSERT(peer->property_count >= 1);
-  ctx = grpc_auth_context_create(NULL, peer->property_count);
-  ctx->properties[0] = grpc_auth_property_init_from_cstring(
-      GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
+  ctx = grpc_auth_context_create(NULL);
+  grpc_auth_context_add_cstring_property(
+      ctx, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
       GRPC_SSL_TRANSPORT_SECURITY_TYPE);
-  ctx->property_count = 1;
   for (i = 0; i < peer->property_count; i++) {
     const tsi_peer_property *prop = &peer->properties[i];
     if (prop->name == NULL) continue;
     if (strcmp(prop->name, TSI_X509_SUBJECT_COMMON_NAME_PEER_PROPERTY) == 0) {
       /* If there is no subject alt name, have the CN as the identity. */
-      if (ctx->peer_identity_property_name == NULL) {
-        ctx->peer_identity_property_name = GRPC_X509_CN_PROPERTY_NAME;
+      if (peer_identity_property_name == NULL) {
+        peer_identity_property_name = GRPC_X509_CN_PROPERTY_NAME;
       }
-      ctx->properties[ctx->property_count++] = grpc_auth_property_init(
-          GRPC_X509_CN_PROPERTY_NAME, prop->value.data, prop->value.length);
+      grpc_auth_context_add_property(ctx, GRPC_X509_CN_PROPERTY_NAME,
+                                     prop->value.data, prop->value.length);
     } else if (strcmp(prop->name,
                       TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY) == 0) {
-      ctx->peer_identity_property_name = GRPC_X509_SAN_PROPERTY_NAME;
-      ctx->properties[ctx->property_count++] = grpc_auth_property_init(
-          GRPC_X509_SAN_PROPERTY_NAME, prop->value.data, prop->value.length);
+      peer_identity_property_name = GRPC_X509_SAN_PROPERTY_NAME;
+      grpc_auth_context_add_property(ctx, GRPC_X509_SAN_PROPERTY_NAME,
+                                     prop->value.data, prop->value.length);
     }
   }
+  if (peer_identity_property_name != NULL) {
+    GPR_ASSERT(grpc_auth_context_set_peer_identity_property_name(
+                   ctx, peer_identity_property_name) == 1);
+  }
   return ctx;
 }
 
@@ -653,9 +657,10 @@ grpc_security_status grpc_ssl_server_security_connector_create(
       config->pem_private_keys_sizes,
       (const unsigned char **)config->pem_cert_chains,
       config->pem_cert_chains_sizes, config->num_key_cert_pairs,
-      config->pem_root_certs, config->pem_root_certs_size, ssl_cipher_suites(),
-      alpn_protocol_strings, alpn_protocol_string_lengths,
-      (uint16_t)num_alpn_protocols, &c->handshaker_factory);
+      config->pem_root_certs, config->pem_root_certs_size,
+      config->force_client_auth, ssl_cipher_suites(), alpn_protocol_strings,
+      alpn_protocol_string_lengths, (uint16_t)num_alpn_protocols,
+      &c->handshaker_factory);
   if (result != TSI_OK) {
     gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.",
             tsi_result_to_string(result));
diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h
index a4c723f026..2c9aa1c5a4 100644
--- a/src/core/security/security_connector.h
+++ b/src/core/security/security_connector.h
@@ -201,6 +201,7 @@ typedef struct {
   size_t num_key_cert_pairs;
   unsigned char *pem_root_certs;
   size_t pem_root_certs_size;
+  int force_client_auth;
 } grpc_ssl_server_config;
 
 /* Creates an SSL server_security_connector.
diff --git a/src/core/security/security_context.c b/src/core/security/security_context.c
index 8ce7876bd8..1ef0fc9255 100644
--- a/src/core/security/security_context.c
+++ b/src/core/security/security_context.c
@@ -42,6 +42,19 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+/* --- grpc_process_auth_metadata_func --- */
+
+static grpc_auth_metadata_processor server_processor = {NULL, NULL};
+
+grpc_auth_metadata_processor grpc_server_get_auth_metadata_processor(void) {
+  return server_processor;
+}
+
+void grpc_server_register_auth_metadata_processor(
+    grpc_auth_metadata_processor processor) {
+  server_processor = processor;
+}
+
 /* --- grpc_call --- */
 
 grpc_call_error grpc_call_set_credentials(grpc_call *call,
@@ -120,15 +133,15 @@ void grpc_server_security_context_destroy(void *ctx) {
 
 static grpc_auth_property_iterator empty_iterator = {NULL, 0, NULL};
 
-grpc_auth_context *grpc_auth_context_create(grpc_auth_context *chained,
-                                            size_t property_count) {
+grpc_auth_context *grpc_auth_context_create(grpc_auth_context *chained) {
   grpc_auth_context *ctx = gpr_malloc(sizeof(grpc_auth_context));
   memset(ctx, 0, sizeof(grpc_auth_context));
-  ctx->properties = gpr_malloc(property_count * sizeof(grpc_auth_property));
-  memset(ctx->properties, 0, property_count * sizeof(grpc_auth_property));
-  ctx->property_count = property_count;
   gpr_ref_init(&ctx->refcount, 1);
-  if (chained != NULL) ctx->chained = GRPC_AUTH_CONTEXT_REF(chained, "chained");
+  if (chained != NULL) {
+    ctx->chained = GRPC_AUTH_CONTEXT_REF(chained, "chained");
+    ctx->peer_identity_property_name =
+        ctx->chained->peer_identity_property_name;
+  }
   return ctx;
 }
 
@@ -162,11 +175,11 @@ void grpc_auth_context_unref(grpc_auth_context *ctx) {
   if (gpr_unref(&ctx->refcount)) {
     size_t i;
     GRPC_AUTH_CONTEXT_UNREF(ctx->chained, "chained");
-    if (ctx->properties != NULL) {
-      for (i = 0; i < ctx->property_count; i++) {
-        grpc_auth_property_reset(&ctx->properties[i]);
+    if (ctx->properties.array != NULL) {
+      for (i = 0; i < ctx->properties.count; i++) {
+        grpc_auth_property_reset(&ctx->properties.array[i]);
       }
-      gpr_free(ctx->properties);
+      gpr_free(ctx->properties.array);
     }
     gpr_free(ctx);
   }
@@ -177,6 +190,20 @@ const char *grpc_auth_context_peer_identity_property_name(
   return ctx->peer_identity_property_name;
 }
 
+int grpc_auth_context_set_peer_identity_property_name(grpc_auth_context *ctx,
+                                                      const char *name) {
+  grpc_auth_property_iterator it =
+      grpc_auth_context_find_properties_by_name(ctx, name);
+  const grpc_auth_property *prop = grpc_auth_property_iterator_next(&it);
+  if (prop == NULL) {
+    gpr_log(GPR_ERROR, "Property name %s not found in auth context.",
+            name != NULL ? name : "NULL");
+    return 0;
+  }
+  ctx->peer_identity_property_name = prop->name;
+  return 1;
+}
+
 int grpc_auth_context_peer_is_authenticated(
     const grpc_auth_context *ctx) {
   return ctx->peer_identity_property_name == NULL ? 0 : 1;
@@ -193,16 +220,16 @@ grpc_auth_property_iterator grpc_auth_context_property_iterator(
 const grpc_auth_property *grpc_auth_property_iterator_next(
     grpc_auth_property_iterator *it) {
   if (it == NULL || it->ctx == NULL) return NULL;
-  while (it->index == it->ctx->property_count) {
+  while (it->index == it->ctx->properties.count) {
     if (it->ctx->chained == NULL) return NULL;
     it->ctx = it->ctx->chained;
     it->index = 0;
   }
   if (it->name == NULL) {
-    return &it->ctx->properties[it->index++];
+    return &it->ctx->properties.array[it->index++];
   } else {
-    while (it->index < it->ctx->property_count) {
-      const grpc_auth_property *prop = &it->ctx->properties[it->index++];
+    while (it->index < it->ctx->properties.count) {
+      const grpc_auth_property *prop = &it->ctx->properties.array[it->index++];
       GPR_ASSERT(prop->name != NULL);
       if (strcmp(it->name, prop->name) == 0) {
         return prop;
@@ -229,24 +256,37 @@ grpc_auth_property_iterator grpc_auth_context_peer_identity(
       ctx, ctx->peer_identity_property_name);
 }
 
-grpc_auth_property grpc_auth_property_init_from_cstring(const char *name,
-                                                        const char *value) {
-  grpc_auth_property prop;
-  prop.name = gpr_strdup(name);
-  prop.value = gpr_strdup(value);
-  prop.value_length = strlen(value);
-  return prop;
+static void ensure_auth_context_capacity(grpc_auth_context *ctx) {
+  if (ctx->properties.count == ctx->properties.capacity) {
+    ctx->properties.capacity =
+        GPR_MAX(ctx->properties.capacity + 8, ctx->properties.capacity * 2);
+    ctx->properties.array =
+        gpr_realloc(ctx->properties.array,
+                    ctx->properties.capacity * sizeof(grpc_auth_property));
+  }
 }
 
-grpc_auth_property grpc_auth_property_init(const char *name, const char *value,
-                                           size_t value_length) {
-  grpc_auth_property prop;
-  prop.name = gpr_strdup(name);
-  prop.value = gpr_malloc(value_length + 1);
-  memcpy(prop.value, value, value_length);
-  prop.value[value_length] = '\0';
-  prop.value_length = value_length;
-  return prop;
+void grpc_auth_context_add_property(grpc_auth_context *ctx, const char *name,
+                                    const char *value, size_t value_length) {
+  grpc_auth_property *prop;
+  ensure_auth_context_capacity(ctx);
+  prop = &ctx->properties.array[ctx->properties.count++];
+  prop->name = gpr_strdup(name);
+  prop->value = gpr_malloc(value_length + 1);
+  memcpy(prop->value, value, value_length);
+  prop->value[value_length] = '\0';
+  prop->value_length = value_length;
+}
+
+void grpc_auth_context_add_cstring_property(grpc_auth_context *ctx,
+                                            const char *name,
+                                            const char *value) {
+  grpc_auth_property *prop;
+  ensure_auth_context_capacity(ctx);
+  prop = &ctx->properties.array[ctx->properties.count++];
+  prop->name = gpr_strdup(name);
+  prop->value = gpr_strdup(value);
+  prop->value_length = strlen(value);
 }
 
 void grpc_auth_property_reset(grpc_auth_property *property) {
@@ -255,3 +295,35 @@ void grpc_auth_property_reset(grpc_auth_property *property) {
   memset(property, 0, sizeof(grpc_auth_property));
 }
 
+grpc_arg grpc_auth_metadata_processor_to_arg(grpc_auth_metadata_processor *p) {
+  grpc_arg arg;
+  memset(&arg, 0, sizeof(grpc_arg));
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_AUTH_METADATA_PROCESSOR_ARG;
+  arg.value.pointer.p = p;
+  return arg;
+}
+
+grpc_auth_metadata_processor *grpc_auth_metadata_processor_from_arg(
+    const grpc_arg *arg) {
+  if (strcmp(arg->key, GRPC_AUTH_METADATA_PROCESSOR_ARG) != 0) return NULL;
+  if (arg->type != GRPC_ARG_POINTER) {
+    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
+            GRPC_AUTH_METADATA_PROCESSOR_ARG);
+    return NULL;
+  }
+  return arg->value.pointer.p;
+}
+
+grpc_auth_metadata_processor *grpc_find_auth_metadata_processor_in_args(
+    const grpc_channel_args *args) {
+  size_t i;
+  if (args == NULL) return NULL;
+  for (i = 0; i < args->num_args; i++) {
+    grpc_auth_metadata_processor *p =
+        grpc_auth_metadata_processor_from_arg(&args->args[i]);
+    if (p != NULL) return p;
+  }
+  return NULL;
+}
+
diff --git a/src/core/security/security_context.h b/src/core/security/security_context.h
index 76a45910bb..7fcd438cf6 100644
--- a/src/core/security/security_context.h
+++ b/src/core/security/security_context.h
@@ -34,29 +34,31 @@
 #ifndef GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONTEXT_H
 #define GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONTEXT_H
 
+#include "src/core/iomgr/pollset.h"
 #include "src/core/security/credentials.h"
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 /* --- grpc_auth_context ---
 
    High level authentication context object. Can optionally be chained. */
 
 /* Property names are always NULL terminated. */
 
+typedef struct {
+  grpc_auth_property *array;
+  size_t count;
+  size_t capacity;
+} grpc_auth_property_array;
+
 struct grpc_auth_context {
   struct grpc_auth_context *chained;
-  grpc_auth_property *properties;
-  size_t property_count;
+  grpc_auth_property_array properties;
   gpr_refcount refcount;
   const char *peer_identity_property_name;
+  grpc_pollset *pollset;
 };
 
-/* Constructor. */
-grpc_auth_context *grpc_auth_context_create(grpc_auth_context *chained,
-                                            size_t property_count);
+/* Creation. */
+grpc_auth_context *grpc_auth_context_create(grpc_auth_context *chained);
 
 /* Refcounting. */
 #ifdef GRPC_AUTH_CONTEXT_REFCOUNT_DEBUG
@@ -76,12 +78,6 @@ grpc_auth_context *grpc_auth_context_ref(grpc_auth_context *policy);
 void grpc_auth_context_unref(grpc_auth_context *policy);
 #endif
 
-grpc_auth_property grpc_auth_property_init_from_cstring(const char *name,
-                                                        const char *value);
-
-grpc_auth_property grpc_auth_property_init(const char *name, const char *value,
-                                           size_t value_length);
-
 void grpc_auth_property_reset(grpc_auth_property *property);
 
 /* --- grpc_client_security_context ---
@@ -107,9 +103,14 @@ typedef struct {
 grpc_server_security_context *grpc_server_security_context_create(void);
 void grpc_server_security_context_destroy(void *ctx);
 
-#ifdef __cplusplus
-}
-#endif
+/* --- Auth metadata processing. --- */
+#define GRPC_AUTH_METADATA_PROCESSOR_ARG "grpc.auth_metadata_processor"
+
+grpc_arg grpc_auth_metadata_processor_to_arg(grpc_auth_metadata_processor *p);
+grpc_auth_metadata_processor *grpc_auth_metadata_processor_from_arg(
+    const grpc_arg *arg);
+grpc_auth_metadata_processor *grpc_find_auth_metadata_processor_in_args(
+    const grpc_channel_args *args);
 
 #endif  /* GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONTEXT_H */
 
diff --git a/src/core/security/server_auth_filter.c b/src/core/security/server_auth_filter.c
index 10eef6d237..2fc689caec 100644
--- a/src/core/security/server_auth_filter.c
+++ b/src/core/security/server_auth_filter.c
@@ -31,20 +31,140 @@
  *
  */
 
+#include <string.h>
+
 #include "src/core/security/auth_filters.h"
 #include "src/core/security/security_connector.h"
 #include "src/core/security/security_context.h"
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
 typedef struct call_data {
-  int unused; /* C89 requires at least one struct element */
+  gpr_uint8 got_client_metadata;
+  grpc_stream_op_buffer *recv_ops;
+  /* Closure to call when finished with the auth_on_recv hook. */
+  grpc_iomgr_closure *on_done_recv;
+  /* Receive closures are chained: we inject this closure as the on_done_recv
+     up-call on transport_op, and remember to call our on_done_recv member after
+     handling it. */
+  grpc_iomgr_closure auth_on_recv;
+  grpc_transport_stream_op transport_op;
+  const grpc_metadata *consumed_md;
+  size_t num_consumed_md;
+  grpc_stream_op *md_op;
+  grpc_auth_context *auth_context;
 } call_data;
 
 typedef struct channel_data {
   grpc_security_connector *security_connector;
+  grpc_auth_metadata_processor processor;
+  grpc_mdctx *mdctx;
 } channel_data;
 
+static grpc_metadata_array metadata_batch_to_md_array(
+    const grpc_metadata_batch *batch) {
+  grpc_linked_mdelem *l;
+  grpc_metadata_array result;
+  grpc_metadata_array_init(&result);
+  for (l = batch->list.head; l != NULL; l = l->next) {
+    grpc_metadata *usr_md = NULL;
+    grpc_mdelem *md = l->md;
+    grpc_mdstr *key = md->key;
+    grpc_mdstr *value = md->value;
+    if (result.count == result.capacity) {
+      result.capacity = GPR_MAX(result.capacity + 8, result.capacity * 2);
+      result.metadata =
+          gpr_realloc(result.metadata, result.capacity * sizeof(grpc_metadata));
+    }
+    usr_md = &result.metadata[result.count++];
+    usr_md->key = grpc_mdstr_as_c_string(key);
+    usr_md->value = grpc_mdstr_as_c_string(value);
+    usr_md->value_length = GPR_SLICE_LENGTH(value->slice);
+  }
+  return result;
+}
+
+static grpc_mdelem *remove_consumed_md(void *user_data, grpc_mdelem *md) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  size_t i;
+  for (i = 0; i < calld->num_consumed_md; i++) {
+    /* Maybe we could do a pointer comparison but we do not have any guarantee
+       that the metadata processor used the same pointers for consumed_md in the
+       callback. */
+    if (memcmp(GPR_SLICE_START_PTR(md->key->slice), calld->consumed_md[i].key,
+               GPR_SLICE_LENGTH(md->key->slice)) == 0 &&
+        memcmp(GPR_SLICE_START_PTR(md->value->slice),
+               calld->consumed_md[i].value,
+               GPR_SLICE_LENGTH(md->value->slice)) == 0) {
+      return NULL; /* Delete. */
+    }
+  }
+  return md;
+}
+
+static void on_md_processing_done(void *user_data,
+                                  const grpc_metadata *consumed_md,
+                                  size_t num_consumed_md, int success) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+
+  if (success) {
+    calld->consumed_md = consumed_md;
+    calld->num_consumed_md = num_consumed_md;
+    grpc_metadata_batch_filter(&calld->md_op->data.metadata, remove_consumed_md,
+                               elem);
+    calld->on_done_recv->cb(calld->on_done_recv->cb_arg, success);
+  } else {
+    gpr_slice message = gpr_slice_from_copied_string(
+        "Authentication metadata processing failed.");
+    grpc_sopb_reset(calld->recv_ops);
+    grpc_transport_stream_op_add_close(&calld->transport_op,
+                                       GRPC_STATUS_UNAUTHENTICATED, &message);
+    grpc_call_next_op(elem, &calld->transport_op);
+  }
+}
+
+static void auth_on_recv(void *user_data, int success) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  channel_data *chand = elem->channel_data;
+  if (success) {
+    size_t i;
+    size_t nops = calld->recv_ops->nops;
+    grpc_stream_op *ops = calld->recv_ops->ops;
+    for (i = 0; i < nops; i++) {
+      grpc_metadata_array md_array;
+      grpc_stream_op *op = &ops[i];
+      if (op->type != GRPC_OP_METADATA || calld->got_client_metadata) continue;
+      calld->got_client_metadata = 1;
+      if (chand->processor.process == NULL) continue;
+      calld->md_op = op;
+      md_array = metadata_batch_to_md_array(&op->data.metadata);
+      chand->processor.process(chand->processor.state, calld->auth_context,
+                               md_array.metadata, md_array.count,
+                               on_md_processing_done, elem);
+      grpc_metadata_array_destroy(&md_array);
+      return;
+    }
+  }
+  calld->on_done_recv->cb(calld->on_done_recv->cb_arg, success);
+}
+
+static void set_recv_ops_md_callbacks(grpc_call_element *elem,
+                                      grpc_transport_stream_op *op) {
+  call_data *calld = elem->call_data;
+
+  if (op->recv_ops && !calld->got_client_metadata) {
+    /* substitute our callback for the higher callback */
+    calld->recv_ops = op->recv_ops;
+    calld->on_done_recv = op->on_done_recv;
+    op->on_done_recv = &calld->auth_on_recv;
+    calld->transport_op = *op;
+  }
+}
+
 /* Called either:
      - in response to an API call (or similar) from above, to send something
      - a network event (or similar) from below, to receive something
@@ -52,9 +172,7 @@ typedef struct channel_data {
    that is being sent or received. */
 static void auth_start_transport_op(grpc_call_element *elem,
                                     grpc_transport_stream_op *op) {
-  /* TODO(jboeuf): Get the metadata and get a new context from it. */
-
-  /* pass control down the stack */
+  set_recv_ops_md_callbacks(elem, op);
   grpc_call_next_op(elem, op);
 }
 
@@ -68,7 +186,8 @@ static void init_call_elem(grpc_call_element *elem,
   grpc_server_security_context *server_ctx = NULL;
 
   /* initialize members */
-  calld->unused = 0;
+  memset(calld, 0, sizeof(*calld));
+  grpc_iomgr_closure_init(&calld->auth_on_recv, auth_on_recv, elem);
 
   GPR_ASSERT(initial_op && initial_op->context != NULL &&
              initial_op->context[GRPC_CONTEXT_SECURITY].value == NULL);
@@ -80,21 +199,29 @@ static void init_call_elem(grpc_call_element *elem,
         initial_op->context[GRPC_CONTEXT_SECURITY].value);
   }
   server_ctx = grpc_server_security_context_create();
-  server_ctx->auth_context = GRPC_AUTH_CONTEXT_REF(
-      chand->security_connector->auth_context, "server_security_context");
+  server_ctx->auth_context =
+      grpc_auth_context_create(chand->security_connector->auth_context);
+  server_ctx->auth_context->pollset = initial_op->bind_pollset;
   initial_op->context[GRPC_CONTEXT_SECURITY].value = server_ctx;
   initial_op->context[GRPC_CONTEXT_SECURITY].destroy =
       grpc_server_security_context_destroy;
+  calld->auth_context = server_ctx->auth_context;
+
+  /* Set the metadata callbacks. */
+  set_recv_ops_md_callbacks(elem, initial_op);
 }
 
 /* Destructor for call_data */
-static void destroy_call_elem(grpc_call_element *elem) {}
+static void destroy_call_elem(grpc_call_element *elem) {
+}
 
 /* Constructor for channel_data */
 static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
                               const grpc_channel_args *args, grpc_mdctx *mdctx,
                               int is_first, int is_last) {
   grpc_security_connector *sc = grpc_find_security_connector_in_args(args);
+  grpc_auth_metadata_processor *processor =
+      grpc_find_auth_metadata_processor_in_args(args);
   /* grab pointers to our data from the channel element */
   channel_data *chand = elem->channel_data;
 
@@ -104,11 +231,14 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
   GPR_ASSERT(!is_first);
   GPR_ASSERT(!is_last);
   GPR_ASSERT(sc != NULL);
+  GPR_ASSERT(processor != NULL);
 
   /* initialize members */
   GPR_ASSERT(!sc->is_client_side);
   chand->security_connector =
       GRPC_SECURITY_CONNECTOR_REF(sc, "server_auth_filter");
+  chand->mdctx = mdctx;
+  chand->processor = *processor;
 }
 
 /* Destructor for channel data */
@@ -120,6 +250,8 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
 }
 
 const grpc_channel_filter grpc_server_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem,          destroy_call_elem,    sizeof(channel_data),
-    init_channel_elem,       destroy_channel_elem, "server-auth"};
+    auth_start_transport_op, grpc_channel_next_op,
+    sizeof(call_data),       init_call_elem,
+    destroy_call_elem,       sizeof(channel_data),
+    init_channel_elem,       destroy_channel_elem,
+    grpc_call_next_get_peer, "server-auth"};
diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c
index 3717b8989f..8d9d036d80 100644
--- a/src/core/security/server_secure_chttp2.c
+++ b/src/core/security/server_secure_chttp2.c
@@ -43,6 +43,7 @@
 #include "src/core/security/auth_filters.h"
 #include "src/core/security/credentials.h"
 #include "src/core/security/security_connector.h"
+#include "src/core/security/security_context.h"
 #include "src/core/security/secure_transport_setup.h"
 #include "src/core/surface/server.h"
 #include "src/core/transport/chttp2_transport.h"
@@ -60,6 +61,7 @@ typedef struct grpc_server_secure_state {
   grpc_server *server;
   grpc_tcp_server *tcp;
   grpc_security_connector *sc;
+  grpc_auth_metadata_processor processor;
   tcp_endpoint_list *handshaking_tcp_endpoints;
   int is_shutdown;
   gpr_mu mu;
@@ -86,9 +88,13 @@ static void setup_transport(void *statep, grpc_transport *transport,
   static grpc_channel_filter const *extra_filters[] = {
       &grpc_server_auth_filter, &grpc_http_server_filter};
   grpc_server_secure_state *state = statep;
-  grpc_arg connector_arg = grpc_security_connector_to_arg(state->sc);
-  grpc_channel_args *args_copy = grpc_channel_args_copy_and_add(
-      grpc_server_get_channel_args(state->server), &connector_arg, 1);
+  grpc_channel_args *args_copy;
+  grpc_arg args_to_add[2];
+  args_to_add[0] = grpc_security_connector_to_arg(state->sc);
+  args_to_add[1] = grpc_auth_metadata_processor_to_arg(&state->processor);
+  args_copy = grpc_channel_args_copy_and_add(
+      grpc_server_get_channel_args(state->server), args_to_add,
+      GPR_ARRAY_SIZE(args_to_add));
   grpc_server_setup_transport(state->server, transport, extra_filters,
                               GPR_ARRAY_SIZE(extra_filters), mdctx, args_copy);
   grpc_channel_args_destroy(args_copy);
@@ -252,9 +258,11 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   grpc_resolved_addresses_destroy(resolved);
 
   state = gpr_malloc(sizeof(*state));
+  memset(state, 0, sizeof(*state));
   state->server = server;
   state->tcp = tcp;
   state->sc = sc;
+  state->processor = creds->processor;
   state->handshaking_tcp_endpoints = NULL;
   state->is_shutdown = 0;
   gpr_mu_init(&state->mu);
diff --git a/src/core/support/host_port.c b/src/core/support/host_port.c
index 0906ebc2a3..a28f04df9c 100644
--- a/src/core/support/host_port.c
+++ b/src/core/support/host_port.c
@@ -50,7 +50,7 @@ int gpr_join_host_port(char **out, const char *host, int port) {
   }
 }
 
-void gpr_split_host_port(const char *name, char **host, char **port) {
+int gpr_split_host_port(const char *name, char **host, char **port) {
   const char *host_start;
   size_t host_len;
   const char *port_start;
@@ -63,7 +63,7 @@ void gpr_split_host_port(const char *name, char **host, char **port) {
     const char *rbracket = strchr(name, ']');
     if (rbracket == NULL) {
       /* Unmatched [ */
-      return;
+      return 0;
     }
     if (rbracket[1] == '\0') {
       /* ]<end> */
@@ -73,14 +73,14 @@ void gpr_split_host_port(const char *name, char **host, char **port) {
       port_start = rbracket + 2;
     } else {
       /* ]<invalid> */
-      return;
+      return 0;
     }
     host_start = name + 1;
     host_len = (size_t)(rbracket - host_start);
     if (memchr(host_start, ':', host_len) == NULL) {
       /* Require all bracketed hosts to contain a colon, because a hostname or
       IPv4 address should never use brackets. */
-      return;
+      return 0;
     }
   } else {
     const char *colon = strchr(name, ':');
@@ -105,4 +105,6 @@ void gpr_split_host_port(const char *name, char **host, char **port) {
   if (port_start != NULL) {
     *port = gpr_strdup(port_start);
   }
+
+  return 1;
 }
diff --git a/src/core/support/stack_lockfree.c b/src/core/support/stack_lockfree.c
index 2844330379..bc741f8c70 100644
--- a/src/core/support/stack_lockfree.c
+++ b/src/core/support/stack_lockfree.c
@@ -72,6 +72,11 @@ typedef union lockfree_node {
 struct gpr_stack_lockfree {
   lockfree_node *entries;
   lockfree_node head; /* An atomic entry describing curr head */
+
+#ifndef NDEBUG
+  /* Bitmap of pushed entries to check for double-push or pop */
+  gpr_atm pushed[(INVALID_ENTRY_INDEX+1)/(8*sizeof(gpr_atm))];
+#endif
 };
 
 gpr_stack_lockfree *gpr_stack_lockfree_create(int entries) {
@@ -86,6 +91,11 @@ gpr_stack_lockfree *gpr_stack_lockfree_create(int entries) {
   /* Clear out all entries */
   memset(stack->entries, 0, entries * sizeof(stack->entries[0]));
   memset(&stack->head, 0, sizeof(stack->head));
+#ifndef NDEBUG
+  memset(&stack->pushed, 0, sizeof(stack->pushed));
+#endif
+
+  GPR_ASSERT(sizeof(stack->entries->atm) == sizeof(stack->entries->contents));
 
   /* Point the head at reserved dummy entry */
   stack->head.contents.index = INVALID_ENTRY_INDEX;
@@ -100,17 +110,36 @@ void gpr_stack_lockfree_destroy(gpr_stack_lockfree *stack) {
 int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) {
   lockfree_node head;
   lockfree_node newhead;
+  lockfree_node curent;
+  lockfree_node newent;
 
   /* First fill in the entry's index and aba ctr for new head */
   newhead.contents.index = (gpr_uint16)entry;
   /* Also post-increment the aba_ctr */
-  newhead.contents.aba_ctr = stack->entries[entry].contents.aba_ctr++;
+  curent.atm = gpr_atm_no_barrier_load(&stack->entries[entry].atm);
+  newhead.contents.aba_ctr = ++curent.contents.aba_ctr;
+  gpr_atm_no_barrier_store(&stack->entries[entry].atm, curent.atm);
+
+#ifndef NDEBUG
+  /* Check for double push */
+  {
+    int pushed_index = entry / (8*sizeof(gpr_atm));
+    int pushed_bit = entry % (8*sizeof(gpr_atm));
+    gpr_atm old_val;
+
+    old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index],
+					   (gpr_atm)(1UL << pushed_bit));
+    GPR_ASSERT((old_val & (1UL<<pushed_bit)) == 0);
+  }
+#endif
 
   do {
     /* Atomically get the existing head value for use */
     head.atm = gpr_atm_no_barrier_load(&(stack->head.atm));
     /* Point to it */
-    stack->entries[entry].contents.index = head.contents.index;
+    newent.atm = gpr_atm_no_barrier_load(&stack->entries[entry].atm);
+    newent.contents.index = head.contents.index;
+    gpr_atm_no_barrier_store(&stack->entries[entry].atm, newent.atm);
   } while (!gpr_atm_rel_cas(&(stack->head.atm), head.atm, newhead.atm));
   /* Use rel_cas above to make sure that entry index is set properly */
   return head.contents.index == INVALID_ENTRY_INDEX;
@@ -119,6 +148,7 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) {
 int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) {
   lockfree_node head;
   lockfree_node newhead;
+
   do {
     head.atm = gpr_atm_acq_load(&(stack->head.atm));
     if (head.contents.index == INVALID_ENTRY_INDEX) {
@@ -128,5 +158,18 @@ int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) {
         gpr_atm_no_barrier_load(&(stack->entries[head.contents.index].atm));
 
   } while (!gpr_atm_no_barrier_cas(&(stack->head.atm), head.atm, newhead.atm));
+#ifndef NDEBUG
+  /* Check for valid pop */
+  {
+    int pushed_index = head.contents.index / (8*sizeof(gpr_atm));
+    int pushed_bit = head.contents.index % (8*sizeof(gpr_atm));
+    gpr_atm old_val;
+
+    old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index],
+					   -(gpr_atm)(1UL << pushed_bit));
+    GPR_ASSERT((old_val & (1UL<<pushed_bit)) != 0);
+  }
+#endif
+
   return head.contents.index;
 }
diff --git a/src/core/surface/call.c b/src/core/surface/call.c
index 0e97110e5e..a7624fd96f 100644
--- a/src/core/surface/call.c
+++ b/src/core/surface/call.c
@@ -41,7 +41,6 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/census/grpc_context.h"
 #include "src/core/channel/channel_stack.h"
 #include "src/core/iomgr/alarm.h"
 #include "src/core/profiling/timers.h"
@@ -145,6 +144,8 @@ typedef enum {
 struct grpc_call {
   grpc_completion_queue *cq;
   grpc_channel *channel;
+  grpc_call *parent;
+  grpc_call *first_child;
   grpc_mdctx *metadata_context;
   /* TODO(ctiller): share with cq if possible? */
   gpr_mu mu;
@@ -178,6 +179,8 @@ struct grpc_call {
   gpr_uint8 cancel_alarm;
   /** bitmask of allocated completion events in completions */
   gpr_uint8 allocated_completions;
+  /** flag indicating that cancellation is inherited */
+  gpr_uint8 cancellation_is_inherited;
 
   /* flags with bits corresponding to write states allowing us to determine
      what was sent */
@@ -272,6 +275,11 @@ struct grpc_call {
 
   /** completion events - for completion queue use */
   grpc_cq_completion completions[MAX_CONCURRENT_COMPLETIONS];
+
+  /** siblings: children of the same parent form a list, and this list is protected under
+      parent->mu */
+  grpc_call *sibling_next;
+  grpc_call *sibling_prev;
 };
 
 #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1))
@@ -295,7 +303,9 @@ static void finished_loose_op(void *call, int success);
 static void lock(grpc_call *call);
 static void unlock(grpc_call *call);
 
-grpc_call *grpc_call_create(grpc_channel *channel, grpc_completion_queue *cq,
+grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
+                            gpr_uint32 propagation_mask,
+                            grpc_completion_queue *cq,
                             const void *server_transport_data,
                             grpc_mdelem **add_initial_metadata,
                             size_t add_initial_metadata_count,
@@ -311,9 +321,10 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_completion_queue *cq,
   gpr_mu_init(&call->completion_mu);
   call->channel = channel;
   call->cq = cq;
-  if (cq) {
+  if (cq != NULL) {
     GRPC_CQ_INTERNAL_REF(cq, "bind");
   }
+  call->parent = parent_call;
   call->is_client = server_transport_data == NULL;
   for (i = 0; i < GRPC_IOREQ_OP_COUNT; i++) {
     call->request_set[i] = REQSET_EMPTY;
@@ -352,7 +363,48 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_completion_queue *cq,
   }
   grpc_call_stack_init(channel_stack, server_transport_data, initial_op_ptr,
                        CALL_STACK_FROM_CALL(call));
-  if (gpr_time_cmp(send_deadline, gpr_inf_future(send_deadline.clock_type)) != 0) {
+  if (parent_call != NULL) {
+    GRPC_CALL_INTERNAL_REF(parent_call, "child");
+    GPR_ASSERT(call->is_client);
+    GPR_ASSERT(!parent_call->is_client);
+
+    gpr_mu_lock(&parent_call->mu);
+
+    if (propagation_mask & GRPC_PROPAGATE_DEADLINE) {
+      send_deadline = gpr_time_min(
+          gpr_convert_clock_type(send_deadline,
+                                 parent_call->send_deadline.clock_type),
+          parent_call->send_deadline);
+    }
+    /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
+     * GRPC_PROPAGATE_STATS_CONTEXT */
+    /* TODO(ctiller): This should change to use the appropriate census start_op
+     * call. */
+    if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
+      GPR_ASSERT(propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT);
+      grpc_call_context_set(call, GRPC_CONTEXT_TRACING,
+                            parent_call->context[GRPC_CONTEXT_TRACING].value,
+                            NULL);
+    } else {
+      GPR_ASSERT(propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT);
+    }
+    if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
+      call->cancellation_is_inherited = 1;
+    }
+
+    if (parent_call->first_child == NULL) {
+      parent_call->first_child = call;
+      call->sibling_next = call->sibling_prev = call;
+    } else {
+      call->sibling_next = parent_call->first_child;
+      call->sibling_prev = parent_call->first_child->sibling_prev;
+      call->sibling_next->sibling_prev = call->sibling_prev->sibling_next = call;
+    }
+
+    gpr_mu_unlock(&parent_call->mu);
+  }
+  if (gpr_time_cmp(send_deadline, gpr_inf_future(send_deadline.clock_type)) !=
+      0) {
     set_deadline_alarm(call, send_deadline);
   }
   return call;
@@ -911,6 +963,8 @@ static int add_slice_to_message(grpc_call *call, gpr_slice slice) {
 
 static void call_on_done_recv(void *pc, int success) {
   grpc_call *call = pc;
+  grpc_call *child_call;
+  grpc_call *next_child_call;
   size_t i;
   GRPC_TIMER_BEGIN(GRPC_PTAG_CALL_ON_DONE_RECV, 0);
   lock(call);
@@ -944,6 +998,19 @@ static void call_on_done_recv(void *pc, int success) {
       GPR_ASSERT(call->read_state <= READ_STATE_STREAM_CLOSED);
       call->read_state = READ_STATE_STREAM_CLOSED;
       call->cancel_alarm |= call->have_alarm;
+      /* propagate cancellation to any interested children */
+      child_call = call->first_child;
+      if (child_call != NULL) {
+        do {
+          next_child_call = child_call->sibling_next;
+          if (child_call->cancellation_is_inherited) {
+            GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel");
+            grpc_call_cancel(child_call);
+            GRPC_CALL_INTERNAL_UNREF(child_call, "propagate_cancel", 0);
+          }
+          child_call = next_child_call;
+        } while (child_call != call->first_child);
+      }
       GRPC_CALL_INTERNAL_UNREF(call, "closed", 0);
     }
     finish_read_ops(call);
@@ -973,7 +1040,7 @@ static int prepare_application_metadata(grpc_call *call, size_t count,
     GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data));
     l->md = grpc_mdelem_from_string_and_buffer(call->metadata_context, md->key,
                                                (const gpr_uint8 *)md->value,
-                                               md->value_length);
+                                               md->value_length, 1);
     if (!grpc_mdstr_is_legal_header(l->md->key)) {
       gpr_log(GPR_ERROR, "attempt to send invalid metadata key");
       return 0;
@@ -1217,6 +1284,22 @@ grpc_call_error grpc_call_start_ioreq_and_call_back(
 
 void grpc_call_destroy(grpc_call *c) {
   int cancel;
+  grpc_call *parent = c->parent;
+
+  if (parent) {
+    gpr_mu_lock(&parent->mu);
+    if (c == parent->first_child) {
+      parent->first_child = c->sibling_next;
+      if (c == parent->first_child) {
+        parent->first_child = NULL;
+      }
+      c->sibling_prev->sibling_next = c->sibling_next;
+      c->sibling_next->sibling_prev = c->sibling_prev;
+    }
+    gpr_mu_unlock(&parent->mu);
+    GRPC_CALL_INTERNAL_UNREF(parent, "child", 1);
+  }
+
   lock(c);
   GPR_ASSERT(!c->destroy_called);
   c->destroy_called = 1;
@@ -1244,7 +1327,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
 static grpc_call_error cancel_with_status(grpc_call *c, grpc_status_code status,
                                           const char *description) {
   grpc_mdstr *details =
-      description ? grpc_mdstr_from_string(c->metadata_context, description)
+      description ? grpc_mdstr_from_string(c->metadata_context, description, 0)
                   : NULL;
 
   GPR_ASSERT(status != GRPC_STATUS_OK);
@@ -1294,6 +1377,11 @@ static void execute_op(grpc_call *call, grpc_transport_stream_op *op) {
   elem->filter->start_transport_stream_op(elem, op);
 }
 
+char *grpc_call_get_peer(grpc_call *call) {
+  grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0);
+  return elem->filter->get_peer(elem);
+}
+
 grpc_call *grpc_call_from_top_element(grpc_call_element *elem) {
   return CALL_FROM_TOP_ELEM(elem);
 }
@@ -1319,7 +1407,8 @@ static void set_deadline_alarm(grpc_call *call, gpr_timespec deadline) {
   }
   GRPC_CALL_INTERNAL_REF(call, "alarm");
   call->have_alarm = 1;
-  grpc_alarm_init(&call->alarm, gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), call_alarm, call,
+  call->send_deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
+  grpc_alarm_init(&call->alarm, call->send_deadline, call_alarm, call,
                   gpr_now(GPR_CLOCK_MONOTONIC));
 }
 
@@ -1364,8 +1453,9 @@ static gpr_uint32 decode_compression(grpc_mdelem *md) {
       gpr_log(GPR_ERROR, "Invalid compression algorithm: '%s'", md_c_str);
       assert(0);
     }
-    grpc_mdelem_set_user_data(md, destroy_compression,
-                              (void *)(gpr_intptr)(algorithm + COMPRESS_OFFSET));
+    grpc_mdelem_set_user_data(
+        md, destroy_compression,
+        (void *)(gpr_intptr)(algorithm + COMPRESS_OFFSET));
   }
   return algorithm;
 }
@@ -1414,7 +1504,9 @@ static void recv_metadata(grpc_call *call, grpc_metadata_batch *md) {
       l->md = 0;
     }
   }
-  if (gpr_time_cmp(md->deadline, gpr_inf_future(GPR_CLOCK_REALTIME)) != 0) {
+  if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) !=
+          0 &&
+      !call->is_client) {
     set_deadline_alarm(call, md->deadline);
   }
   if (!is_trailing) {
@@ -1502,6 +1594,9 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
         if (!are_write_flags_valid(op->flags)) {
           return GRPC_CALL_ERROR_INVALID_FLAGS;
         }
+        if (op->data.send_message == NULL) {
+          return GRPC_CALL_ERROR_INVALID_MESSAGE;
+        }
         req = &reqs[out++];
         req->op = GRPC_IOREQ_SEND_MESSAGE;
         req->data.send_message = op->data.send_message;
@@ -1537,7 +1632,7 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
             op->data.send_status_from_server.status_details != NULL
                 ? grpc_mdstr_from_string(
                       call->metadata_context,
-                      op->data.send_status_from_server.status_details)
+                      op->data.send_status_from_server.status_details, 0)
                 : NULL;
         req = &reqs[out++];
         req->op = GRPC_IOREQ_SEND_CLOSE;
@@ -1551,6 +1646,7 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
         req = &reqs[out++];
         req->op = GRPC_IOREQ_RECV_INITIAL_METADATA;
         req->data.recv_metadata = op->data.recv_initial_metadata;
+        req->data.recv_metadata->count = 0;
         req->flags = op->flags;
         break;
       case GRPC_OP_RECV_MESSAGE:
@@ -1582,6 +1678,7 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
         req->op = GRPC_IOREQ_RECV_TRAILING_METADATA;
         req->data.recv_metadata =
             op->data.recv_status_on_client.trailing_metadata;
+        req->data.recv_metadata->count = 0;
         req = &reqs[out++];
         req->op = GRPC_IOREQ_RECV_CLOSE;
         finish_func = finish_batch_with_close;
diff --git a/src/core/surface/call.h b/src/core/surface/call.h
index 5641561f85..00638e43b5 100644
--- a/src/core/surface/call.h
+++ b/src/core/surface/call.h
@@ -89,7 +89,9 @@ typedef struct {
 typedef void (*grpc_ioreq_completion_func)(grpc_call *call, int success,
                                            void *user_data);
 
-grpc_call *grpc_call_create(grpc_channel *channel, grpc_completion_queue *cq,
+grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
+                            gpr_uint32 propagation_mask,
+                            grpc_completion_queue *cq,
                             const void *server_transport_data,
                             grpc_mdelem **add_initial_metadata,
                             size_t add_initial_metadata_count,
diff --git a/src/core/surface/channel.c b/src/core/surface/channel.c
index cd71e03b19..b80398858d 100644
--- a/src/core/surface/channel.c
+++ b/src/core/surface/channel.c
@@ -36,12 +36,14 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
 #include "src/core/iomgr/iomgr.h"
 #include "src/core/support/string.h"
 #include "src/core/surface/call.h"
 #include "src/core/surface/init.h"
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
 
 /** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS.
  *  Avoids needing to take a metadata context lock for sending status
@@ -74,6 +76,7 @@ struct grpc_channel {
   gpr_mu registered_call_mu;
   registered_call *registered_calls;
   grpc_iomgr_closure destroy_closure;
+  char *target;
 };
 
 #define CHANNEL_STACK_FROM_CHANNEL(c) ((grpc_channel_stack *)((c) + 1))
@@ -86,33 +89,35 @@ struct grpc_channel {
 #define DEFAULT_MAX_MESSAGE_LENGTH (100 * 1024 * 1024)
 
 grpc_channel *grpc_channel_create_from_filters(
-    const grpc_channel_filter **filters, size_t num_filters,
+    const char *target, const grpc_channel_filter **filters, size_t num_filters,
     const grpc_channel_args *args, grpc_mdctx *mdctx, int is_client) {
   size_t i;
   size_t size =
       sizeof(grpc_channel) + grpc_channel_stack_size(filters, num_filters);
   grpc_channel *channel = gpr_malloc(size);
   memset(channel, 0, sizeof(*channel));
+  channel->target = gpr_strdup(target);
   GPR_ASSERT(grpc_is_initialized() && "call grpc_init()");
   channel->is_client = is_client;
   /* decremented by grpc_channel_destroy */
   gpr_ref_init(&channel->refs, 1);
   channel->metadata_context = mdctx;
-  channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status");
+  channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status", 0);
   channel->grpc_compression_algorithm_string =
-      grpc_mdstr_from_string(mdctx, "grpc-encoding");
+      grpc_mdstr_from_string(mdctx, "grpc-encoding", 0);
   channel->grpc_encodings_accepted_by_peer_string =
-      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding");
-  channel->grpc_message_string = grpc_mdstr_from_string(mdctx, "grpc-message");
+      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding", 0);
+  channel->grpc_message_string =
+      grpc_mdstr_from_string(mdctx, "grpc-message", 0);
   for (i = 0; i < NUM_CACHED_STATUS_ELEMS; i++) {
     char buf[GPR_LTOA_MIN_BUFSIZE];
     gpr_ltoa(i, buf);
     channel->grpc_status_elem[i] = grpc_mdelem_from_metadata_strings(
         mdctx, GRPC_MDSTR_REF(channel->grpc_status_string),
-        grpc_mdstr_from_string(mdctx, buf));
+        grpc_mdstr_from_string(mdctx, buf, 0));
   }
-  channel->path_string = grpc_mdstr_from_string(mdctx, ":path");
-  channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority");
+  channel->path_string = grpc_mdstr_from_string(mdctx, ":path", 0);
+  channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority", 0);
   gpr_mu_init(&channel->registered_call_mu);
   channel->registered_calls = NULL;
 
@@ -140,32 +145,43 @@ grpc_channel *grpc_channel_create_from_filters(
   return channel;
 }
 
+char *grpc_channel_get_target(grpc_channel *channel) {
+  return gpr_strdup(channel->target);
+}
+
 static grpc_call *grpc_channel_create_call_internal(
-    grpc_channel *channel, grpc_completion_queue *cq, grpc_mdelem *path_mdelem,
+    grpc_channel *channel, grpc_call *parent_call, gpr_uint32 propagation_mask,
+    grpc_completion_queue *cq, grpc_mdelem *path_mdelem,
     grpc_mdelem *authority_mdelem, gpr_timespec deadline) {
   grpc_mdelem *send_metadata[2];
+  int num_metadata = 0;
 
   GPR_ASSERT(channel->is_client);
 
-  send_metadata[0] = path_mdelem;
-  send_metadata[1] = authority_mdelem;
+  send_metadata[num_metadata++] = path_mdelem;
+  if (authority_mdelem != NULL) {
+    send_metadata[num_metadata++] = authority_mdelem;
+  }
 
-  return grpc_call_create(channel, cq, NULL, send_metadata,
-                          GPR_ARRAY_SIZE(send_metadata), deadline);
+  return grpc_call_create(channel, parent_call, propagation_mask, cq, NULL, 
+                          send_metadata, num_metadata, deadline);
 }
 
 grpc_call *grpc_channel_create_call(grpc_channel *channel,
+                                    grpc_call *parent_call,
+                                    gpr_uint32 propagation_mask,
                                     grpc_completion_queue *cq,
                                     const char *method, const char *host,
                                     gpr_timespec deadline) {
   return grpc_channel_create_call_internal(
-      channel, cq,
+      channel, parent_call, propagation_mask, cq,
       grpc_mdelem_from_metadata_strings(
           channel->metadata_context, GRPC_MDSTR_REF(channel->path_string),
-          grpc_mdstr_from_string(channel->metadata_context, method)),
+          grpc_mdstr_from_string(channel->metadata_context, method, 0)),
+      host ?
       grpc_mdelem_from_metadata_strings(
           channel->metadata_context, GRPC_MDSTR_REF(channel->authority_string),
-          grpc_mdstr_from_string(channel->metadata_context, host)),
+          grpc_mdstr_from_string(channel->metadata_context, host, 0)) : NULL,
       deadline);
 }
 
@@ -174,10 +190,10 @@ void *grpc_channel_register_call(grpc_channel *channel, const char *method,
   registered_call *rc = gpr_malloc(sizeof(registered_call));
   rc->path = grpc_mdelem_from_metadata_strings(
       channel->metadata_context, GRPC_MDSTR_REF(channel->path_string),
-      grpc_mdstr_from_string(channel->metadata_context, method));
-  rc->authority = grpc_mdelem_from_metadata_strings(
+      grpc_mdstr_from_string(channel->metadata_context, method, 0));
+  rc->authority = host ? grpc_mdelem_from_metadata_strings(
       channel->metadata_context, GRPC_MDSTR_REF(channel->authority_string),
-      grpc_mdstr_from_string(channel->metadata_context, host));
+      grpc_mdstr_from_string(channel->metadata_context, host, 0)) : NULL;
   gpr_mu_lock(&channel->registered_call_mu);
   rc->next = channel->registered_calls;
   channel->registered_calls = rc;
@@ -186,12 +202,14 @@ void *grpc_channel_register_call(grpc_channel *channel, const char *method,
 }
 
 grpc_call *grpc_channel_create_registered_call(
-    grpc_channel *channel, grpc_completion_queue *completion_queue,
-    void *registered_call_handle, gpr_timespec deadline) {
+    grpc_channel *channel, grpc_call *parent_call, gpr_uint32 propagation_mask,
+    grpc_completion_queue *completion_queue, void *registered_call_handle,
+    gpr_timespec deadline) {
   registered_call *rc = registered_call_handle;
   return grpc_channel_create_call_internal(
-      channel, completion_queue, GRPC_MDELEM_REF(rc->path),
-      GRPC_MDELEM_REF(rc->authority), deadline);
+      channel, parent_call, propagation_mask, completion_queue, 
+      GRPC_MDELEM_REF(rc->path), 
+      rc->authority ? GRPC_MDELEM_REF(rc->authority) : NULL, deadline);
 }
 
 #ifdef GRPC_CHANNEL_REF_COUNT_DEBUG
@@ -221,11 +239,14 @@ static void destroy_channel(void *p, int ok) {
     registered_call *rc = channel->registered_calls;
     channel->registered_calls = rc->next;
     GRPC_MDELEM_UNREF(rc->path);
-    GRPC_MDELEM_UNREF(rc->authority);
+    if (rc->authority) {
+      GRPC_MDELEM_UNREF(rc->authority);
+    }
     gpr_free(rc);
   }
   grpc_mdctx_unref(channel->metadata_context);
   gpr_mu_destroy(&channel->registered_call_mu);
+  gpr_free(channel->target);
   gpr_free(channel);
 }
 
@@ -284,7 +305,7 @@ grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) {
     gpr_ltoa(i, tmp);
     return grpc_mdelem_from_metadata_strings(
         channel->metadata_context, GRPC_MDSTR_REF(channel->grpc_status_string),
-        grpc_mdstr_from_string(channel->metadata_context, tmp));
+        grpc_mdstr_from_string(channel->metadata_context, tmp, 0));
   }
 }
 
diff --git a/src/core/surface/channel.h b/src/core/surface/channel.h
index 1d1550bbe7..f271616f60 100644
--- a/src/core/surface/channel.h
+++ b/src/core/surface/channel.h
@@ -38,7 +38,7 @@
 #include "src/core/client_config/subchannel_factory.h"
 
 grpc_channel *grpc_channel_create_from_filters(
-    const grpc_channel_filter **filters, size_t count,
+    const char *target, const grpc_channel_filter **filters, size_t count,
     const grpc_channel_args *args, grpc_mdctx *mdctx, int is_client);
 
 /** Get a (borrowed) pointer to this channels underlying channel stack */
diff --git a/src/core/surface/channel_connectivity.c b/src/core/surface/channel_connectivity.c
new file mode 100644
index 0000000000..1223706457
--- /dev/null
+++ b/src/core/surface/channel_connectivity.c
@@ -0,0 +1,185 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/surface/channel.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/channel/client_channel.h"
+#include "src/core/iomgr/alarm.h"
+#include "src/core/surface/completion_queue.h"
+
+grpc_connectivity_state grpc_channel_check_connectivity_state(
+    grpc_channel *channel, int try_to_connect) {
+  /* forward through to the underlying client channel */
+  grpc_channel_element *client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
+  if (client_channel_elem->filter != &grpc_client_channel_filter) {
+    gpr_log(GPR_ERROR,
+            "grpc_channel_check_connectivity_state called on something that is "
+            "not a client channel, but '%s'",
+            client_channel_elem->filter->name);
+    return GRPC_CHANNEL_FATAL_FAILURE;
+  }
+  return grpc_client_channel_check_connectivity_state(client_channel_elem,
+                                                      try_to_connect);
+}
+
+typedef enum {
+  WAITING,
+  CALLING_BACK,
+  CALLING_BACK_AND_FINISHED,
+  CALLED_BACK
+} callback_phase;
+
+typedef struct {
+  gpr_mu mu;
+  callback_phase phase;
+  int success;
+  grpc_iomgr_closure on_complete;
+  grpc_alarm alarm;
+  grpc_connectivity_state state;
+  grpc_completion_queue *cq;
+  grpc_cq_completion completion_storage;
+  grpc_channel *channel;
+  void *tag;
+} state_watcher;
+
+static void delete_state_watcher(state_watcher *w) {
+  grpc_channel_element *client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(w->channel));
+  grpc_client_channel_del_interested_party(client_channel_elem, grpc_cq_pollset(w->cq));
+  GRPC_CHANNEL_INTERNAL_UNREF(w->channel, "watch_connectivity");
+  gpr_mu_destroy(&w->mu);
+  gpr_free(w);
+}
+
+static void finished_completion(void *pw, grpc_cq_completion *ignored) {
+  int delete = 0;
+  state_watcher *w = pw;
+  gpr_mu_lock(&w->mu);
+  switch (w->phase) {
+    case WAITING:
+    case CALLED_BACK:
+      gpr_log(GPR_ERROR, "should never reach here");
+      abort();
+      break;
+    case CALLING_BACK:
+      w->phase = CALLED_BACK;
+      break;
+    case CALLING_BACK_AND_FINISHED:
+      delete = 1;
+      break;
+  }
+  gpr_mu_unlock(&w->mu);
+
+  if (delete) {
+    delete_state_watcher(w);
+  }
+}
+
+static void partly_done(state_watcher *w, int due_to_completion) {
+  int delete = 0;
+
+  if (due_to_completion) {
+    gpr_mu_lock(&w->mu);
+    w->success = 1;
+    gpr_mu_unlock(&w->mu);
+    grpc_alarm_cancel(&w->alarm);
+  }
+
+  gpr_mu_lock(&w->mu);
+  switch (w->phase) {
+    case WAITING:
+      w->phase = CALLING_BACK;
+      grpc_cq_end_op(w->cq, w->tag, w->success, finished_completion, w,
+                     &w->completion_storage);
+      break;
+    case CALLING_BACK:
+      w->phase = CALLING_BACK_AND_FINISHED;
+      break;
+    case CALLING_BACK_AND_FINISHED:
+      gpr_log(GPR_ERROR, "should never reach here");
+      abort();
+      break;
+    case CALLED_BACK:
+      delete = 1;
+      break;
+  }
+  gpr_mu_unlock(&w->mu);
+
+  if (delete) {
+    delete_state_watcher(w);
+  }
+}
+
+static void watch_complete(void *pw, int success) { partly_done(pw, 1); }
+
+static void timeout_complete(void *pw, int success) { partly_done(pw, 0); }
+
+void grpc_channel_watch_connectivity_state(
+    grpc_channel *channel, grpc_connectivity_state last_observed_state,
+    gpr_timespec deadline, grpc_completion_queue *cq, void *tag) {
+  grpc_channel_element *client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
+  state_watcher *w = gpr_malloc(sizeof(*w));
+
+  grpc_cq_begin_op(cq);
+
+  gpr_mu_init(&w->mu);
+  grpc_iomgr_closure_init(&w->on_complete, watch_complete, w);
+  w->phase = WAITING;
+  w->state = last_observed_state;
+  w->success = 0;
+  w->cq = cq;
+  w->tag = tag;
+  w->channel = channel;
+
+  grpc_alarm_init(
+      &w->alarm, gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), 
+      timeout_complete, w, gpr_now(GPR_CLOCK_MONOTONIC));
+
+  if (client_channel_elem->filter != &grpc_client_channel_filter) {
+    gpr_log(GPR_ERROR,
+            "grpc_channel_watch_connectivity_state called on something that is "
+            "not a client channel, but '%s'",
+            client_channel_elem->filter->name);
+    grpc_iomgr_add_delayed_callback(&w->on_complete, 1);
+  } else {
+    GRPC_CHANNEL_INTERNAL_REF(channel, "watch_connectivity");
+    grpc_client_channel_add_interested_party(client_channel_elem, grpc_cq_pollset(cq));
+    grpc_client_channel_watch_connectivity_state(client_channel_elem, &w->state,
+                                                 &w->on_complete);
+  }
+}
diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c
index 91c7b35550..707d615688 100644
--- a/src/core/surface/channel_create.c
+++ b/src/core/surface/channel_create.c
@@ -109,6 +109,7 @@ typedef struct {
   gpr_refcount refs;
   grpc_mdctx *mdctx;
   grpc_channel_args *merge_args;
+  grpc_channel *master;
 } subchannel_factory;
 
 static void subchannel_factory_ref(grpc_subchannel_factory *scf) {
@@ -119,6 +120,7 @@ static void subchannel_factory_ref(grpc_subchannel_factory *scf) {
 static void subchannel_factory_unref(grpc_subchannel_factory *scf) {
   subchannel_factory *f = (subchannel_factory *)scf;
   if (gpr_unref(&f->refs)) {
+    GRPC_CHANNEL_INTERNAL_UNREF(f->master, "subchannel_factory");
     grpc_channel_args_destroy(f->merge_args);
     grpc_mdctx_unref(f->mdctx);
     gpr_free(f);
@@ -137,6 +139,7 @@ static grpc_subchannel *subchannel_factory_create_subchannel(
   gpr_ref_init(&c->refs, 1);
   args->mdctx = f->mdctx;
   args->args = final_args;
+  args->master = f->master;
   s = grpc_subchannel_create(&c->base, args);
   grpc_connector_unref(&c->base);
   grpc_channel_args_destroy(final_args);
@@ -151,8 +154,8 @@ static const grpc_subchannel_factory_vtable subchannel_factory_vtable = {
    Asynchronously: - resolve target
                    - connect to it (trying alternatives as presented)
                    - perform handshakes */
-grpc_channel *grpc_channel_create(const char *target,
-                                  const grpc_channel_args *args) {
+grpc_channel *grpc_insecure_channel_create(const char *target,
+                                           const grpc_channel_args *args) {
   grpc_channel *channel = NULL;
 #define MAX_FILTERS 3
   const grpc_channel_filter *filters[MAX_FILTERS];
@@ -168,18 +171,22 @@ grpc_channel *grpc_channel_create(const char *target,
   filters[n++] = &grpc_client_channel_filter;
   GPR_ASSERT(n <= MAX_FILTERS);
 
+  channel =
+      grpc_channel_create_from_filters(target, filters, n, args, mdctx, 1);
+
   f = gpr_malloc(sizeof(*f));
   f->base.vtable = &subchannel_factory_vtable;
   gpr_ref_init(&f->refs, 1);
   grpc_mdctx_ref(mdctx);
   f->mdctx = mdctx;
   f->merge_args = grpc_channel_args_copy(args);
+  f->master = channel;
+  GRPC_CHANNEL_INTERNAL_REF(f->master, "subchannel_factory");
   resolver = grpc_resolver_create(target, &f->base);
   if (!resolver) {
     return NULL;
   }
 
-  channel = grpc_channel_create_from_filters(filters, n, args, mdctx, 1);
   grpc_client_channel_set_resolver(grpc_channel_get_channel_stack(channel),
                                    resolver);
   GRPC_RESOLVER_UNREF(resolver, "create");
diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c
index 3f60b0b0ba..36d69cfe5f 100644
--- a/src/core/surface/completion_queue.c
+++ b/src/core/surface/completion_queue.c
@@ -45,6 +45,11 @@
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
 
+typedef struct {
+  grpc_pollset_worker *worker;
+  void *tag;
+} plucker;
+
 /* Completion queue structure */
 struct grpc_completion_queue {
   /** completed events */
@@ -60,6 +65,8 @@ struct grpc_completion_queue {
   int shutdown;
   int shutdown_called;
   int is_server_cq;
+  int num_pluckers;
+  plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
 };
 
 grpc_completion_queue *grpc_completion_queue_create(void) {
@@ -107,6 +114,11 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) {
 }
 
 void grpc_cq_begin_op(grpc_completion_queue *cc) {
+#ifndef NDEBUG
+  gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
+  GPR_ASSERT(!cc->shutdown_called);
+  gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
+#endif
   gpr_ref(&cc->pending_events);
 }
 
@@ -117,6 +129,8 @@ void grpc_cq_end_op(grpc_completion_queue *cc, void *tag, int success,
                     void (*done)(void *done_arg, grpc_cq_completion *storage),
                     void *done_arg, grpc_cq_completion *storage) {
   int shutdown;
+  int i;
+  grpc_pollset_worker *pluck_worker;
 
   storage->tag = tag;
   storage->done = done;
@@ -130,7 +144,14 @@ void grpc_cq_end_op(grpc_completion_queue *cc, void *tag, int success,
     cc->completed_tail->next =
         ((gpr_uintptr)storage) | (1u & (gpr_uintptr)cc->completed_tail->next);
     cc->completed_tail = storage;
-    grpc_pollset_kick(&cc->pollset);
+    pluck_worker = NULL;
+    for (i = 0; i < cc->num_pluckers; i++) {
+      if (cc->pluckers[i].tag == tag) {
+        pluck_worker = cc->pluckers[i].worker;
+        break;
+      }
+    }
+    grpc_pollset_kick(&cc->pollset, pluck_worker);
     gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
   } else {
     cc->completed_tail->next =
@@ -147,6 +168,7 @@ void grpc_cq_end_op(grpc_completion_queue *cc, void *tag, int success,
 grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
                                       gpr_timespec deadline) {
   grpc_event ret;
+  grpc_pollset_worker worker;
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
 
@@ -172,7 +194,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
       ret.type = GRPC_QUEUE_SHUTDOWN;
       break;
     }
-    if (!grpc_pollset_work(&cc->pollset, deadline)) {
+    if (!grpc_pollset_work(&cc->pollset, &worker, deadline)) {
       gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
       memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
@@ -184,11 +206,37 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
   return ret;
 }
 
+static int add_plucker(grpc_completion_queue *cc, void *tag,
+                       grpc_pollset_worker *worker) {
+  if (cc->num_pluckers == GRPC_MAX_COMPLETION_QUEUE_PLUCKERS) {
+    return 0;
+  }
+  cc->pluckers[cc->num_pluckers].tag = tag;
+  cc->pluckers[cc->num_pluckers].worker = worker;
+  cc->num_pluckers++;
+  return 1;
+}
+
+static void del_plucker(grpc_completion_queue *cc, void *tag,
+                        grpc_pollset_worker *worker) {
+  int i;
+  for (i = 0; i < cc->num_pluckers; i++) {
+    if (cc->pluckers[i].tag == tag && cc->pluckers[i].worker == worker) {
+      cc->num_pluckers--;
+      GPR_SWAP(plucker, cc->pluckers[i], cc->pluckers[cc->num_pluckers]);
+      return;
+    }
+  }
+  gpr_log(GPR_ERROR, "should never reach here");
+  abort();
+}
+
 grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
                                        gpr_timespec deadline) {
   grpc_event ret;
   grpc_cq_completion *c;
   grpc_cq_completion *prev;
+  grpc_pollset_worker worker;
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
 
@@ -219,12 +267,24 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
       ret.type = GRPC_QUEUE_SHUTDOWN;
       break;
     }
-    if (!grpc_pollset_work(&cc->pollset, deadline)) {
+    if (!add_plucker(cc, tag, &worker)) {
+      gpr_log(GPR_DEBUG, 
+              "Too many outstanding grpc_completion_queue_pluck calls: maximum is %d",
+              GRPC_MAX_COMPLETION_QUEUE_PLUCKERS);
       gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
       memset(&ret, 0, sizeof(ret));
+      /* TODO(ctiller): should we use a different result here */
       ret.type = GRPC_QUEUE_TIMEOUT;
       break;
     }
+    if (!grpc_pollset_work(&cc->pollset, &worker, deadline)) {
+      del_plucker(cc, tag, &worker);
+      gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
+      memset(&ret, 0, sizeof(ret));
+      ret.type = GRPC_QUEUE_TIMEOUT;
+      break;
+    }
+    del_plucker(cc, tag, &worker);
   }
 done:
   GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret);
@@ -261,15 +321,6 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
   return &cc->pollset;
 }
 
-void grpc_cq_hack_spin_pollset(grpc_completion_queue *cc) {
-  gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
-  grpc_pollset_kick(&cc->pollset);
-  grpc_pollset_work(&cc->pollset,
-                    gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
-                                 gpr_time_from_millis(100, GPR_TIMESPAN)));
-  gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
-}
-
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; }
 
 int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; }
diff --git a/src/core/surface/completion_queue.h b/src/core/surface/completion_queue.h
index f944f48d8e..8de024aaea 100644
--- a/src/core/surface/completion_queue.h
+++ b/src/core/surface/completion_queue.h
@@ -77,8 +77,6 @@ void grpc_cq_end_op(grpc_completion_queue *cc, void *tag, int success,
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
 
-void grpc_cq_hack_spin_pollset(grpc_completion_queue *cc);
-
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
 int grpc_cq_is_server_cq(grpc_completion_queue *cc);
 
diff --git a/src/core/surface/init.c b/src/core/surface/init.c
index 04e27d30ac..442bc72f21 100644
--- a/src/core/surface/init.c
+++ b/src/core/surface/init.c
@@ -39,6 +39,7 @@
 #include "src/core/channel/channel_stack.h"
 #include "src/core/client_config/resolver_registry.h"
 #include "src/core/client_config/resolvers/dns_resolver.h"
+#include "src/core/client_config/resolvers/sockaddr_resolver.h"
 #include "src/core/debug/trace.h"
 #include "src/core/iomgr/iomgr.h"
 #include "src/core/profiling/timers.h"
@@ -46,10 +47,7 @@
 #include "src/core/surface/init.h"
 #include "src/core/surface/surface_trace.h"
 #include "src/core/transport/chttp2_transport.h"
-
-#ifdef GPR_POSIX_SOCKET
-#include "src/core/client_config/resolvers/unix_resolver_posix.h"
-#endif
+#include "src/core/transport/connectivity_state.h"
 
 static gpr_once g_basic_init = GPR_ONCE_INIT;
 static gpr_mu g_init_mu;
@@ -68,6 +66,8 @@ void grpc_init(void) {
     gpr_time_init();
     grpc_resolver_registry_init("dns:///");
     grpc_register_resolver_type("dns", grpc_dns_resolver_factory_create());
+    grpc_register_resolver_type("ipv4", grpc_ipv4_resolver_factory_create());
+    grpc_register_resolver_type("ipv6", grpc_ipv6_resolver_factory_create());
 #ifdef GPR_POSIX_SOCKET
     grpc_register_resolver_type("unix", grpc_unix_resolver_factory_create());
 #endif
@@ -76,11 +76,15 @@ void grpc_init(void) {
     grpc_register_tracer("http", &grpc_http_trace);
     grpc_register_tracer("flowctl", &grpc_flowctl_trace);
     grpc_register_tracer("batch", &grpc_trace_batch);
+    grpc_register_tracer("connectivity_state", &grpc_connectivity_state_trace);
     grpc_security_pre_init();
     grpc_iomgr_init();
     grpc_tracer_init("GRPC_TRACE");
-    if (census_initialize(CENSUS_NONE)) {
-      gpr_log(GPR_ERROR, "Could not initialize census.");
+    /* Only initialize census if noone else has. */
+    if (census_enabled() == CENSUS_FEATURE_NONE) {
+      if (census_initialize(census_supported())) { /* enable all features. */
+        gpr_log(GPR_ERROR, "Could not initialize census.");
+      }
     }
     grpc_timers_global_init();
   }
diff --git a/src/core/surface/lame_client.c b/src/core/surface/lame_client.c
index 3f2bb5c8a9..c4215a2cfb 100644
--- a/src/core/surface/lame_client.c
+++ b/src/core/surface/lame_client.c
@@ -47,7 +47,10 @@ typedef struct {
   grpc_linked_mdelem details;
 } call_data;
 
-typedef struct { grpc_mdctx *mdctx; } channel_data;
+typedef struct {
+  grpc_mdctx *mdctx;
+  grpc_channel *master;
+} channel_data;
 
 static void lame_start_transport_stream_op(grpc_call_element *elem,
                                            grpc_transport_stream_op *op) {
@@ -82,6 +85,11 @@ static void lame_start_transport_stream_op(grpc_call_element *elem,
   }
 }
 
+static char *lame_get_peer(grpc_call_element *elem) {
+  channel_data *chand = elem->channel_data;
+  return grpc_channel_get_target(chand->master);
+}
+
 static void lame_start_transport_op(grpc_channel_element *elem,
                                     grpc_transport_op *op) {
   if (op->on_connectivity_state_change) {
@@ -112,6 +120,7 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
   GPR_ASSERT(is_first);
   GPR_ASSERT(is_last);
   chand->mdctx = mdctx;
+  chand->master = master;
 }
 
 static void destroy_channel_elem(grpc_channel_element *elem) {}
@@ -125,11 +134,12 @@ static const grpc_channel_filter lame_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
+    lame_get_peer,
     "lame-client",
 };
 
-grpc_channel *grpc_lame_client_channel_create(void) {
+grpc_channel *grpc_lame_client_channel_create(const char *target) {
   static const grpc_channel_filter *filters[] = {&lame_filter};
-  return grpc_channel_create_from_filters(filters, 1, NULL, grpc_mdctx_create(),
-                                          1);
+  return grpc_channel_create_from_filters(target, filters, 1, NULL,
+                                          grpc_mdctx_create(), 1);
 }
diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c
index d87ec97b53..c3150250b8 100644
--- a/src/core/surface/secure_channel_create.c
+++ b/src/core/surface/secure_channel_create.c
@@ -88,8 +88,8 @@ static void on_secure_transport_setup_done(void *arg,
         c->args.channel_args, secure_endpoint, c->args.metadata_context, 1);
     grpc_chttp2_transport_start_reading(c->result->transport, NULL, 0);
     c->result->filters = gpr_malloc(sizeof(grpc_channel_filter *) * 2);
-    c->result->filters[0] = &grpc_client_auth_filter;
-    c->result->filters[1] = &grpc_http_client_filter;
+    c->result->filters[0] = &grpc_http_client_filter;
+    c->result->filters[1] = &grpc_client_auth_filter;
     c->result->num_filters = 2;
   }
   notify = c->notify;
@@ -134,6 +134,7 @@ typedef struct {
   grpc_mdctx *mdctx;
   grpc_channel_args *merge_args;
   grpc_channel_security_connector *security_connector;
+  grpc_channel *master;
 } subchannel_factory;
 
 static void subchannel_factory_ref(grpc_subchannel_factory *scf) {
@@ -146,6 +147,7 @@ static void subchannel_factory_unref(grpc_subchannel_factory *scf) {
   if (gpr_unref(&f->refs)) {
     GRPC_SECURITY_CONNECTOR_UNREF(&f->security_connector->base,
                                   "subchannel_factory");
+    GRPC_CHANNEL_INTERNAL_UNREF(f->master, "subchannel_factory");
     grpc_channel_args_destroy(f->merge_args);
     grpc_mdctx_unref(f->mdctx);
     gpr_free(f);
@@ -165,6 +167,7 @@ static grpc_subchannel *subchannel_factory_create_subchannel(
   gpr_ref_init(&c->refs, 1);
   args->mdctx = f->mdctx;
   args->args = final_args;
+  args->master = f->master;
   s = grpc_subchannel_create(&c->base, args);
   grpc_connector_unref(&c->base);
   grpc_channel_args_destroy(final_args);
@@ -196,13 +199,13 @@ grpc_channel *grpc_secure_channel_create(grpc_credentials *creds,
 
   if (grpc_find_security_connector_in_args(args) != NULL) {
     gpr_log(GPR_ERROR, "Cannot set security context in channel args.");
-    return grpc_lame_client_channel_create();
+    return grpc_lame_client_channel_create(target);
   }
 
   if (grpc_credentials_create_security_connector(
           creds, target, args, NULL, &connector, &new_args_from_connector) !=
       GRPC_SECURITY_OK) {
-    return grpc_lame_client_channel_create();
+    return grpc_lame_client_channel_create(target);
   }
   mdctx = grpc_mdctx_create();
 
@@ -218,6 +221,9 @@ grpc_channel *grpc_secure_channel_create(grpc_credentials *creds,
   filters[n++] = &grpc_client_channel_filter;
   GPR_ASSERT(n <= MAX_FILTERS);
 
+  channel =
+      grpc_channel_create_from_filters(target, filters, n, args_copy, mdctx, 1);
+
   f = gpr_malloc(sizeof(*f));
   f->base.vtable = &subchannel_factory_vtable;
   gpr_ref_init(&f->refs, 1);
@@ -226,12 +232,13 @@ grpc_channel *grpc_secure_channel_create(grpc_credentials *creds,
   GRPC_SECURITY_CONNECTOR_REF(&connector->base, "subchannel_factory");
   f->security_connector = connector;
   f->merge_args = grpc_channel_args_copy(args_copy);
+  f->master = channel;
+  GRPC_CHANNEL_INTERNAL_REF(channel, "subchannel_factory");
   resolver = grpc_resolver_create(target, &f->base);
   if (!resolver) {
     return NULL;
   }
 
-  channel = grpc_channel_create_from_filters(filters, n, args_copy, mdctx, 1);
   grpc_client_channel_set_resolver(grpc_channel_get_channel_stack(channel),
                                    resolver);
   GRPC_RESOLVER_UNREF(resolver, "create");
diff --git a/src/core/surface/server.c b/src/core/surface/server.c
index 439452aea2..cd1dc589e1 100644
--- a/src/core/surface/server.c
+++ b/src/core/surface/server.c
@@ -327,6 +327,14 @@ static void request_matcher_zombify_all_pending_calls(
   }
 }
 
+static void request_matcher_kill_requests(grpc_server *server,
+                                          request_matcher *rm) {
+  int request_id;
+  while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
+    fail_call(server, &server->requested_calls[request_id]);
+  }
+}
+
 /*
  * server proper
  */
@@ -400,6 +408,15 @@ static void finish_start_new_rpc(grpc_server *server, grpc_call_element *elem,
   call_data *calld = elem->call_data;
   int request_id;
 
+  if (gpr_atm_acq_load(&server->shutdown_flag)) {
+    gpr_mu_lock(&calld->mu_state);
+    calld->state = ZOMBIED;
+    gpr_mu_unlock(&calld->mu_state);
+    grpc_iomgr_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
+    grpc_iomgr_add_callback(&calld->kill_zombie_closure);
+    return;
+  }
+
   request_id = gpr_stack_lockfree_pop(request_matcher->requests);
   if (request_id == -1) {
     gpr_mu_lock(&server->mu_call);
@@ -483,12 +500,25 @@ static int num_channels(grpc_server *server) {
   return n;
 }
 
+static void kill_pending_work_locked(grpc_server *server) {
+  registered_method *rm;
+  request_matcher_kill_requests(server, &server->unregistered_request_matcher);
+  request_matcher_zombify_all_pending_calls(
+      &server->unregistered_request_matcher);
+  for (rm = server->registered_methods; rm; rm = rm->next) {
+    request_matcher_kill_requests(server, &rm->request_matcher);
+    request_matcher_zombify_all_pending_calls(&rm->request_matcher);
+  }
+}
+
 static void maybe_finish_shutdown(grpc_server *server) {
   size_t i;
   if (!gpr_atm_acq_load(&server->shutdown_flag) || server->shutdown_published) {
     return;
   }
 
+  kill_pending_work_locked(server);
+
   if (server->root_channel_data.next != &server->root_channel_data ||
       server->listeners_destroyed < num_listeners(server)) {
     if (gpr_time_cmp(gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME),
@@ -530,6 +560,7 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
 static void server_on_recv(void *ptr, int success) {
   grpc_call_element *elem = ptr;
   call_data *calld = elem->call_data;
+  gpr_timespec op_deadline;
 
   if (success && !calld->got_initial_metadata) {
     size_t i;
@@ -539,12 +570,15 @@ static void server_on_recv(void *ptr, int success) {
       grpc_stream_op *op = &ops[i];
       if (op->type != GRPC_OP_METADATA) continue;
       grpc_metadata_batch_filter(&op->data.metadata, server_filter, elem);
-      if (0 != gpr_time_cmp(op->data.metadata.deadline,
-                            gpr_inf_future(GPR_CLOCK_REALTIME))) {
+      op_deadline = op->data.metadata.deadline;
+      if (0 !=
+          gpr_time_cmp(op_deadline, gpr_inf_future(op_deadline.clock_type))) {
         calld->deadline = op->data.metadata.deadline;
       }
-      calld->got_initial_metadata = 1;
-      start_new_rpc(elem);
+      if (calld->host && calld->path) {
+        calld->got_initial_metadata = 1;
+        start_new_rpc(elem);
+      }
       break;
     }
   }
@@ -610,8 +644,8 @@ static void accept_stream(void *cd, grpc_transport *transport,
                           const void *transport_server_data) {
   channel_data *chand = cd;
   /* create a call */
-  grpc_call_create(chand->channel, NULL, transport_server_data, NULL, 0,
-                   gpr_inf_future(GPR_CLOCK_REALTIME));
+  grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data, NULL,
+                   0, gpr_inf_future(GPR_CLOCK_MONOTONIC));
 }
 
 static void channel_connectivity_changed(void *cd, int iomgr_status_ignored) {
@@ -677,8 +711,8 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master,
   GPR_ASSERT(!is_last);
   chand->server = NULL;
   chand->channel = NULL;
-  chand->path_key = grpc_mdstr_from_string(metadata_context, ":path");
-  chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority");
+  chand->path_key = grpc_mdstr_from_string(metadata_context, ":path", 0);
+  chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority", 0);
   chand->next = chand->prev = chand;
   chand->registered_methods = NULL;
   chand->connectivity_state = GRPC_CHANNEL_IDLE;
@@ -722,6 +756,7 @@ static const grpc_channel_filter server_surface_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
+    grpc_call_next_get_peer,
     "server",
 };
 
@@ -878,8 +913,8 @@ void grpc_server_setup_transport(grpc_server *s, grpc_transport *transport,
     grpc_transport_perform_op(transport, &op);
   }
 
-  channel =
-      grpc_channel_create_from_filters(filters, num_filters, args, mdctx, 0);
+  channel = grpc_channel_create_from_filters(NULL, filters, num_filters, args,
+                                             mdctx, 0);
   chand = (channel_data *)grpc_channel_stack_element(
               grpc_channel_get_channel_stack(channel), 0)
               ->channel_data;
@@ -899,8 +934,8 @@ void grpc_server_setup_transport(grpc_server *s, grpc_transport *transport,
     chand->registered_methods = gpr_malloc(alloc);
     memset(chand->registered_methods, 0, alloc);
     for (rm = s->registered_methods; rm; rm = rm->next) {
-      host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host) : NULL;
-      method = grpc_mdstr_from_string(mdctx, rm->method);
+      host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host, 0) : NULL;
+      method = grpc_mdstr_from_string(mdctx, rm->method, 0);
       hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
       for (probes = 0; chand->registered_methods[(hash + probes) % slots]
                            .server_registered_method != NULL;
@@ -933,52 +968,15 @@ void grpc_server_setup_transport(grpc_server *s, grpc_transport *transport,
   op.set_accept_stream_user_data = chand;
   op.on_connectivity_state_change = &chand->channel_connectivity_changed;
   op.connectivity_state = &chand->connectivity_state;
+  op.disconnect = gpr_atm_acq_load(&s->shutdown_flag);
   grpc_transport_perform_op(transport, &op);
 }
 
-typedef struct {
-  requested_call **requests;
-  size_t count;
-  size_t capacity;
-} request_killer;
-
-static void request_killer_init(request_killer *rk) {
-  memset(rk, 0, sizeof(*rk));
-}
-
-static void request_killer_add(request_killer *rk, requested_call *rc) {
-  if (rk->capacity == rk->count) {
-    rk->capacity = GPR_MAX(8, rk->capacity * 2);
-    rk->requests =
-        gpr_realloc(rk->requests, rk->capacity * sizeof(*rk->requests));
-  }
-  rk->requests[rk->count++] = rc;
-}
-
-static void request_killer_add_request_matcher(request_killer *rk,
-                                               grpc_server *server,
-                                               request_matcher *rm) {
-  int request_id;
-  while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
-    request_killer_add(rk, &server->requested_calls[request_id]);
-  }
-}
-
-static void request_killer_run(request_killer *rk, grpc_server *server) {
-  size_t i;
-  for (i = 0; i < rk->count; i++) {
-    fail_call(server, rk->requests[i]);
-  }
-  gpr_free(rk->requests);
-}
-
 void grpc_server_shutdown_and_notify(grpc_server *server,
                                      grpc_completion_queue *cq, void *tag) {
   listener *l;
-  registered_method *rm;
   shutdown_tag *sdt;
   channel_broadcaster broadcaster;
-  request_killer reqkill;
 
   GRPC_SERVER_LOG_SHUTDOWN(GPR_INFO, server, cq, tag);
 
@@ -999,27 +997,16 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
   server->last_shutdown_message_time = gpr_now(GPR_CLOCK_REALTIME);
 
   channel_broadcaster_init(server, &broadcaster);
-  request_killer_init(&reqkill);
 
   /* collect all unregistered then registered calls */
   gpr_mu_lock(&server->mu_call);
-  request_killer_add_request_matcher(&reqkill, server,
-                                     &server->unregistered_request_matcher);
-  request_matcher_zombify_all_pending_calls(
-      &server->unregistered_request_matcher);
-  for (rm = server->registered_methods; rm; rm = rm->next) {
-    request_killer_add_request_matcher(&reqkill, server, &rm->request_matcher);
-    request_matcher_zombify_all_pending_calls(&rm->request_matcher);
-  }
+  kill_pending_work_locked(server);
   gpr_mu_unlock(&server->mu_call);
 
   gpr_atm_rel_store(&server->shutdown_flag, 1);
   maybe_finish_shutdown(server);
   gpr_mu_unlock(&server->mu_global);
 
-  /* terminate all the requested calls */
-  request_killer_run(&reqkill, server);
-
   /* Shutdown listeners */
   for (l = server->listeners; l; l = l->next) {
     l->destroy(server, l->arg);
@@ -1259,6 +1246,8 @@ static void done_request_event(void *req, grpc_cq_completion *c) {
   } else {
     gpr_free(req);
   }
+
+  server_unref(server);
 }
 
 static void fail_call(grpc_server *server, requested_call *rc) {
@@ -1271,6 +1260,7 @@ static void fail_call(grpc_server *server, requested_call *rc) {
       rc->data.registered.initial_metadata->count = 0;
       break;
   }
+  server_ref(server);
   grpc_cq_end_op(rc->cq_for_notification, rc->tag, 0, done_request_event, rc,
                  &rc->completion);
 }
@@ -1281,6 +1271,8 @@ static void publish_registered_or_batch(grpc_call *call, int success,
       grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
   requested_call *rc = prc;
   call_data *calld = elem->call_data;
+  channel_data *chand = elem->channel_data;
+  server_ref(chand->server);
   grpc_cq_end_op(calld->cq_new, rc->tag, success, done_request_event, rc,
                  &rc->completion);
   GRPC_CALL_INTERNAL_UNREF(call, "server", 0);
diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c
index 78c53466b3..4ab845bc00 100644
--- a/src/core/surface/server_chttp2.c
+++ b/src/core/surface/server_chttp2.c
@@ -80,7 +80,7 @@ static void destroy(grpc_server *server, void *tcpp) {
   grpc_tcp_server_destroy(tcp, grpc_server_listener_destroy_done, server);
 }
 
-int grpc_server_add_http2_port(grpc_server *server, const char *addr) {
+int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
   size_t i;
diff --git a/src/core/transport/chttp2/alpn.c b/src/core/transport/chttp2/alpn.c
index 3ccd5796ba..69da4e6718 100644
--- a/src/core/transport/chttp2/alpn.c
+++ b/src/core/transport/chttp2/alpn.c
@@ -36,8 +36,7 @@
 #include <grpc/support/useful.h>
 
 /* in order of preference */
-static const char *const supported_versions[] = {"h2", "h2-17", "h2-16",
-                                                 "h2-15", "h2-14"};
+static const char *const supported_versions[] = {"h2"};
 
 int grpc_chttp2_is_alpn_version_supported(const char *version, size_t size) {
   size_t i;
diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h
index e5e6f445b7..42cf0ecd5b 100644
--- a/src/core/transport/chttp2/internal.h
+++ b/src/core/transport/chttp2/internal.h
@@ -60,7 +60,6 @@ typedef enum {
   GRPC_CHTTP2_LIST_WRITABLE,
   GRPC_CHTTP2_LIST_WRITING,
   GRPC_CHTTP2_LIST_WRITTEN,
-  GRPC_CHTTP2_LIST_WRITABLE_WINDOW_UPDATE,
   GRPC_CHTTP2_LIST_PARSING_SEEN,
   GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING,
   GRPC_CHTTP2_LIST_CANCELLED_WAITING_FOR_WRITING,
@@ -120,6 +119,10 @@ typedef enum {
   GRPC_WRITE_STATE_SENT_CLOSE
 } grpc_chttp2_write_state;
 
+/* flags that can be or'd into stream_global::writing_now */
+#define GRPC_CHTTP2_WRITING_DATA 1
+#define GRPC_CHTTP2_WRITING_WINDOW 2
+
 typedef enum {
   GRPC_DONT_SEND_CLOSED = 0,
   GRPC_SEND_CLOSED,
@@ -286,6 +289,7 @@ struct grpc_chttp2_transport {
   grpc_endpoint *ep;
   grpc_mdctx *metadata_context;
   gpr_refcount refs;
+  char *peer_string;
 
   gpr_mu mu;
 
@@ -382,6 +386,10 @@ typedef struct {
   gpr_uint8 published_cancelled;
   /** is this stream in the stream map? (boolean) */
   gpr_uint8 in_stream_map;
+  /** bitmask of GRPC_CHTTP2_WRITING_xxx above */
+  gpr_uint8 writing_now;
+  /** has anything been written to this stream? */
+  gpr_uint8 written_anything;
 
   /** stream state already published to the upper layer */
   grpc_stream_state published_state;
@@ -474,11 +482,17 @@ void grpc_chttp2_publish_reads(grpc_chttp2_transport_global *global,
 void grpc_chttp2_list_add_writable_stream(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global);
+void grpc_chttp2_list_add_first_writable_stream(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global);
 int grpc_chttp2_list_pop_writable_stream(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_global **stream_global,
     grpc_chttp2_stream_writing **stream_writing);
+void grpc_chttp2_list_remove_writable_stream(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global);
 
 void grpc_chttp2_list_add_incoming_window_updated(
     grpc_chttp2_transport_global *transport_global,
@@ -510,18 +524,6 @@ int grpc_chttp2_list_pop_written_stream(
     grpc_chttp2_stream_global **stream_global,
     grpc_chttp2_stream_writing **stream_writing);
 
-void grpc_chttp2_list_add_writable_window_update_stream(
-    grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_stream_global *stream_global);
-int grpc_chttp2_list_pop_writable_window_update_stream(
-    grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_transport_writing *transport_writing,
-    grpc_chttp2_stream_global **stream_global,
-    grpc_chttp2_stream_writing **stream_writing);
-void grpc_chttp2_list_remove_writable_window_update_stream(
-    grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_stream_global *stream_global);
-
 void grpc_chttp2_list_add_parsing_seen_stream(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing);
diff --git a/src/core/transport/chttp2/parsing.c b/src/core/transport/chttp2/parsing.c
index 904b9afce7..d84960009b 100644
--- a/src/core/transport/chttp2/parsing.c
+++ b/src/core/transport/chttp2/parsing.c
@@ -182,8 +182,7 @@ void grpc_chttp2_publish_reads(
       stream_global->max_recv_bytes -= 
           stream_parsing->incoming_window_delta;
       stream_parsing->incoming_window_delta = 0;
-      grpc_chttp2_list_add_writable_window_update_stream(transport_global,
-                                                         stream_global);
+      grpc_chttp2_list_add_writable_stream(transport_global, stream_global);
     }
 
     /* update outgoing flow control window */
@@ -607,7 +606,7 @@ static void on_header(void *tp, grpc_mdelem *md) {
     }
     grpc_chttp2_incoming_metadata_buffer_set_deadline(
         &stream_parsing->incoming_metadata,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), *cached_timeout));
+        gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), *cached_timeout));
     GRPC_MDELEM_UNREF(md);
   } else {
     grpc_chttp2_incoming_metadata_buffer_add(&stream_parsing->incoming_metadata,
diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/stream_encoder.c
index 65b31a5afd..0f04169741 100644
--- a/src/core/transport/chttp2/stream_encoder.c
+++ b/src/core/transport/chttp2/stream_encoder.c
@@ -441,7 +441,7 @@ static void deadline_enc(grpc_chttp2_hpack_compressor *c, gpr_timespec deadline,
       gpr_time_sub(deadline, gpr_now(deadline.clock_type)), timeout_str);
   mdelem = grpc_mdelem_from_metadata_strings(
       c->mdctx, GRPC_MDSTR_REF(c->timeout_key_str),
-      grpc_mdstr_from_string(c->mdctx, timeout_str));
+      grpc_mdstr_from_string(c->mdctx, timeout_str, 0));
   mdelem = hpack_enc(c, mdelem, st);
   if (mdelem) GRPC_MDELEM_UNREF(mdelem);
 }
@@ -456,7 +456,7 @@ void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor *c,
                                        grpc_mdctx *ctx) {
   memset(c, 0, sizeof(*c));
   c->mdctx = ctx;
-  c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout");
+  c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout", 0);
 }
 
 void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c) {
diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c
index 590f6abfbc..9c3ad7a777 100644
--- a/src/core/transport/chttp2/stream_lists.c
+++ b/src/core/transport/chttp2/stream_lists.c
@@ -108,6 +108,23 @@ static void stream_list_maybe_remove(grpc_chttp2_transport *t,
   }
 }
 
+static void stream_list_add_head(grpc_chttp2_transport *t,
+                                 grpc_chttp2_stream *s,
+                                 grpc_chttp2_stream_list_id id) {
+  grpc_chttp2_stream *old_head;
+  GPR_ASSERT(!s->included[id]);
+  old_head = t->lists[id].head;
+  s->links[id].next = old_head;
+  s->links[id].prev = NULL;
+  if (old_head) {
+    old_head->links[id].prev = s;
+  } else {
+    t->lists[id].tail = s;
+  }
+  t->lists[id].head = s;
+  s->included[id] = 1;
+}
+
 static void stream_list_add_tail(grpc_chttp2_transport *t,
                                  grpc_chttp2_stream *s,
                                  grpc_chttp2_stream_list_id id) {
@@ -119,7 +136,6 @@ static void stream_list_add_tail(grpc_chttp2_transport *t,
   if (old_tail) {
     old_tail->links[id].next = s;
   } else {
-    s->links[id].prev = NULL;
     t->lists[id].head = s;
   }
   t->lists[id].tail = s;
@@ -144,6 +160,15 @@ void grpc_chttp2_list_add_writable_stream(
                   STREAM_FROM_GLOBAL(stream_global), GRPC_CHTTP2_LIST_WRITABLE);
 }
 
+void grpc_chttp2_list_add_first_writable_stream(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global) {
+  GPR_ASSERT(stream_global->id != 0);
+  stream_list_add_head(TRANSPORT_FROM_GLOBAL(transport_global),
+                       STREAM_FROM_GLOBAL(stream_global),
+                       GRPC_CHTTP2_LIST_WRITABLE);
+}
+
 int grpc_chttp2_list_pop_writable_stream(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_transport_writing *transport_writing,
@@ -157,6 +182,14 @@ int grpc_chttp2_list_pop_writable_stream(
   return r;
 }
 
+void grpc_chttp2_list_remove_writable_stream(
+    grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global) {
+  stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global),
+                           STREAM_FROM_GLOBAL(stream_global),
+                           GRPC_CHTTP2_LIST_WRITABLE);
+}
+
 void grpc_chttp2_list_add_writing_stream(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing) {
@@ -202,36 +235,6 @@ int grpc_chttp2_list_pop_written_stream(
   return r;
 }
 
-void grpc_chttp2_list_add_writable_window_update_stream(
-    grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_stream_global *stream_global) {
-  GPR_ASSERT(stream_global->id != 0);
-  stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global),
-                  STREAM_FROM_GLOBAL(stream_global),
-                  GRPC_CHTTP2_LIST_WRITABLE_WINDOW_UPDATE);
-}
-
-int grpc_chttp2_list_pop_writable_window_update_stream(
-    grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_transport_writing *transport_writing,
-    grpc_chttp2_stream_global **stream_global,
-    grpc_chttp2_stream_writing **stream_writing) {
-  grpc_chttp2_stream *stream;
-  int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream,
-                          GRPC_CHTTP2_LIST_WRITABLE_WINDOW_UPDATE);
-  *stream_global = &stream->global;
-  *stream_writing = &stream->writing;
-  return r;
-}
-
-void grpc_chttp2_list_remove_writable_window_update_stream(
-    grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_stream_global *stream_global) {
-  stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global),
-                           STREAM_FROM_GLOBAL(stream_global),
-                           GRPC_CHTTP2_LIST_WRITABLE_WINDOW_UPDATE);
-}
-
 void grpc_chttp2_list_add_parsing_seen_stream(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing) {
diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c
index d8ec117aa5..b55e81fdca 100644
--- a/src/core/transport/chttp2/writing.c
+++ b/src/core/transport/chttp2/writing.c
@@ -44,6 +44,7 @@ int grpc_chttp2_unlocking_check_writes(
     grpc_chttp2_transport_writing *transport_writing) {
   grpc_chttp2_stream_global *stream_global;
   grpc_chttp2_stream_writing *stream_writing;
+  grpc_chttp2_stream_global *first_reinserted_stream = NULL;
   gpr_uint32 window_delta;
 
   /* simple writes are queued to qbuf, and flushed here */
@@ -64,50 +65,53 @@ int grpc_chttp2_unlocking_check_writes(
   }
 
   /* for each grpc_chttp2_stream that's become writable, frame it's data
-     (according to
-     available window sizes) and add to the output buffer */
-  while (grpc_chttp2_list_pop_writable_stream(transport_global,
-                                              transport_writing, &stream_global,
-                                              &stream_writing)) {
+     (according to available window sizes) and add to the output buffer */
+  while (grpc_chttp2_list_pop_writable_stream(
+      transport_global, transport_writing, &stream_global, &stream_writing)) {
+    if (stream_global == first_reinserted_stream) {
+      /* prevent infinite loop */
+      grpc_chttp2_list_add_first_writable_stream(transport_global,
+                                                 stream_global);
+      break;
+    }
+
     stream_writing->id = stream_global->id;
-    window_delta = grpc_chttp2_preencode(
-        stream_global->outgoing_sopb->ops, &stream_global->outgoing_sopb->nops,
-        GPR_MIN(transport_global->outgoing_window,
-                stream_global->outgoing_window),
-        &stream_writing->sopb);
-    GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT(
-        "write", transport_global, outgoing_window, -(gpr_int64)window_delta);
-    GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("write", transport_global, stream_global,
-                                     outgoing_window, -(gpr_int64)window_delta);
-    transport_global->outgoing_window -= window_delta;
-    stream_global->outgoing_window -= window_delta;
-
-    if (stream_global->write_state == GRPC_WRITE_STATE_QUEUED_CLOSE &&
-        stream_global->outgoing_sopb->nops == 0) {
-      if (!transport_global->is_client && !stream_global->read_closed) {
-        stream_writing->send_closed = GRPC_SEND_CLOSED_WITH_RST_STREAM;
-      } else {
-        stream_writing->send_closed = GRPC_SEND_CLOSED;
+    stream_writing->send_closed = GRPC_DONT_SEND_CLOSED;
+
+    if (stream_global->outgoing_sopb) {
+      window_delta =
+          grpc_chttp2_preencode(stream_global->outgoing_sopb->ops,
+                                &stream_global->outgoing_sopb->nops,
+                                GPR_MIN(transport_global->outgoing_window,
+                                        stream_global->outgoing_window),
+                                &stream_writing->sopb);
+      GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT(
+          "write", transport_global, outgoing_window, -(gpr_int64)window_delta);
+      GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("write", transport_global, stream_global,
+                                       outgoing_window,
+                                       -(gpr_int64)window_delta);
+      transport_global->outgoing_window -= window_delta;
+      stream_global->outgoing_window -= window_delta;
+
+      if (stream_global->write_state == GRPC_WRITE_STATE_QUEUED_CLOSE &&
+          stream_global->outgoing_sopb->nops == 0) {
+        if (!transport_global->is_client && !stream_global->read_closed) {
+          stream_writing->send_closed = GRPC_SEND_CLOSED_WITH_RST_STREAM;
+        } else {
+          stream_writing->send_closed = GRPC_SEND_CLOSED;
+        }
       }
-    }
-    if (stream_writing->sopb.nops > 0 ||
-        stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) {
-      grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing);
-    }
 
-    if (stream_global->outgoing_window > 0 &&
-        stream_global->outgoing_sopb->nops != 0) {
-      grpc_chttp2_list_add_writable_stream(transport_global, stream_global);
+      if (stream_global->outgoing_window > 0 &&
+          stream_global->outgoing_sopb->nops != 0) {
+        grpc_chttp2_list_add_writable_stream(transport_global, stream_global);
+        if (first_reinserted_stream == NULL &&
+            transport_global->outgoing_window == 0) {
+          first_reinserted_stream = stream_global;
+        }
+      }
     }
-  }
 
-  /* for each grpc_chttp2_stream that wants to update its window, add that
-   * window here */
-  while (grpc_chttp2_list_pop_writable_window_update_stream(transport_global,
-                                                            transport_writing,
-                                                            &stream_global,
-                                                            &stream_writing)) {
-    stream_writing->id = stream_global->id;
     if (!stream_global->read_closed && stream_global->unannounced_incoming_window > 0) {
       stream_writing->announce_window = stream_global->unannounced_incoming_window;
       GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("write", transport_global, stream_global,
@@ -118,6 +122,13 @@ int grpc_chttp2_unlocking_check_writes(
       stream_global->unannounced_incoming_window = 0;
       grpc_chttp2_list_add_incoming_window_updated(transport_global,
                                                    stream_global);
+      stream_global->writing_now |= GRPC_CHTTP2_WRITING_WINDOW;
+    }
+    if (stream_writing->sopb.nops > 0 ||
+        stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) {
+      stream_global->writing_now |= GRPC_CHTTP2_WRITING_DATA;
+    }
+    if (stream_global->writing_now != 0) {
       grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing);
     }
   }
@@ -173,6 +184,7 @@ static void finalize_outbuf(grpc_chttp2_transport_writing *transport_writing) {
                          stream_writing->send_closed != GRPC_DONT_SEND_CLOSED,
                          stream_writing->id, &transport_writing->hpack_compressor,
                          &transport_writing->outbuf);
+      stream_writing->sopb.nops = 0;
     }
     if (stream_writing->announce_window > 0) {
       gpr_slice_buffer_add(
@@ -181,7 +193,6 @@ static void finalize_outbuf(grpc_chttp2_transport_writing *transport_writing) {
               stream_writing->id, stream_writing->announce_window));
       stream_writing->announce_window = 0;
     }
-    stream_writing->sopb.nops = 0;
     if (stream_writing->send_closed == GRPC_SEND_CLOSED_WITH_RST_STREAM) {
       gpr_slice_buffer_add(&transport_writing->outbuf,
                            grpc_chttp2_rst_stream_create(stream_writing->id,
@@ -205,20 +216,25 @@ void grpc_chttp2_cleanup_writing(
 
   while (grpc_chttp2_list_pop_written_stream(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
-    if (stream_global->outgoing_sopb != NULL &&
-        stream_global->outgoing_sopb->nops == 0) {
-      stream_global->outgoing_sopb = NULL;
-      grpc_chttp2_schedule_closure(transport_global,
-                                   stream_global->send_done_closure, 1);
-    }
+    GPR_ASSERT(stream_global->writing_now != 0);
     if (stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) {
       stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE;
       if (!transport_global->is_client) {
         stream_global->read_closed = 1;
       }
-      grpc_chttp2_list_add_read_write_state_changed(transport_global,
-                                                    stream_global);
     }
+    if (stream_global->writing_now & GRPC_CHTTP2_WRITING_DATA) {
+      if (stream_global->outgoing_sopb != NULL &&
+          stream_global->outgoing_sopb->nops == 0) {
+        GPR_ASSERT(stream_global->write_state != GRPC_WRITE_STATE_QUEUED_CLOSE);
+        stream_global->outgoing_sopb = NULL;
+        grpc_chttp2_schedule_closure(transport_global,
+                                     stream_global->send_done_closure, 1);
+      }
+    }
+    stream_global->writing_now = 0;
+    grpc_chttp2_list_add_read_write_state_changed(transport_global,
+                                                  stream_global);
   }
   transport_writing->outbuf.count = 0;
   transport_writing->outbuf.length = 0;
diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c
index c923d5e42f..a9f91b64d5 100644
--- a/src/core/transport/chttp2_transport.c
+++ b/src/core/transport/chttp2_transport.c
@@ -107,9 +107,16 @@ static void cancel_from_api(grpc_chttp2_transport_global *transport_global,
                             grpc_chttp2_stream_global *stream_global,
                             grpc_status_code status);
 
+static void close_from_api(grpc_chttp2_transport_global *transport_global,
+                           grpc_chttp2_stream_global *stream_global,
+                           grpc_status_code status,
+                           gpr_slice *optional_message);
+
 /** Add endpoint from this transport to pollset */
 static void add_to_pollset_locked(grpc_chttp2_transport *t,
                                   grpc_pollset *pollset);
+static void add_to_pollset_set_locked(grpc_chttp2_transport *t,
+                                  grpc_pollset_set *pollset_set);
 
 /** Start new streams that have been created if we can */
 static void maybe_start_some_streams(
@@ -117,7 +124,7 @@ static void maybe_start_some_streams(
 
 static void connectivity_state_set(
     grpc_chttp2_transport_global *transport_global,
-    grpc_connectivity_state state);
+    grpc_connectivity_state state, const char *reason);
 
 /*
  * CONSTRUCTION/DESTRUCTION/REFCOUNTING
@@ -168,6 +175,7 @@ static void destruct_transport(grpc_chttp2_transport *t) {
 
   grpc_mdctx_unref(t->metadata_context);
 
+  gpr_free(t->peer_string);
   gpr_free(t);
 }
 
@@ -217,6 +225,7 @@ static void init_transport(grpc_chttp2_transport *t,
   gpr_ref_init(&t->refs, 2);
   gpr_mu_init(&t->mu);
   grpc_mdctx_ref(mdctx);
+  t->peer_string = grpc_endpoint_get_peer(ep);
   t->metadata_context = mdctx;
   t->endpoint_reading = 1;
   t->global.next_stream_id = is_client ? 1 : 2;
@@ -228,12 +237,12 @@ static void init_transport(grpc_chttp2_transport *t,
   t->global.pings.next = t->global.pings.prev = &t->global.pings;
   t->parsing.is_client = is_client;
   t->parsing.str_grpc_timeout =
-      grpc_mdstr_from_string(t->metadata_context, "grpc-timeout");
+      grpc_mdstr_from_string(t->metadata_context, "grpc-timeout", 0);
   t->parsing.deframe_state =
       is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0;
   t->writing.is_client = is_client;
   grpc_connectivity_state_init(&t->channel_callback.state_tracker,
-                               GRPC_CHANNEL_READY);
+                               GRPC_CHANNEL_READY, "transport");
 
   gpr_slice_buffer_init(&t->global.qbuf);
 
@@ -327,7 +336,8 @@ static void destroy_transport(grpc_transport *gt) {
 static void close_transport_locked(grpc_chttp2_transport *t) {
   if (!t->closed) {
     t->closed = 1;
-    connectivity_state_set(&t->global, GRPC_CHANNEL_FATAL_FAILURE);
+    connectivity_state_set(&t->global, GRPC_CHANNEL_FATAL_FAILURE,
+                           "close_transport");
     if (t->ep) {
       grpc_endpoint_shutdown(t->ep);
     }
@@ -393,12 +403,16 @@ static void destroy_stream(grpc_transport *gt, grpc_stream *gs) {
   }
 
   grpc_chttp2_list_remove_incoming_window_updated(&t->global, &s->global);
-  grpc_chttp2_list_remove_writable_window_update_stream(&t->global, &s->global);
+  grpc_chttp2_list_remove_writable_stream(&t->global, &s->global);
 
   gpr_mu_unlock(&t->mu);
 
   for (i = 0; i < STREAM_LIST_COUNT; i++) {
-    GPR_ASSERT(!s->included[i]);
+    if (s->included[i]) {
+      gpr_log(GPR_ERROR, "%s stream %d still included in list %d",
+              t->global.is_client ? "client" : "server", s->global.id, i);
+      abort();
+    }
   }
 
   GPR_ASSERT(s->global.outgoing_sopb == NULL);
@@ -530,7 +544,8 @@ void grpc_chttp2_add_incoming_goaway(
   gpr_free(msg);
   gpr_slice_unref(goaway_text);
   transport_global->seen_goaway = 1;
-  connectivity_state_set(transport_global, GRPC_CHANNEL_FATAL_FAILURE);
+  connectivity_state_set(transport_global, GRPC_CHANNEL_FATAL_FAILURE,
+                         "got_goaway");
 }
 
 static void maybe_start_some_streams(
@@ -555,7 +570,8 @@ static void maybe_start_some_streams(
     transport_global->next_stream_id += 2;
 
     if (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID) {
-      connectivity_state_set(transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE);
+      connectivity_state_set(transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE,
+                             "no_more_stream_ids");
     }
 
     stream_global->outgoing_window =
@@ -574,8 +590,6 @@ static void maybe_start_some_streams(
     grpc_chttp2_list_add_incoming_window_updated(transport_global,
                                                  stream_global);
     grpc_chttp2_list_add_writable_stream(transport_global, stream_global);
-    grpc_chttp2_list_add_writable_window_update_stream(transport_global,
-                                                       stream_global);
 
   }
   /* cancel out streams that will never be started */
@@ -593,10 +607,16 @@ static void perform_stream_op_locked(
     cancel_from_api(transport_global, stream_global, op->cancel_with_status);
   }
 
+  if (op->close_with_status != GRPC_STATUS_OK) {
+    close_from_api(transport_global, stream_global, op->close_with_status,
+                   op->optional_close_message);
+  }
+
   if (op->send_ops) {
     GPR_ASSERT(stream_global->outgoing_sopb == NULL);
     stream_global->send_done_closure = op->on_done_send;
     if (!stream_global->cancelled) {
+      stream_global->written_anything = 1;
       stream_global->outgoing_sopb = op->send_ops;
       if (op->is_last_send &&
           stream_global->write_state == GRPC_WRITE_STATE_OPEN) {
@@ -641,8 +661,7 @@ static void perform_stream_op_locked(
     if (stream_global->id != 0) {
       grpc_chttp2_list_add_read_write_state_changed(transport_global,
                                                     stream_global);
-      grpc_chttp2_list_add_writable_window_update_stream(transport_global,
-                                                         stream_global);
+      grpc_chttp2_list_add_writable_stream(transport_global, stream_global);
     }
   }
 
@@ -686,6 +705,7 @@ static void send_ping_locked(grpc_chttp2_transport *t,
 
 static void perform_transport_op(grpc_transport *gt, grpc_transport_op *op) {
   grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt;
+  int close_transport = 0;
 
   lock(t);
 
@@ -705,9 +725,7 @@ static void perform_transport_op(grpc_transport *gt, grpc_transport_op *op) {
         t->global.last_incoming_stream_id,
         grpc_chttp2_grpc_status_to_http2_error(op->goaway_status),
         gpr_slice_ref(*op->goaway_message), &t->global.qbuf);
-    if (!grpc_chttp2_has_streams(t)) {
-      close_transport_locked(t);
-    }
+    close_transport = !grpc_chttp2_has_streams(t);
   }
 
   if (op->set_accept_stream != NULL) {
@@ -720,6 +738,10 @@ static void perform_transport_op(grpc_transport *gt, grpc_transport_op *op) {
     add_to_pollset_locked(t, op->bind_pollset);
   }
 
+  if (op->bind_pollset_set) {
+    add_to_pollset_set_locked(t, op->bind_pollset_set);
+  }
+
   if (op->send_ping) {
     send_ping_locked(t, op->send_ping);
   }
@@ -729,6 +751,12 @@ static void perform_transport_op(grpc_transport *gt, grpc_transport_op *op) {
   }
 
   unlock(t);
+
+  if (close_transport) {
+    lock(t);
+    close_transport_locked(t);
+    unlock(t);
+  }
 }
 
 /*
@@ -750,6 +778,7 @@ static void remove_stream(grpc_chttp2_transport *t, gpr_uint32 id) {
   if (!s) {
     s = grpc_chttp2_stream_map_delete(&t->new_stream_map, id);
   }
+  grpc_chttp2_list_remove_writable_stream(&t->global, &s->global);
   GPR_ASSERT(s);
   s->global.in_stream_map = 0;
   if (t->parsing.incoming_stream == &s->parsing) {
@@ -805,6 +834,12 @@ static void unlock_check_read_write_state(grpc_chttp2_transport *t) {
                                                            stream_global);
       } else {
         stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE;
+        if (stream_global->outgoing_sopb != NULL) {
+          grpc_sopb_reset(stream_global->outgoing_sopb);
+          stream_global->outgoing_sopb = NULL;
+          grpc_chttp2_schedule_closure(transport_global,
+                                       stream_global->send_done_closure, 1);
+        }
         stream_global->read_closed = 1;
         if (!stream_global->published_cancelled) {
           char buffer[GPR_LTOA_MIN_BUFSIZE];
@@ -831,6 +866,9 @@ static void unlock_check_read_write_state(grpc_chttp2_transport *t) {
     if (!stream_global->publish_sopb) {
       continue;
     }
+    if (stream_global->writing_now != 0) {
+      continue;
+    }
     /* FIXME(ctiller): we include in_stream_map in our computation of
        whether the stream is write-closed. This is completely bogus,
        but has the effect of delaying stream-closed until the stream
@@ -873,6 +911,108 @@ static void cancel_from_api(grpc_chttp2_transport_global *transport_global,
                                                 stream_global);
 }
 
+static void close_from_api(grpc_chttp2_transport_global *transport_global,
+                           grpc_chttp2_stream_global *stream_global,
+                           grpc_status_code status,
+                           gpr_slice *optional_message) {
+  gpr_slice hdr;
+  gpr_slice status_hdr;
+  gpr_slice message_pfx;
+  gpr_uint8 *p;
+  gpr_uint32 len = 0;
+
+  GPR_ASSERT(status >= 0 && (int)status < 100);
+
+  stream_global->cancelled = 1;
+  stream_global->cancelled_status = status;
+  GPR_ASSERT(stream_global->id != 0);
+  GPR_ASSERT(!stream_global->written_anything);
+
+  /* Hand roll a header block.
+     This is unnecessarily ugly - at some point we should find a more elegant
+     solution.
+     It's complicated by the fact that our send machinery would be dead by the
+     time we got around to sending this, so instead we ignore HPACK compression
+     and just write the uncompressed bytes onto the wire. */
+  status_hdr = gpr_slice_malloc(15 + (status >= 10));
+  p = GPR_SLICE_START_PTR(status_hdr);
+  *p++ = 0x40; /* literal header */
+  *p++ = 11;   /* len(grpc-status) */
+  *p++ = 'g';
+  *p++ = 'r';
+  *p++ = 'p';
+  *p++ = 'c';
+  *p++ = '-';
+  *p++ = 's';
+  *p++ = 't';
+  *p++ = 'a';
+  *p++ = 't';
+  *p++ = 'u';
+  *p++ = 's';
+  if (status < 10) {
+    *p++ = 1;
+    *p++ = '0' + status;
+  } else {
+    *p++ = 2;
+    *p++ = '0' + (status / 10);
+    *p++ = '0' + (status % 10);
+  }
+  GPR_ASSERT(p == GPR_SLICE_END_PTR(status_hdr));
+  len += GPR_SLICE_LENGTH(status_hdr);
+
+  if (optional_message) {
+    GPR_ASSERT(GPR_SLICE_LENGTH(*optional_message) < 127);
+    message_pfx = gpr_slice_malloc(15);
+    p = GPR_SLICE_START_PTR(message_pfx);
+    *p++ = 0x40;
+    *p++ = 12; /* len(grpc-message) */
+    *p++ = 'g';
+    *p++ = 'r';
+    *p++ = 'p';
+    *p++ = 'c';
+    *p++ = '-';
+    *p++ = 'm';
+    *p++ = 'e';
+    *p++ = 's';
+    *p++ = 's';
+    *p++ = 'a';
+    *p++ = 'g';
+    *p++ = 'e';
+    *p++ = GPR_SLICE_LENGTH(*optional_message);
+    GPR_ASSERT(p == GPR_SLICE_END_PTR(message_pfx));
+    len += GPR_SLICE_LENGTH(message_pfx);
+    len += GPR_SLICE_LENGTH(*optional_message);
+  }
+
+  hdr = gpr_slice_malloc(9);
+  p = GPR_SLICE_START_PTR(hdr);
+  *p++ = len >> 16;
+  *p++ = len >> 8;
+  *p++ = len;
+  *p++ = GRPC_CHTTP2_FRAME_HEADER;
+  *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
+  *p++ = stream_global->id >> 24;
+  *p++ = stream_global->id >> 16;
+  *p++ = stream_global->id >> 8;
+  *p++ = stream_global->id;
+  GPR_ASSERT(p == GPR_SLICE_END_PTR(hdr));
+
+  gpr_slice_buffer_add(&transport_global->qbuf, hdr);
+  gpr_slice_buffer_add(&transport_global->qbuf, status_hdr);
+  if (optional_message) {
+    gpr_slice_buffer_add(&transport_global->qbuf, message_pfx);
+    gpr_slice_buffer_add(&transport_global->qbuf,
+                         gpr_slice_ref(*optional_message));
+  }
+
+  gpr_slice_buffer_add(
+      &transport_global->qbuf,
+      grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR));
+
+  grpc_chttp2_list_add_read_write_state_changed(transport_global,
+                                                stream_global);
+}
+
 static void cancel_stream_cb(grpc_chttp2_transport_global *transport_global,
                              void *user_data,
                              grpc_chttp2_stream_global *stream_global) {
@@ -1001,12 +1141,12 @@ static void schedule_closure_for_connectivity(void *a,
 
 static void connectivity_state_set(
     grpc_chttp2_transport_global *transport_global,
-    grpc_connectivity_state state) {
+    grpc_connectivity_state state, const char *reason) {
   GRPC_CHTTP2_IF_TRACING(
       gpr_log(GPR_DEBUG, "set connectivity_state=%d", state));
   grpc_connectivity_state_set_with_scheduler(
       &TRANSPORT_FROM_GLOBAL(transport_global)->channel_callback.state_tracker,
-      state, schedule_closure_for_connectivity, transport_global);
+      state, schedule_closure_for_connectivity, transport_global, reason);
 }
 
 void grpc_chttp2_schedule_closure(
@@ -1034,6 +1174,13 @@ static void add_to_pollset_locked(grpc_chttp2_transport *t,
   }
 }
 
+static void add_to_pollset_set_locked(grpc_chttp2_transport *t,
+                                  grpc_pollset_set *pollset_set) {
+  if (t->ep) {
+    grpc_endpoint_add_to_pollset_set(t->ep, pollset_set);
+  }
+}
+
 /*
  * TRACING
  */
@@ -1069,9 +1216,17 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *reason,
  * INTEGRATION GLUE
  */
 
-static const grpc_transport_vtable vtable = {
-    sizeof(grpc_chttp2_stream), init_stream,    perform_stream_op,
-    perform_transport_op,       destroy_stream, destroy_transport};
+static char *chttp2_get_peer(grpc_transport *t) {
+  return gpr_strdup(((grpc_chttp2_transport *)t)->peer_string);
+}
+
+static const grpc_transport_vtable vtable = {sizeof(grpc_chttp2_stream),
+                                             init_stream,
+                                             perform_stream_op,
+                                             perform_transport_op,
+                                             destroy_stream,
+                                             destroy_transport,
+                                             chttp2_get_peer};
 
 grpc_transport *grpc_create_chttp2_transport(
     const grpc_channel_args *channel_args, grpc_endpoint *ep, grpc_mdctx *mdctx,
diff --git a/src/core/transport/connectivity_state.c b/src/core/transport/connectivity_state.c
index 1091ceae44..61d26f06f0 100644
--- a/src/core/transport/connectivity_state.c
+++ b/src/core/transport/connectivity_state.c
@@ -34,11 +34,33 @@
 #include "src/core/transport/connectivity_state.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+int grpc_connectivity_state_trace = 0;
+
+const char *grpc_connectivity_state_name(grpc_connectivity_state state) {
+  switch (state) {
+    case GRPC_CHANNEL_IDLE:
+      return "IDLE";
+    case GRPC_CHANNEL_CONNECTING:
+      return "CONNECTING";
+    case GRPC_CHANNEL_READY:
+      return "READY";
+    case GRPC_CHANNEL_TRANSIENT_FAILURE:
+      return "TRANSIENT_FAILURE";
+    case GRPC_CHANNEL_FATAL_FAILURE:
+      return "FATAL_FAILURE";
+  }
+  abort();
+  return "UNKNOWN";
+}
 
 void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker,
-                                  grpc_connectivity_state init_state) {
+                                  grpc_connectivity_state init_state,
+                                  const char *name) {
   tracker->current_state = init_state;
   tracker->watchers = NULL;
+  tracker->name = gpr_strdup(name);
 }
 
 void grpc_connectivity_state_destroy(grpc_connectivity_state_tracker *tracker) {
@@ -54,6 +76,7 @@ void grpc_connectivity_state_destroy(grpc_connectivity_state_tracker *tracker) {
     }
     gpr_free(w);
   }
+  gpr_free(tracker->name);
 }
 
 grpc_connectivity_state grpc_connectivity_state_check(
@@ -64,6 +87,11 @@ grpc_connectivity_state grpc_connectivity_state_check(
 int grpc_connectivity_state_notify_on_state_change(
     grpc_connectivity_state_tracker *tracker, grpc_connectivity_state *current,
     grpc_iomgr_closure *notify) {
+  if (grpc_connectivity_state_trace) {
+    gpr_log(GPR_DEBUG, "CONWATCH: %s: from %s [cur=%s]", tracker->name,
+            grpc_connectivity_state_name(*current),
+            grpc_connectivity_state_name(tracker->current_state));
+  }
   if (tracker->current_state != *current) {
     *current = tracker->current_state;
     grpc_iomgr_add_callback(notify);
@@ -79,12 +107,19 @@ int grpc_connectivity_state_notify_on_state_change(
 
 void grpc_connectivity_state_set_with_scheduler(
     grpc_connectivity_state_tracker *tracker, grpc_connectivity_state state,
-    void (*scheduler)(void *arg, grpc_iomgr_closure *closure), void *arg) {
+    void (*scheduler)(void *arg, grpc_iomgr_closure *closure), void *arg,
+    const char *reason) {
   grpc_connectivity_state_watcher *new = NULL;
   grpc_connectivity_state_watcher *w;
+  if (grpc_connectivity_state_trace) {
+    gpr_log(GPR_DEBUG, "SET: %s: %s --> %s [%s]", tracker->name,
+            grpc_connectivity_state_name(tracker->current_state),
+            grpc_connectivity_state_name(state), reason);
+  }
   if (tracker->current_state == state) {
     return;
   }
+  GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_FATAL_FAILURE);
   tracker->current_state = state;
   while ((w = tracker->watchers)) {
     tracker->watchers = w->next;
@@ -106,7 +141,8 @@ static void default_scheduler(void *ignored, grpc_iomgr_closure *closure) {
 }
 
 void grpc_connectivity_state_set(grpc_connectivity_state_tracker *tracker,
-                                 grpc_connectivity_state state) {
+                                 grpc_connectivity_state state,
+                                 const char *reason) {
   grpc_connectivity_state_set_with_scheduler(tracker, state, default_scheduler,
-                                             NULL);
+                                             NULL, reason);
 }
diff --git a/src/core/transport/connectivity_state.h b/src/core/transport/connectivity_state.h
index bbdcbcb069..a3b0b80c98 100644
--- a/src/core/transport/connectivity_state.h
+++ b/src/core/transport/connectivity_state.h
@@ -51,17 +51,24 @@ typedef struct {
   grpc_connectivity_state current_state;
   /** all our watchers */
   grpc_connectivity_state_watcher *watchers;
+  /** a name to help debugging */
+  char *name;
 } grpc_connectivity_state_tracker;
 
+extern int grpc_connectivity_state_trace;
+
 void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker,
-                                  grpc_connectivity_state init_state);
+                                  grpc_connectivity_state init_state,
+                                  const char *name);
 void grpc_connectivity_state_destroy(grpc_connectivity_state_tracker *tracker);
 
 void grpc_connectivity_state_set(grpc_connectivity_state_tracker *tracker,
-                                 grpc_connectivity_state state);
+                                 grpc_connectivity_state state,
+                                 const char *reason);
 void grpc_connectivity_state_set_with_scheduler(
     grpc_connectivity_state_tracker *tracker, grpc_connectivity_state state,
-    void (*scheduler)(void *arg, grpc_iomgr_closure *closure), void *arg);
+    void (*scheduler)(void *arg, grpc_iomgr_closure *closure), void *arg,
+    const char *reason);
 
 grpc_connectivity_state grpc_connectivity_state_check(
     grpc_connectivity_state_tracker *tracker);
diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c
index e95b7a21f9..44d32b6cb2 100644
--- a/src/core/transport/metadata.c
+++ b/src/core/transport/metadata.c
@@ -135,7 +135,9 @@ static void unlock(grpc_mdctx *ctx) {
   if (ctx->refs == 0) {
     /* uncomment if you're having trouble diagnosing an mdelem leak to make
        things clearer (slows down destruction a lot, however) */
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
     gc_mdtab(ctx);
+#endif
     if (ctx->mdtab_count && ctx->mdtab_count == ctx->mdtab_free) {
       discard_metadata(ctx);
     }
@@ -309,7 +311,37 @@ static void slice_unref(void *p) {
   unlock(ctx);
 }
 
-grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str) {
+grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, int canonicalize_key) {
+  if (canonicalize_key) {
+    size_t len;
+    size_t i;
+    int canonical = 1;
+
+    for (i = 0; str[i]; i++) {
+      if (str[i] >= 'A' && str[i] <= 'Z') {
+        canonical = 0;
+        /* Keep going in loop just to get string length */
+      }
+    }
+    len = i;
+
+    if (canonical) {
+      return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, len);
+    } else {
+      char *copy = gpr_malloc(len);
+      grpc_mdstr *ret;
+      for (i = 0; i < len; i++) {
+        if (str[i] >= 'A' && str[i] <= 'Z') {
+          copy[i] = str[i] - 'A' + 'a';
+        } else {
+          copy[i] = str[i];
+        }
+      }
+      ret = grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)copy, len);
+      gpr_free(copy);
+      return ret;
+    }
+  }
   return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, strlen(str));
 }
 
@@ -491,8 +523,8 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx,
 grpc_mdelem *grpc_mdelem_from_strings(grpc_mdctx *ctx, const char *key,
                                       const char *value) {
   return grpc_mdelem_from_metadata_strings(ctx,
-                                           grpc_mdstr_from_string(ctx, key),
-                                           grpc_mdstr_from_string(ctx, value));
+                                           grpc_mdstr_from_string(ctx, key, 0),
+                                           grpc_mdstr_from_string(ctx, value, 0));
 }
 
 grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key,
@@ -504,9 +536,10 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key,
 grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx,
                                                 const char *key,
                                                 const gpr_uint8 *value,
-                                                size_t value_length) {
+                                                size_t value_length,
+                                                int canonicalize_key) {
   return grpc_mdelem_from_metadata_strings(
-      ctx, grpc_mdstr_from_string(ctx, key),
+      ctx, grpc_mdstr_from_string(ctx, key, canonicalize_key),
       grpc_mdstr_from_buffer(ctx, value, value_length));
 }
 
diff --git a/src/core/transport/metadata.h b/src/core/transport/metadata.h
index 99b15322c3..15ef9bb555 100644
--- a/src/core/transport/metadata.h
+++ b/src/core/transport/metadata.h
@@ -95,7 +95,7 @@ size_t grpc_mdctx_get_mdtab_free_test_only(grpc_mdctx *mdctx);
 
 /* Constructors for grpc_mdstr instances; take a variety of data types that
    clients may have handy */
-grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str);
+grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, int perform_key_canonicalization);
 /* Unrefs the slice. */
 grpc_mdstr *grpc_mdstr_from_slice(grpc_mdctx *ctx, gpr_slice slice);
 grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *str,
@@ -117,7 +117,8 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key,
 grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx,
                                                 const char *key,
                                                 const gpr_uint8 *value,
-                                                size_t value_length);
+                                                size_t value_length,
+                                                int canonicalize_key);
 
 /* Mutator and accessor for grpc_mdelem user data. The destructor function
    is used as a type tag and is checked during user_data fetch. */
diff --git a/src/core/transport/stream_op.h b/src/core/transport/stream_op.h
index f27ef1b66b..227320cf2a 100644
--- a/src/core/transport/stream_op.h
+++ b/src/core/transport/stream_op.h
@@ -108,7 +108,7 @@ void grpc_metadata_batch_move(grpc_metadata_batch *dst,
                               grpc_metadata_batch *src);
 
 /** Add \a storage to the beginning of \a batch. storage->md is
-    assumed to be valid. 
+    assumed to be valid.
     \a storage is owned by the caller and must survive for the
     lifetime of batch. This usually means it should be around
     for the lifetime of the call. */
diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c
index 2689e3028a..c0d92cf93f 100644
--- a/src/core/transport/transport.c
+++ b/src/core/transport/transport.c
@@ -32,6 +32,8 @@
  */
 
 #include "src/core/transport/transport.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 #include "src/core/transport/transport_impl.h"
 
 size_t grpc_transport_stream_size(grpc_transport *transport) {
@@ -65,6 +67,10 @@ void grpc_transport_destroy_stream(grpc_transport *transport,
   transport->vtable->destroy_stream(transport, stream);
 }
 
+char *grpc_transport_get_peer(grpc_transport *transport) {
+  return transport->vtable->get_peer(transport);
+}
+
 void grpc_transport_stream_op_finish_with_failure(
     grpc_transport_stream_op *op) {
   if (op->send_ops) {
@@ -79,12 +85,54 @@ void grpc_transport_stream_op_finish_with_failure(
 }
 
 void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op,
-                                               grpc_status_code status,
-                                               grpc_mdstr *message) {
+                                               grpc_status_code status) {
+  GPR_ASSERT(status != GRPC_STATUS_OK);
   if (op->cancel_with_status == GRPC_STATUS_OK) {
     op->cancel_with_status = status;
   }
-  if (message) {
-    GRPC_MDSTR_UNREF(message);
+  if (op->close_with_status != GRPC_STATUS_OK) {
+    op->close_with_status = GRPC_STATUS_OK;
+    if (op->optional_close_message != NULL) {
+      gpr_slice_unref(*op->optional_close_message);
+      op->optional_close_message = NULL;
+    }
+  }
+}
+
+typedef struct {
+  gpr_slice message;
+  grpc_iomgr_closure *then_call;
+  grpc_iomgr_closure closure;
+} close_message_data;
+
+static void free_message(void *p, int iomgr_success) {
+  close_message_data *cmd = p;
+  gpr_slice_unref(cmd->message);
+  if (cmd->then_call != NULL) {
+    cmd->then_call->cb(cmd->then_call->cb_arg, iomgr_success);
+  }
+  gpr_free(cmd);
+}
+
+void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op,
+                                        grpc_status_code status,
+                                        gpr_slice *optional_message) {
+  close_message_data *cmd;
+  GPR_ASSERT(status != GRPC_STATUS_OK);
+  if (op->cancel_with_status != GRPC_STATUS_OK ||
+      op->close_with_status != GRPC_STATUS_OK) {
+    if (optional_message) {
+      gpr_slice_unref(*optional_message);
+    }
+    return;
+  }
+  if (optional_message) {
+    cmd = gpr_malloc(sizeof(*cmd));
+    cmd->message = *optional_message;
+    cmd->then_call = op->on_consumed;
+    grpc_iomgr_closure_init(&cmd->closure, free_message, cmd);
+    op->on_consumed = &cmd->closure;
+    op->optional_close_message = &cmd->message;
   }
+  op->close_with_status = status;
 }
diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h
index 64503604ee..92c1f38c5e 100644
--- a/src/core/transport/transport.h
+++ b/src/core/transport/transport.h
@@ -80,8 +80,14 @@ typedef struct grpc_transport_stream_op {
 
   grpc_pollset *bind_pollset;
 
+  /** If != GRPC_STATUS_OK, cancel this stream */
   grpc_status_code cancel_with_status;
 
+  /** If != GRPC_STATUS_OK, send grpc-status, grpc-message, and close this
+      stream for both reading and writing */
+  grpc_status_code close_with_status;
+  gpr_slice *optional_close_message;
+
   /* Indexes correspond to grpc_context_index enum values */
   grpc_call_context_element *context;
 } grpc_transport_stream_op;
@@ -109,6 +115,8 @@ typedef struct grpc_transport_op {
   void *set_accept_stream_user_data;
   /** add this transport to a pollset */
   grpc_pollset *bind_pollset;
+  /** add this transport to a pollset_set */
+  grpc_pollset_set *bind_pollset_set;
   /** send a ping, call this back if not NULL */
   grpc_iomgr_closure *send_ping;
 } grpc_transport_op;
@@ -146,8 +154,11 @@ void grpc_transport_destroy_stream(grpc_transport *transport,
 void grpc_transport_stream_op_finish_with_failure(grpc_transport_stream_op *op);
 
 void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op,
-                                               grpc_status_code status,
-                                               grpc_mdstr *message);
+                                               grpc_status_code status);
+
+void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op,
+                                        grpc_status_code status,
+                                        gpr_slice *optional_message);
 
 char *grpc_transport_stream_op_string(grpc_transport_stream_op *op);
 
@@ -182,4 +193,7 @@ void grpc_transport_close(grpc_transport *transport);
 /* Destroy the transport */
 void grpc_transport_destroy(grpc_transport *transport);
 
+/* Get the transports peer */
+char *grpc_transport_get_peer(grpc_transport *transport);
+
 #endif /* GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_H */
diff --git a/src/core/transport/transport_impl.h b/src/core/transport/transport_impl.h
index 515721dfb6..d3bbdf6c27 100644
--- a/src/core/transport/transport_impl.h
+++ b/src/core/transport/transport_impl.h
@@ -58,6 +58,9 @@ typedef struct grpc_transport_vtable {
 
   /* implementation of grpc_transport_destroy */
   void (*destroy)(grpc_transport *self);
+
+  /* implementation of grpc_transport_get_peer */
+  char *(*get_peer)(grpc_transport *self);
 } grpc_transport_vtable;
 
 /* an instance of a grpc transport */
diff --git a/src/core/transport/transport_op_string.c b/src/core/transport/transport_op_string.c
index 10d796fc15..f62c340e97 100644
--- a/src/core/transport/transport_op_string.c
+++ b/src/core/transport/transport_op_string.c
@@ -116,10 +116,9 @@ char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) {
   if (op->send_ops) {
     if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
     first = 0;
-    gpr_strvec_add(&b, gpr_strdup("SEND"));
-    if (op->is_last_send) {
-      gpr_strvec_add(&b, gpr_strdup("_LAST"));
-    }
+    gpr_asprintf(&tmp, "SEND%s:%p", op->is_last_send ? "_LAST" : "",
+                 op->on_done_send);
+    gpr_strvec_add(&b, tmp);
     gpr_strvec_add(&b, gpr_strdup("["));
     gpr_strvec_add(&b, grpc_sopb_string(op->send_ops));
     gpr_strvec_add(&b, gpr_strdup("]"));
@@ -128,7 +127,8 @@ char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) {
   if (op->recv_ops) {
     if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
     first = 0;
-    gpr_asprintf(&tmp, "RECV:max_recv_bytes=%d", op->max_recv_bytes);
+    gpr_asprintf(&tmp, "RECV:%p:max_recv_bytes=%d", op->on_done_recv,
+                 op->max_recv_bytes);
     gpr_strvec_add(&b, tmp);
   }
 
diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c
index 6156a39d09..609fc06ed5 100644
--- a/src/core/tsi/ssl_transport_security.c
+++ b/src/core/tsi/ssl_transport_security.c
@@ -1293,8 +1293,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory(
     const size_t* pem_private_keys_sizes, const unsigned char** pem_cert_chains,
     const size_t* pem_cert_chains_sizes, size_t key_cert_pair_count,
     const unsigned char* pem_client_root_certs,
-    size_t pem_client_root_certs_size, const char* cipher_list,
-    const unsigned char** alpn_protocols,
+    size_t pem_client_root_certs_size, int force_client_auth,
+    const char* cipher_list, const unsigned char** alpn_protocols,
     const unsigned char* alpn_protocols_lengths, uint16_t num_alpn_protocols,
     tsi_ssl_handshaker_factory** factory) {
   tsi_ssl_server_handshaker_factory* impl = NULL;
@@ -1349,6 +1349,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory(
       if (result != TSI_OK) break;
 
       if (pem_client_root_certs != NULL) {
+        int flags = SSL_VERIFY_PEER;
         STACK_OF(X509_NAME)* root_names = NULL;
         result = ssl_ctx_load_verification_certs(
             impl->ssl_contexts[i], pem_client_root_certs,
@@ -1358,7 +1359,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory(
           break;
         }
         SSL_CTX_set_client_CA_list(impl->ssl_contexts[i], root_names);
-        SSL_CTX_set_verify(impl->ssl_contexts[i], SSL_VERIFY_PEER, NULL);
+        if (force_client_auth) flags |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+        SSL_CTX_set_verify(impl->ssl_contexts[i], flags, NULL);
         /* TODO(jboeuf): Add revocation verification. */
       }
 
diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h
index b2aa2f393e..4bf6c81b75 100644
--- a/src/core/tsi/ssl_transport_security.h
+++ b/src/core/tsi/ssl_transport_security.h
@@ -107,10 +107,14 @@ tsi_result tsi_create_ssl_client_handshaker_factory(
    - key_cert_pair_count indicates the number of items in the private_key_files
      and cert_chain_files parameters.
    - pem_client_roots is the buffer containing the PEM encoding of the client
-     root certificates. This parameter may be NULL in which case the server
-     will not ask the client to authenticate itself with a certificate (server-
-     only authentication mode).
-   - pem_client_roots_size is the size of the associated buffer.
+     root certificates. This parameter may be NULL in which case the server will
+     not authenticate the client. If not NULL, the force_client_auth parameter
+     specifies if the server will accept only authenticated clients or both
+     authenticated and non-authenticated clients.
+   - pem_client_root_certs_size is the size of the associated buffer.
+   - force_client_auth, if set to non-zero will force the client to authenticate
+     with an SSL cert. Note that this option is ignored if pem_client_root_certs
+     is NULL or pem_client_roots_certs_size is 0
    - cipher_suites contains an optional list of the ciphers that the server
      supports. The format of this string is described in:
      https://www.openssl.org/docs/apps/ciphers.html.
@@ -131,8 +135,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory(
     const size_t* pem_private_keys_sizes, const unsigned char** pem_cert_chains,
     const size_t* pem_cert_chains_sizes, size_t key_cert_pair_count,
     const unsigned char* pem_client_root_certs,
-    size_t pem_client_root_certs_size, const char* cipher_suites,
-    const unsigned char** alpn_protocols,
+    size_t pem_client_root_certs_size, int force_client_auth,
+    const char* cipher_suites, const unsigned char** alpn_protocols,
     const unsigned char* alpn_protocols_lengths, uint16_t num_alpn_protocols,
     tsi_ssl_handshaker_factory** factory);