Merge github.com:grpc/grpc into reuse_port

150 files changed, 5327 insertions, 959 deletions

diff --git a/src/core/ext/census/gen/README.md b/src/core/ext/census/gen/README.md
new file mode 100644
index 0000000000..72bef6542d
--- /dev/null
+++ b/src/core/ext/census/gen/README.md
@@ -0,0 +1,6 @@
+Files generated for use by Census stats and trace recording subsystem.
+
+#Files
+* census.pb.{h,c} - Generated from src/core/ext/census/census.proto, using the
+  script `tools/codegen/core/gen_nano_proto.sh src/proto/census/census.proto
+  $PWD/src/core/ext/census/gen src/core/ext/census/gen`
diff --git a/src/core/ext/census/gen/census.pb.c b/src/core/ext/census/gen/census.pb.c
new file mode 100644
index 0000000000..d614636c90
--- /dev/null
+++ b/src/core/ext/census/gen/census.pb.c
@@ -0,0 +1,179 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.3.5-dev */
+
+#include "src/core/ext/census/gen/census.pb.h"
+
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+
+
+const pb_field_t google_census_Duration_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Duration, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Duration, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Timestamp_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Timestamp, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Timestamp, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_fields[5] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_Metric, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_Metric, description, name, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_Metric, unit, description, &google_census_Metric_MeasurementUnit_fields),
+    PB_FIELD(  4, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Metric, id, unit, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_BasicUnit_fields[2] = {
+    PB_FIELD(  1, UENUM   , OPTIONAL, STATIC  , FIRST, google_census_Metric_BasicUnit, type, type, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_MeasurementUnit_fields[4] = {
+    PB_FIELD(  1, INT32   , OPTIONAL, STATIC  , FIRST, google_census_Metric_MeasurementUnit, prefix, prefix, 0),
+    PB_FIELD(  2, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Metric_MeasurementUnit, numerator, prefix, &google_census_Metric_BasicUnit_fields),
+    PB_FIELD(  3, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Metric_MeasurementUnit, denominator, numerator, &google_census_Metric_BasicUnit_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_fields[3] = {
+    PB_ONEOF_FIELD(options,   1, MESSAGE , ONEOF, STATIC  , FIRST, google_census_AggregationDescriptor, bucket_boundaries, bucket_boundaries, &google_census_AggregationDescriptor_BucketBoundaries_fields),
+    PB_ONEOF_FIELD(options,   2, MESSAGE , ONEOF, STATIC  , FIRST, google_census_AggregationDescriptor, interval_boundaries, interval_boundaries, &google_census_AggregationDescriptor_IntervalBoundaries_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_BucketBoundaries_fields[2] = {
+    PB_FIELD(  1, DOUBLE  , REPEATED, CALLBACK, FIRST, google_census_AggregationDescriptor_BucketBoundaries, bounds, bounds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_IntervalBoundaries_fields[2] = {
+    PB_FIELD(  1, DOUBLE  , REPEATED, CALLBACK, FIRST, google_census_AggregationDescriptor_IntervalBoundaries, window_size, window_size, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Distribution_fields[5] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Distribution, count, count, 0),
+    PB_FIELD(  2, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_Distribution, mean, count, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_Distribution, range, mean, &google_census_Distribution_Range_fields),
+    PB_FIELD(  4, INT64   , REPEATED, CALLBACK, OTHER, google_census_Distribution, bucket_count, range, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Distribution_Range_fields[3] = {
+    PB_FIELD(  1, DOUBLE  , OPTIONAL, STATIC  , FIRST, google_census_Distribution_Range, min, min, 0),
+    PB_FIELD(  2, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_Distribution_Range, max, min, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_IntervalStats_fields[2] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, google_census_IntervalStats, window, window, &google_census_IntervalStats_Window_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_IntervalStats_Window_fields[4] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, google_census_IntervalStats_Window, window_size, window_size, &google_census_Duration_fields),
+    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, google_census_IntervalStats_Window, count, window_size, 0),
+    PB_FIELD(  3, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_IntervalStats_Window, mean, count, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Tag_fields[3] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, google_census_Tag, key, key, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , OTHER, google_census_Tag, value, key, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_View_fields[6] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_View, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_View, description, name, 0),
+    PB_FIELD(  3, INT32   , OPTIONAL, STATIC  , OTHER, google_census_View, metric_id, description, 0),
+    PB_FIELD(  4, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_View, aggregation, metric_id, &google_census_AggregationDescriptor_fields),
+    PB_FIELD(  5, STRING  , REPEATED, CALLBACK, OTHER, google_census_View, tag_key, aggregation, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Aggregation_fields[6] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_Aggregation, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_Aggregation, description, name, 0),
+    PB_ONEOF_FIELD(data,   3, MESSAGE , ONEOF, STATIC  , OTHER, google_census_Aggregation, distribution, description, &google_census_Distribution_fields),
+    PB_ONEOF_FIELD(data,   4, MESSAGE , ONEOF, STATIC  , OTHER, google_census_Aggregation, interval_stats, description, &google_census_IntervalStats_fields),
+    PB_FIELD(  5, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Aggregation, tag, data.interval_stats, &google_census_Tag_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_ViewAggregations_fields[4] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, google_census_ViewAggregations, aggregation, aggregation, &google_census_Aggregation_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_ViewAggregations, start, aggregation, &google_census_Timestamp_fields),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_ViewAggregations, end, start, &google_census_Timestamp_fields),
+    PB_LAST_FIELD
+};
+
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_32BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in 8 or 16 bit
+ * field descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Distribution, range) < 65536 && pb_membersize(google_census_IntervalStats, window) < 65536 && pb_membersize(google_census_IntervalStats_Window, window_size) < 65536 && pb_membersize(google_census_View, aggregation) < 65536 && pb_membersize(google_census_Aggregation, data.distribution) < 65536 && pb_membersize(google_census_Aggregation, data.interval_stats) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Distribution, range) < 65536 && pb_membersize(google_census_IntervalStats, window) < 65536 && pb_membersize(google_census_IntervalStats_Window, window_size) < 65536 && pb_membersize(google_census_View, aggregation) < 65536 && pb_membersize(google_census_Aggregation, data.distribution) < 65536 && pb_membersize(google_census_Aggregation, data.interval_stats) < 65536 && pb_membersize(google_census_Aggregation, tag) < 65536 && pb_membersize(google_census_ViewAggregations, aggregation) < 65536 && pb_membersize(google_census_ViewAggregations, start) < 65536 && pb_membersize(google_census_ViewAggregations, end) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_google_census_Duration_google_census_Timestamp_google_census_Metric_google_census_Metric_BasicUnit_google_census_Metric_MeasurementUnit_google_census_AggregationDescriptor_google_census_AggregationDescriptor_BucketBoundaries_google_census_AggregationDescriptor_IntervalBoundaries_google_census_Distribution_google_census_Distribution_Range_google_census_IntervalStats_google_census_IntervalStats_Window_google_census_Tag_google_census_View_google_census_Aggregation_google_census_ViewAggregations)
+#endif
+
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_16BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in the default
+ * 8 bit descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Distribution, range) < 256 && pb_membersize(google_census_IntervalStats, window) < 256 && pb_membersize(google_census_IntervalStats_Window, window_size) < 256 && pb_membersize(google_census_View, aggregation) < 256 && pb_membersize(google_census_Aggregation, data.distribution) < 256 && pb_membersize(google_census_Aggregation, data.interval_stats) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Distribution, range) < 256 && pb_membersize(google_census_IntervalStats, window) < 256 && pb_membersize(google_census_IntervalStats_Window, window_size) < 256 && pb_membersize(google_census_View, aggregation) < 256 && pb_membersize(google_census_Aggregation, data.distribution) < 256 && pb_membersize(google_census_Aggregation, data.interval_stats) < 256 && pb_membersize(google_census_Aggregation, tag) < 256 && pb_membersize(google_census_ViewAggregations, aggregation) < 256 && pb_membersize(google_census_ViewAggregations, start) < 256 && pb_membersize(google_census_ViewAggregations, end) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_google_census_Duration_google_census_Timestamp_google_census_Metric_google_census_Metric_BasicUnit_google_census_Metric_MeasurementUnit_google_census_AggregationDescriptor_google_census_AggregationDescriptor_BucketBoundaries_google_census_AggregationDescriptor_IntervalBoundaries_google_census_Distribution_google_census_Distribution_Range_google_census_IntervalStats_google_census_IntervalStats_Window_google_census_Tag_google_census_View_google_census_Aggregation_google_census_ViewAggregations)
+#endif
+
+
+/* On some platforms (such as AVR), double is really float.
+ * These are not directly supported by nanopb, but see example_avr_double.
+ * To get rid of this error, remove any double fields from your .proto.
+ */
+PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)
+
diff --git a/src/core/ext/census/gen/census.pb.h b/src/core/ext/census/gen/census.pb.h
new file mode 100644
index 0000000000..d040fe29e7
--- /dev/null
+++ b/src/core/ext/census/gen/census.pb.h
@@ -0,0 +1,294 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.3.5-dev */
+
+#ifndef GRPC_CORE_EXT_CENSUS_GEN_CENSUS_PB_H
+#define GRPC_CORE_EXT_CENSUS_GEN_CENSUS_PB_H
+#include "third_party/nanopb/pb.h"
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Enum definitions */
+typedef enum _google_census_Metric_BasicUnit_Measure {
+    google_census_Metric_BasicUnit_Measure_UNKNOWN = 0,
+    google_census_Metric_BasicUnit_Measure_BITS = 1,
+    google_census_Metric_BasicUnit_Measure_BYTES = 2,
+    google_census_Metric_BasicUnit_Measure_SECS = 3,
+    google_census_Metric_BasicUnit_Measure_CORES = 4,
+    google_census_Metric_BasicUnit_Measure_MAX_UNITS = 5
+} google_census_Metric_BasicUnit_Measure;
+
+/* Struct definitions */
+typedef struct _google_census_AggregationDescriptor_BucketBoundaries {
+    pb_callback_t bounds;
+} google_census_AggregationDescriptor_BucketBoundaries;
+
+typedef struct _google_census_AggregationDescriptor_IntervalBoundaries {
+    pb_callback_t window_size;
+} google_census_AggregationDescriptor_IntervalBoundaries;
+
+typedef struct _google_census_IntervalStats {
+    pb_callback_t window;
+} google_census_IntervalStats;
+
+typedef struct _google_census_AggregationDescriptor {
+    pb_size_t which_options;
+    union {
+        google_census_AggregationDescriptor_BucketBoundaries bucket_boundaries;
+        google_census_AggregationDescriptor_IntervalBoundaries interval_boundaries;
+    } options;
+} google_census_AggregationDescriptor;
+
+typedef struct _google_census_Distribution_Range {
+    bool has_min;
+    double min;
+    bool has_max;
+    double max;
+} google_census_Distribution_Range;
+
+typedef struct _google_census_Duration {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} google_census_Duration;
+
+typedef struct _google_census_Metric_BasicUnit {
+    bool has_type;
+    google_census_Metric_BasicUnit_Measure type;
+} google_census_Metric_BasicUnit;
+
+typedef struct _google_census_Metric_MeasurementUnit {
+    bool has_prefix;
+    int32_t prefix;
+    pb_callback_t numerator;
+    pb_callback_t denominator;
+} google_census_Metric_MeasurementUnit;
+
+typedef struct _google_census_Tag {
+    bool has_key;
+    char key[255];
+    bool has_value;
+    char value[255];
+} google_census_Tag;
+
+typedef struct _google_census_Timestamp {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} google_census_Timestamp;
+
+typedef struct _google_census_Distribution {
+    bool has_count;
+    int64_t count;
+    bool has_mean;
+    double mean;
+    bool has_range;
+    google_census_Distribution_Range range;
+    pb_callback_t bucket_count;
+} google_census_Distribution;
+
+typedef struct _google_census_IntervalStats_Window {
+    bool has_window_size;
+    google_census_Duration window_size;
+    bool has_count;
+    int64_t count;
+    bool has_mean;
+    double mean;
+} google_census_IntervalStats_Window;
+
+typedef struct _google_census_Metric {
+    pb_callback_t name;
+    pb_callback_t description;
+    bool has_unit;
+    google_census_Metric_MeasurementUnit unit;
+    bool has_id;
+    int32_t id;
+} google_census_Metric;
+
+typedef struct _google_census_View {
+    pb_callback_t name;
+    pb_callback_t description;
+    bool has_metric_id;
+    int32_t metric_id;
+    bool has_aggregation;
+    google_census_AggregationDescriptor aggregation;
+    pb_callback_t tag_key;
+} google_census_View;
+
+typedef struct _google_census_ViewAggregations {
+    pb_callback_t aggregation;
+    bool has_start;
+    google_census_Timestamp start;
+    bool has_end;
+    google_census_Timestamp end;
+} google_census_ViewAggregations;
+
+typedef struct _google_census_Aggregation {
+    pb_callback_t name;
+    pb_callback_t description;
+    pb_size_t which_data;
+    union {
+        google_census_Distribution distribution;
+        google_census_IntervalStats interval_stats;
+    } data;
+    pb_callback_t tag;
+} google_census_Aggregation;
+
+/* Default values for struct fields */
+
+/* Initializer values for message structs */
+#define google_census_Duration_init_default      {false, 0, false, 0}
+#define google_census_Timestamp_init_default     {false, 0, false, 0}
+#define google_census_Metric_init_default        {{{NULL}, NULL}, {{NULL}, NULL}, false, google_census_Metric_MeasurementUnit_init_default, false, 0}
+#define google_census_Metric_BasicUnit_init_default {false, (google_census_Metric_BasicUnit_Measure)0}
+#define google_census_Metric_MeasurementUnit_init_default {false, 0, {{NULL}, NULL}, {{NULL}, NULL}}
+#define google_census_AggregationDescriptor_init_default {0, {google_census_AggregationDescriptor_BucketBoundaries_init_default}}
+#define google_census_AggregationDescriptor_BucketBoundaries_init_default {{{NULL}, NULL}}
+#define google_census_AggregationDescriptor_IntervalBoundaries_init_default {{{NULL}, NULL}}
+#define google_census_Distribution_init_default  {false, 0, false, 0, false, google_census_Distribution_Range_init_default, {{NULL}, NULL}}
+#define google_census_Distribution_Range_init_default {false, 0, false, 0}
+#define google_census_IntervalStats_init_default {{{NULL}, NULL}}
+#define google_census_IntervalStats_Window_init_default {false, google_census_Duration_init_default, false, 0, false, 0}
+#define google_census_Tag_init_default           {false, "", false, ""}
+#define google_census_View_init_default          {{{NULL}, NULL}, {{NULL}, NULL}, false, 0, false, google_census_AggregationDescriptor_init_default, {{NULL}, NULL}}
+#define google_census_Aggregation_init_default   {{{NULL}, NULL}, {{NULL}, NULL}, 0, {google_census_Distribution_init_default}, {{NULL}, NULL}}
+#define google_census_ViewAggregations_init_default {{{NULL}, NULL}, false, google_census_Timestamp_init_default, false, google_census_Timestamp_init_default}
+#define google_census_Duration_init_zero         {false, 0, false, 0}
+#define google_census_Timestamp_init_zero        {false, 0, false, 0}
+#define google_census_Metric_init_zero           {{{NULL}, NULL}, {{NULL}, NULL}, false, google_census_Metric_MeasurementUnit_init_zero, false, 0}
+#define google_census_Metric_BasicUnit_init_zero {false, (google_census_Metric_BasicUnit_Measure)0}
+#define google_census_Metric_MeasurementUnit_init_zero {false, 0, {{NULL}, NULL}, {{NULL}, NULL}}
+#define google_census_AggregationDescriptor_init_zero {0, {google_census_AggregationDescriptor_BucketBoundaries_init_zero}}
+#define google_census_AggregationDescriptor_BucketBoundaries_init_zero {{{NULL}, NULL}}
+#define google_census_AggregationDescriptor_IntervalBoundaries_init_zero {{{NULL}, NULL}}
+#define google_census_Distribution_init_zero     {false, 0, false, 0, false, google_census_Distribution_Range_init_zero, {{NULL}, NULL}}
+#define google_census_Distribution_Range_init_zero {false, 0, false, 0}
+#define google_census_IntervalStats_init_zero    {{{NULL}, NULL}}
+#define google_census_IntervalStats_Window_init_zero {false, google_census_Duration_init_zero, false, 0, false, 0}
+#define google_census_Tag_init_zero              {false, "", false, ""}
+#define google_census_View_init_zero             {{{NULL}, NULL}, {{NULL}, NULL}, false, 0, false, google_census_AggregationDescriptor_init_zero, {{NULL}, NULL}}
+#define google_census_Aggregation_init_zero      {{{NULL}, NULL}, {{NULL}, NULL}, 0, {google_census_Distribution_init_zero}, {{NULL}, NULL}}
+#define google_census_ViewAggregations_init_zero {{{NULL}, NULL}, false, google_census_Timestamp_init_zero, false, google_census_Timestamp_init_zero}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define google_census_AggregationDescriptor_BucketBoundaries_bounds_tag 1
+#define google_census_AggregationDescriptor_IntervalBoundaries_window_size_tag 1
+#define google_census_IntervalStats_window_tag   1
+#define google_census_AggregationDescriptor_bucket_boundaries_tag 1
+
+#define google_census_AggregationDescriptor_interval_boundaries_tag 2
+#define google_census_Distribution_Range_min_tag 1
+#define google_census_Distribution_Range_max_tag 2
+#define google_census_Duration_seconds_tag       1
+#define google_census_Duration_nanos_tag         2
+#define google_census_Metric_BasicUnit_type_tag  1
+#define google_census_Metric_MeasurementUnit_prefix_tag 1
+#define google_census_Metric_MeasurementUnit_numerator_tag 2
+#define google_census_Metric_MeasurementUnit_denominator_tag 3
+#define google_census_Tag_key_tag                1
+#define google_census_Tag_value_tag              2
+#define google_census_Timestamp_seconds_tag      1
+#define google_census_Timestamp_nanos_tag        2
+#define google_census_Distribution_count_tag     1
+#define google_census_Distribution_mean_tag      2
+#define google_census_Distribution_range_tag     3
+#define google_census_Distribution_bucket_count_tag 4
+#define google_census_IntervalStats_Window_window_size_tag 1
+#define google_census_IntervalStats_Window_count_tag 2
+#define google_census_IntervalStats_Window_mean_tag 3
+#define google_census_Metric_name_tag            1
+#define google_census_Metric_description_tag     2
+#define google_census_Metric_unit_tag            3
+#define google_census_Metric_id_tag              4
+#define google_census_View_name_tag              1
+#define google_census_View_description_tag       2
+#define google_census_View_metric_id_tag         3
+#define google_census_View_aggregation_tag       4
+#define google_census_View_tag_key_tag           5
+#define google_census_ViewAggregations_aggregation_tag 1
+#define google_census_ViewAggregations_start_tag 2
+#define google_census_ViewAggregations_end_tag   3
+#define google_census_Aggregation_distribution_tag 3
+
+#define google_census_Aggregation_interval_stats_tag 4
+#define google_census_Aggregation_name_tag       1
+#define google_census_Aggregation_description_tag 2
+#define google_census_Aggregation_tag_tag        5
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t google_census_Duration_fields[3];
+extern const pb_field_t google_census_Timestamp_fields[3];
+extern const pb_field_t google_census_Metric_fields[5];
+extern const pb_field_t google_census_Metric_BasicUnit_fields[2];
+extern const pb_field_t google_census_Metric_MeasurementUnit_fields[4];
+extern const pb_field_t google_census_AggregationDescriptor_fields[3];
+extern const pb_field_t google_census_AggregationDescriptor_BucketBoundaries_fields[2];
+extern const pb_field_t google_census_AggregationDescriptor_IntervalBoundaries_fields[2];
+extern const pb_field_t google_census_Distribution_fields[5];
+extern const pb_field_t google_census_Distribution_Range_fields[3];
+extern const pb_field_t google_census_IntervalStats_fields[2];
+extern const pb_field_t google_census_IntervalStats_Window_fields[4];
+extern const pb_field_t google_census_Tag_fields[3];
+extern const pb_field_t google_census_View_fields[6];
+extern const pb_field_t google_census_Aggregation_fields[6];
+extern const pb_field_t google_census_ViewAggregations_fields[4];
+
+/* Maximum encoded size of messages (where known) */
+#define google_census_Duration_size              22
+#define google_census_Timestamp_size             22
+#define google_census_Metric_BasicUnit_size      2
+#define google_census_Distribution_Range_size    18
+#define google_census_IntervalStats_Window_size  44
+#define google_census_Tag_size                   516
+
+/* Message IDs (where set with "msgid" option) */
+#ifdef PB_MSGID
+
+#define CENSUS_MESSAGES \
+
+
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/src/core/ext/census/grpc_filter.c b/src/core/ext/census/grpc_filter.c
index df7016efe3..72e4e5427e 100644
--- a/src/core/ext/census/grpc_filter.c
+++ b/src/core/ext/census/grpc_filter.c
@@ -134,7 +134,9 @@ static void client_init_call_elem(grpc_exec_ctx *exec_ctx,
 }
 
 static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_element *elem, void *ignored) {
+                                     grpc_call_element *elem,
+                                     const grpc_call_stats *stats,
+                                     void *ignored) {
   call_data *d = elem->call_data;
   GPR_ASSERT(d != NULL);
   /* TODO(hongyu): record rpc client stats and census_rpc_end_op here */
@@ -152,7 +154,9 @@ static void server_init_call_elem(grpc_exec_ctx *exec_ctx,
 }
 
 static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_element *elem, void *ignored) {
+                                     grpc_call_element *elem,
+                                     const grpc_call_stats *stats,
+                                     void *ignored) {
   call_data *d = elem->call_data;
   GPR_ASSERT(d != NULL);
   /* TODO(hongyu): record rpc server stats and census_tracing_end_op here */
@@ -176,7 +180,7 @@ const grpc_channel_filter grpc_client_census_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     client_init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     client_destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
@@ -189,7 +193,7 @@ const grpc_channel_filter grpc_server_census_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     server_init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     server_destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/ext/client_config/channel_connectivity.c b/src/core/ext/client_config/channel_connectivity.c
index b452cc33ac..20c01a9a7c 100644
--- a/src/core/ext/client_config/channel_connectivity.c
+++ b/src/core/ext/client_config/channel_connectivity.c
@@ -62,7 +62,7 @@ grpc_connectivity_state grpc_channel_check_connectivity_state(
           "not a (u)client channel, but '%s'",
           client_channel_elem->filter->name);
   grpc_exec_ctx_finish(&exec_ctx);
-  return GRPC_CHANNEL_FATAL_FAILURE;
+  return GRPC_CHANNEL_SHUTDOWN;
 }
 
 typedef enum {
@@ -75,7 +75,6 @@ typedef enum {
 typedef struct {
   gpr_mu mu;
   callback_phase phase;
-  grpc_error *error;
   grpc_closure on_complete;
   grpc_timer alarm;
   grpc_connectivity_state state;
@@ -95,7 +94,6 @@ static void delete_state_watcher(grpc_exec_ctx *exec_ctx, state_watcher *w) {
     abort();
   }
   gpr_mu_destroy(&w->mu);
-  GRPC_ERROR_UNREF(w->error);
   gpr_free(w);
 }
 
@@ -131,17 +129,25 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w,
   }
 
   gpr_mu_lock(&w->mu);
-  const char *msg = grpc_error_string(error);
-  grpc_error_free_string(msg);
 
   if (due_to_completion) {
-    GRPC_ERROR_UNREF(w->error);
-    w->error = GRPC_ERROR_NONE;
+    if (grpc_trace_operation_failures) {
+      GRPC_LOG_IF_ERROR("watch_completion_error", GRPC_ERROR_REF(error));
+    }
+    GRPC_ERROR_UNREF(error);
+    error = GRPC_ERROR_NONE;
+  } else {
+    if (error == GRPC_ERROR_NONE) {
+      error =
+          GRPC_ERROR_CREATE("Timed out waiting for connection state change");
+    } else if (error == GRPC_ERROR_CANCELLED) {
+      error = GRPC_ERROR_NONE;
+    }
   }
   switch (w->phase) {
     case WAITING:
       w->phase = CALLING_BACK;
-      grpc_cq_end_op(exec_ctx, w->cq, w->tag, GRPC_ERROR_REF(w->error),
+      grpc_cq_end_op(exec_ctx, w->cq, w->tag, GRPC_ERROR_REF(error),
                      finished_completion, w, &w->completion_storage);
       break;
     case CALLING_BACK:
@@ -194,7 +200,6 @@ void grpc_channel_watch_connectivity_state(
   grpc_closure_init(&w->on_complete, watch_complete, w);
   w->phase = WAITING;
   w->state = last_observed_state;
-  w->error = GRPC_ERROR_CREATE("Timeout waiting for channel state");
   w->cq = cq;
   w->tag = tag;
   w->channel = channel;
diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c
index d00b966812..1d5a7d5224 100644
--- a/src/core/ext/client_config/client_channel.c
+++ b/src/core/ext/client_config/client_channel.c
@@ -120,7 +120,7 @@ static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx,
                                                   grpc_error *error,
                                                   const char *reason) {
   if ((state == GRPC_CHANNEL_TRANSIENT_FAILURE ||
-       state == GRPC_CHANNEL_FATAL_FAILURE) &&
+       state == GRPC_CHANNEL_SHUTDOWN) &&
       chand->lb_policy != NULL) {
     /* cancel fail-fast picks */
     grpc_lb_policy_cancel_picks(
@@ -139,8 +139,7 @@ static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx,
   /* check if the notification is for a stale policy */
   if (w->lb_policy != w->chand->lb_policy) return;
 
-  if (publish_state == GRPC_CHANNEL_FATAL_FAILURE &&
-      w->chand->resolver != NULL) {
+  if (publish_state == GRPC_CHANNEL_SHUTDOWN && w->chand->resolver != NULL) {
     publish_state = GRPC_CHANNEL_TRANSIENT_FAILURE;
     grpc_resolver_channel_saw_error(exec_ctx, w->chand->resolver);
     GRPC_LB_POLICY_UNREF(exec_ctx, w->chand->lb_policy, "channel");
@@ -148,7 +147,7 @@ static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx,
   }
   set_channel_connectivity_state_locked(exec_ctx, w->chand, publish_state,
                                         GRPC_ERROR_REF(error), "lb_changed");
-  if (w->state != GRPC_CHANNEL_FATAL_FAILURE) {
+  if (w->state != GRPC_CHANNEL_SHUTDOWN) {
     watch_lb_policy(exec_ctx, w->chand, w->lb_policy, w->state);
   }
 }
@@ -246,7 +245,7 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
     }
     grpc_error *refs[] = {error, state_error};
     set_channel_connectivity_state_locked(
-        exec_ctx, chand, GRPC_CHANNEL_FATAL_FAILURE,
+        exec_ctx, chand, GRPC_CHANNEL_SHUTDOWN,
         GRPC_ERROR_CREATE_REFERENCING("Got config after disconnection", refs,
                                       GPR_ARRAY_SIZE(refs)),
         "resolver_gone");
@@ -277,7 +276,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                   grpc_transport_op *op) {
   channel_data *chand = elem->channel_data;
 
-  grpc_exec_ctx_push(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
 
   GPR_ASSERT(op->set_accept_stream == false);
   if (op->bind_pollset != NULL) {
@@ -296,9 +295,9 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
 
   if (op->send_ping != NULL) {
     if (chand->lb_policy == NULL) {
-      grpc_exec_ctx_push(exec_ctx, op->send_ping,
-                         GRPC_ERROR_CREATE("Ping with no load balancing"),
-                         NULL);
+      grpc_exec_ctx_sched(exec_ctx, op->send_ping,
+                          GRPC_ERROR_CREATE("Ping with no load balancing"),
+                          NULL);
     } else {
       grpc_lb_policy_ping_one(exec_ctx, chand->lb_policy, op->send_ping);
       op->bind_pollset = NULL;
@@ -309,7 +308,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
   if (op->disconnect_with_error != GRPC_ERROR_NONE) {
     if (chand->resolver != NULL) {
       set_channel_connectivity_state_locked(
-          exec_ctx, chand, GRPC_CHANNEL_FATAL_FAILURE,
+          exec_ctx, chand, GRPC_CHANNEL_SHUTDOWN,
           GRPC_ERROR_REF(op->disconnect_with_error), "disconnect");
       grpc_resolver_shutdown(exec_ctx, chand->resolver);
       GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
@@ -354,11 +353,11 @@ static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
   if (cpa->connected_subchannel == NULL) {
     /* cancelled, do nothing */
   } else if (error != GRPC_ERROR_NONE) {
-    grpc_exec_ctx_push(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error), NULL);
+    grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error), NULL);
   } else if (cc_pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
                                 cpa->initial_metadata_flags,
                                 cpa->connected_subchannel, cpa->on_ready)) {
-    grpc_exec_ctx_push(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE, NULL);
   }
   gpr_free(cpa);
 }
@@ -387,8 +386,8 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
       cpa = closure->cb_arg;
       if (cpa->connected_subchannel == connected_subchannel) {
         cpa->connected_subchannel = NULL;
-        grpc_exec_ctx_push(exec_ctx, cpa->on_ready,
-                           GRPC_ERROR_CREATE("Pick cancelled"), NULL);
+        grpc_exec_ctx_sched(exec_ctx, cpa->on_ready,
+                            GRPC_ERROR_CREATE("Pick cancelled"), NULL);
       }
     }
     gpr_mu_unlock(&chand->mu_config);
@@ -399,7 +398,7 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
     int r;
     GRPC_LB_POLICY_REF(lb_policy, "cc_pick_subchannel");
     gpr_mu_unlock(&chand->mu_config);
-    r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollset,
+    r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollent,
                             initial_metadata, initial_metadata_flags,
                             connected_subchannel, on_ready);
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "cc_pick_subchannel");
@@ -423,8 +422,8 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
     grpc_closure_list_append(&chand->waiting_for_config_closures, &cpa->closure,
                              GRPC_ERROR_NONE);
   } else {
-    grpc_exec_ctx_push(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected"),
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected"),
+                        NULL);
   }
   gpr_mu_unlock(&chand->mu_config);
   return 0;
@@ -439,6 +438,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats,
                               void *and_free_memory) {
   grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data);
   gpr_free(and_free_memory);
@@ -484,10 +484,11 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   gpr_mu_destroy(&chand->mu_config);
 }
 
-static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                           grpc_pollset *pollset) {
+static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                          grpc_call_element *elem,
+                                          grpc_polling_entity *pollent) {
   call_data *calld = elem->call_data;
-  calld->pollset = pollset;
+  calld->pollent = pollent;
 }
 
 const grpc_channel_filter grpc_client_channel_filter = {
@@ -495,7 +496,7 @@ const grpc_channel_filter grpc_client_channel_filter = {
     cc_start_transport_op,
     sizeof(call_data),
     init_call_elem,
-    cc_set_pollset,
+    cc_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/ext/client_config/connector.h b/src/core/ext/client_config/connector.h
index dd85dfcb7d..ea9d23706e 100644
--- a/src/core/ext/client_config/connector.h
+++ b/src/core/ext/client_config/connector.h
@@ -64,7 +64,7 @@ typedef struct {
   grpc_transport *transport;
 
   /** channel arguments (to be passed to the filters) */
-  const grpc_channel_args *channel_args;
+  grpc_channel_args *channel_args;
 } grpc_connect_out_args;
 
 struct grpc_connector_vtable {
diff --git a/src/core/ext/client_config/lb_policy.c b/src/core/ext/client_config/lb_policy.c
index fcff0c9a1b..8b980b2cca 100644
--- a/src/core/ext/client_config/lb_policy.c
+++ b/src/core/ext/client_config/lb_policy.c
@@ -60,8 +60,9 @@ static gpr_atm ref_mutate(grpc_lb_policy *c, gpr_atm delta,
                             : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
 #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-          "LB_POLICY: %p % 12s 0x%08x -> 0x%08x [%s]", c, purpose, old_val,
-          old_val + delta, reason);
+          "LB_POLICY: 0x%" PRIxPTR " %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR
+          " [%s]",
+          (intptr_t)c, purpose, old_val, old_val + delta, reason);
 #endif
   return old_val;
 }
@@ -99,12 +100,12 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx,
 }
 
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                        grpc_pollset *pollset,
+                        grpc_polling_entity *pollent,
                         grpc_metadata_batch *initial_metadata,
                         uint32_t initial_metadata_flags,
                         grpc_connected_subchannel **target,
                         grpc_closure *on_complete) {
-  return policy->vtable->pick(exec_ctx, policy, pollset, initial_metadata,
+  return policy->vtable->pick(exec_ctx, policy, pollent, initial_metadata,
                               initial_metadata_flags, target, on_complete);
 }
 
diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h
index 13b9abc474..3cfd041d3a 100644
--- a/src/core/ext/client_config/lb_policy.h
+++ b/src/core/ext/client_config/lb_policy.h
@@ -35,6 +35,7 @@
 #define GRPC_CORE_EXT_CLIENT_CONFIG_LB_POLICY_H
 
 #include "src/core/ext/client_config/subchannel.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /** A load balancing policy: specified by a vtable and a struct (which
@@ -59,7 +60,8 @@ struct grpc_lb_policy_vtable {
 
   /** implement grpc_lb_policy_pick */
   int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-              grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
+              grpc_polling_entity *pollent,
+              grpc_metadata_batch *initial_metadata,
               uint32_t initial_metadata_flags,
               grpc_connected_subchannel **target, grpc_closure *on_complete);
   void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
@@ -125,7 +127,7 @@ void grpc_lb_policy_init(grpc_lb_policy *policy,
     \a target.
     Picking can be asynchronous. Any IO should be done under \a pollset. */
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                        grpc_pollset *pollset,
+                        grpc_polling_entity *pollent,
                         grpc_metadata_batch *initial_metadata,
                         uint32_t initial_metadata_flags,
                         grpc_connected_subchannel **target,
diff --git a/src/core/ext/client_config/subchannel.c b/src/core/ext/client_config/subchannel.c
index dd61caea94..468067ea57 100644
--- a/src/core/ext/client_config/subchannel.c
+++ b/src/core/ext/client_config/subchannel.c
@@ -290,8 +290,8 @@ void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
   gpr_atm old_refs;
   old_refs = ref_mutate(c, -(gpr_atm)1, 1 REF_MUTATE_PURPOSE("WEAK_UNREF"));
   if (old_refs == 1) {
-    grpc_exec_ctx_push(exec_ctx, grpc_closure_create(subchannel_destroy, c),
-                       GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(subchannel_destroy, c),
+                        GRPC_ERROR_NONE, NULL);
   }
 }
 
@@ -481,12 +481,12 @@ static void subchannel_on_child_state_changed(grpc_exec_ctx *exec_ctx, void *p,
   /* if we failed just leave this closure */
   if (sw->connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
     /* any errors on a subchannel ==> we're done, create a new one */
-    sw->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE;
+    sw->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
   }
   grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
                               sw->connectivity_state, GRPC_ERROR_REF(error),
                               "reflect_child");
-  if (sw->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) {
+  if (sw->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
     grpc_connected_subchannel_notify_on_state_change(
         exec_ctx, GET_CONNECTED_SUBCHANNEL(c, no_barrier), NULL,
         &sw->connectivity_state, &sw->closure);
@@ -621,6 +621,7 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
                                  grpc_error *error) {
   grpc_subchannel *c = arg;
+  grpc_channel_args *delete_channel_args = c->connecting_result.channel_args;
 
   GRPC_SUBCHANNEL_WEAK_REF(c, "connected");
   gpr_mu_lock(&c->mu);
@@ -638,13 +639,20 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
         "connect_failed");
     gpr_timespec time_til_next = gpr_time_sub(c->next_attempt, now);
     const char *errmsg = grpc_error_string(error);
-    gpr_log(GPR_INFO, "Connect failed, retry in %d.%09d seconds: %s",
-            time_til_next.tv_sec, time_til_next.tv_nsec, errmsg);
-    grpc_error_free_string(errmsg);
+    gpr_log(GPR_INFO, "Connect failed: %s", errmsg);
+    if (gpr_time_cmp(time_til_next, gpr_time_0(time_til_next.clock_type)) <=
+        0) {
+      gpr_log(GPR_INFO, "Retry immediately");
+    } else {
+      gpr_log(GPR_INFO, "Retry in %" PRId64 ".%09d seconds",
+              time_til_next.tv_sec, time_til_next.tv_nsec);
+    }
     grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now);
+    grpc_error_free_string(errmsg);
   }
   gpr_mu_unlock(&c->mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
+  grpc_channel_args_destroy(delete_channel_args);
 }
 
 /*
@@ -656,7 +664,7 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call,
   grpc_subchannel_call *c = call;
   GPR_TIMER_BEGIN("grpc_subchannel_call_unref.destroy", 0);
   grpc_connected_subchannel *connection = c->connection;
-  grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), c);
+  grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), NULL, c);
   GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, connection, "subchannel_call");
   GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0);
 }
@@ -694,7 +702,7 @@ grpc_connected_subchannel *grpc_subchannel_get_connected_subchannel(
 
 grpc_subchannel_call *grpc_connected_subchannel_create_call(
     grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *con,
-    grpc_pollset *pollset) {
+    grpc_polling_entity *pollent) {
   grpc_channel_stack *chanstk = CHANNEL_STACK_FROM_CONNECTION(con);
   grpc_subchannel_call *call =
       gpr_malloc(sizeof(grpc_subchannel_call) + chanstk->call_stack_size);
@@ -703,7 +711,7 @@ grpc_subchannel_call *grpc_connected_subchannel_create_call(
   GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call");
   grpc_call_stack_init(exec_ctx, chanstk, 1, subchannel_call_destroy, call,
                        NULL, NULL, callstk);
-  grpc_call_stack_set_pollset(exec_ctx, callstk, pollset);
+  grpc_call_stack_set_pollset_or_pollset_set(exec_ctx, callstk, pollent);
   return call;
 }
 
diff --git a/src/core/ext/client_config/subchannel.h b/src/core/ext/client_config/subchannel.h
index e73f394584..b6d39f5dc5 100644
--- a/src/core/ext/client_config/subchannel.h
+++ b/src/core/ext/client_config/subchannel.h
@@ -36,6 +36,7 @@
 
 #include "src/core/ext/client_config/connector.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /** A (sub-)channel that knows how to connect to exactly one target
@@ -109,7 +110,7 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
 /** construct a subchannel call */
 grpc_subchannel_call *grpc_connected_subchannel_create_call(
     grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *connected_subchannel,
-    grpc_pollset *pollset);
+    grpc_polling_entity *pollent);
 
 /** process a transport level op */
 void grpc_connected_subchannel_process_transport_op(
diff --git a/src/core/ext/client_config/subchannel_call_holder.c b/src/core/ext/client_config/subchannel_call_holder.c
index 07b7813482..e31800edd9 100644
--- a/src/core/ext/client_config/subchannel_call_holder.c
+++ b/src/core/ext/client_config/subchannel_call_holder.c
@@ -68,6 +68,7 @@ void grpc_subchannel_call_holder_init(
   holder->waiting_ops_capacity = 0;
   holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
   holder->owning_call = owning_call;
+  holder->pollent = NULL;
 }
 
 void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
@@ -163,7 +164,7 @@ retry:
     gpr_atm_rel_store(
         &holder->subchannel_call,
         (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
-            exec_ctx, holder->connected_subchannel, holder->pollset));
+            exec_ctx, holder->connected_subchannel, holder->pollent));
     retry_waiting_locked(exec_ctx, holder);
     goto retry;
   }
@@ -194,7 +195,7 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
     gpr_atm_rel_store(
         &holder->subchannel_call,
         (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
-            exec_ctx, holder->connected_subchannel, holder->pollset));
+            exec_ctx, holder->connected_subchannel, holder->pollent));
     retry_waiting_locked(exec_ctx, holder);
   }
   gpr_mu_unlock(&holder->mu);
@@ -222,8 +223,8 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
   holder->waiting_ops_count = 0;
   holder->waiting_ops_capacity = 0;
   GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
-  grpc_exec_ctx_push(exec_ctx, grpc_closure_create(retry_ops, a),
-                     GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
+                      GRPC_ERROR_NONE, NULL);
 }
 
 static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
diff --git a/src/core/ext/client_config/subchannel_call_holder.h b/src/core/ext/client_config/subchannel_call_holder.h
index 9299908788..8d2deb02f3 100644
--- a/src/core/ext/client_config/subchannel_call_holder.h
+++ b/src/core/ext/client_config/subchannel_call_holder.h
@@ -35,6 +35,7 @@
 #define GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H
 
 #include "src/core/ext/client_config/subchannel.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 
 /** Pick a subchannel for grpc_subchannel_call_holder;
     Return 1 if subchannel is available immediately (in which case on_ready
@@ -71,7 +72,7 @@ typedef struct grpc_subchannel_call_holder {
 
   grpc_subchannel_call_holder_creation_phase creation_phase;
   grpc_connected_subchannel *connected_subchannel;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
 
   grpc_transport_stream_op *waiting_ops;
   size_t waiting_ops_count;
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
index d5dc39ab94..46fe588f72 100644
--- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
+++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
@@ -33,8 +33,8 @@
 /* Automatically generated nanopb header */
 /* Generated by nanopb-0.3.5-dev */
 
-#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED
-#define PB_LOAD_BALANCER_PB_H_INCLUDED
+#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
+#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
 #include "third_party/nanopb/pb.h"
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.
diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c
index b5c2c426de..9decf70692 100644
--- a/src/core/ext/lb_policy/pick_first/pick_first.c
+++ b/src/core/ext/lb_policy/pick_first/pick_first.c
@@ -39,7 +39,7 @@
 
 typedef struct pending_pick {
   struct pending_pick *next;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   uint32_t initial_metadata_flags;
   grpc_connected_subchannel **target;
   grpc_closure *on_complete;
@@ -104,7 +104,7 @@ static void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   pp = p->pending_picks;
   p->pending_picks = NULL;
   grpc_connectivity_state_set(
-      exec_ctx, &p->state_tracker, GRPC_CHANNEL_FATAL_FAILURE,
+      exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
       GRPC_ERROR_CREATE("Channel shutdown"), "shutdown");
   /* cancel subscription */
   if (selected != NULL) {
@@ -119,9 +119,9 @@ static void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while (pp != NULL) {
     pending_pick *next = pp->next;
     *pp->target = NULL;
-    grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                 pp->pollset);
-    grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+    grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                             p->base.interested_parties);
+    grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
     gpr_free(pp);
     pp = next;
   }
@@ -137,11 +137,11 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   while (pp != NULL) {
     pending_pick *next = pp->next;
     if (pp->target == target) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
       *target = NULL;
-      grpc_exec_ctx_push(exec_ctx, pp->on_complete,
-                         GRPC_ERROR_CREATE("Pick Cancelled"), NULL);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
+                          GRPC_ERROR_CREATE("Pick Cancelled"), NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -164,10 +164,10 @@ static void pf_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
-      grpc_exec_ctx_push(exec_ctx, pp->on_complete,
-                         GRPC_ERROR_CREATE("Pick Cancelled"), NULL);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
+                          GRPC_ERROR_CREATE("Pick Cancelled"), NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -199,7 +199,8 @@ static void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                   grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
+                   grpc_polling_entity *pollent,
+                   grpc_metadata_batch *initial_metadata,
                    uint32_t initial_metadata_flags,
                    grpc_connected_subchannel **target,
                    grpc_closure *on_complete) {
@@ -224,10 +225,11 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset);
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+                                           p->base.interested_parties);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
-    pp->pollset = pollset;
+    pp->pollent = pollent;
     pp->target = target;
     pp->initial_metadata_flags = initial_metadata_flags;
     pp->on_complete = on_complete;
@@ -279,12 +281,12 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
   } else if (selected != NULL) {
     if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) {
       /* if the selected channel goes bad, we're done */
-      p->checking_connectivity = GRPC_CHANNEL_FATAL_FAILURE;
+      p->checking_connectivity = GRPC_CHANNEL_SHUTDOWN;
     }
     grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
                                 p->checking_connectivity, GRPC_ERROR_REF(error),
                                 "selected_changed");
-    if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) {
+    if (p->checking_connectivity != GRPC_CHANNEL_SHUTDOWN) {
       grpc_connected_subchannel_notify_on_state_change(
           exec_ctx, selected, p->base.interested_parties,
           &p->checking_connectivity, &p->connectivity_changed);
@@ -306,16 +308,16 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         /* drop the pick list: we are connected now */
         GRPC_LB_POLICY_WEAK_REF(&p->base, "destroy_subchannels");
         gpr_atm_rel_store(&p->selected, (gpr_atm)selected);
-        grpc_exec_ctx_push(exec_ctx,
-                           grpc_closure_create(destroy_subchannels, p),
-                           GRPC_ERROR_NONE, NULL);
+        grpc_exec_ctx_sched(exec_ctx,
+                            grpc_closure_create(destroy_subchannels, p),
+                            GRPC_ERROR_NONE, NULL);
         /* update any calls that were waiting for a pick */
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
           *pp->target = selected;
-          grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                       pp->pollset);
-          grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+          grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                                   p->base.interested_parties);
+          grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
           gpr_free(pp);
         }
         grpc_connected_subchannel_notify_on_state_change(
@@ -353,7 +355,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
             p->base.interested_parties, &p->checking_connectivity,
             &p->connectivity_changed);
         break;
-      case GRPC_CHANNEL_FATAL_FAILURE:
+      case GRPC_CHANNEL_SHUTDOWN:
         p->num_subchannels--;
         GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel],
                  p->subchannels[p->num_subchannels]);
@@ -361,15 +363,15 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                               "pick_first");
         if (p->num_subchannels == 0) {
           grpc_connectivity_state_set(
-              exec_ctx, &p->state_tracker, GRPC_CHANNEL_FATAL_FAILURE,
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
               GRPC_ERROR_CREATE_REFERENCING("Pick first exhausted channels",
                                             &error, 1),
               "no_more_channels");
           while ((pp = p->pending_picks)) {
             p->pending_picks = pp->next;
             *pp->target = NULL;
-            grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
-                               NULL);
+            grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
+                                NULL);
             gpr_free(pp);
           }
           GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base,
@@ -421,8 +423,8 @@ static void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   if (selected) {
     grpc_connected_subchannel_ping(exec_ctx, selected, closure);
   } else {
-    grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_CREATE("Not connected"),
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("Not connected"),
+                        NULL);
   }
 }
 
diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c
index 95305e9cdd..7bcf608ab9 100644
--- a/src/core/ext/lb_policy/round_robin/round_robin.c
+++ b/src/core/ext/lb_policy/round_robin/round_robin.c
@@ -31,6 +31,34 @@
  *
  */
 
+/** Round Robin Policy.
+ *
+ * This policy keeps:
+ * - A circular list of ready (connected) subchannels, the *readylist*. An empty
+ *   readylist consists solely of its root (dummy) node.
+ * - A pointer to the last element picked from the readylist, the *lastpick*.
+ *   Initially set to point to the readylist's root.
+ *
+ * Behavior:
+ * - When a subchannel connects, it's *prepended* to the readylist's root node.
+ *   Ie, if readylist = A <-> B <-> ROOT <-> C
+ *                      ^                    ^
+ *                      |____________________|
+ *   and subchannel D becomes connected, the addition of D to the readylist
+ *   results in  readylist = A <-> B <-> D <-> ROOT <-> C
+ *                           ^                          ^
+ *                           |__________________________|
+ * - When a subchannel disconnects, it's removed from the readylist. If the
+ *   subchannel being removed was the most recently picked, the *lastpick*
+ *   pointer moves to the removed node's previous element. Note that if the
+ *   readylist only had one element, this is still legal, as the lastpick would
+ *   point to the dummy root node, for an empty readylist.
+ * - Upon picking, *lastpick* is updated to point to the returned (connected)
+ *   subchannel. Note that it's possible that the selected subchannel becomes
+ *   disconnected in the interim between the selection and the actual usage of
+ *   the subchannel by the caller.
+ */
+
 #include <string.h>
 
 #include <grpc/support/alloc.h>
@@ -48,7 +76,7 @@ int grpc_lb_round_robin_trace = 0;
  * Once a pick is available, \a target is updated and \a on_complete called. */
 typedef struct pending_pick {
   struct pending_pick *next;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   uint32_t initial_metadata_flags;
   grpc_connected_subchannel **target;
   grpc_closure *on_complete;
@@ -173,9 +201,7 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p,
     return;
   }
   if (node == p->ready_list_last_pick) {
-    /* If removing the lastly picked node, reset the last pick pointer to the
-     * dummy root of the list */
-    p->ready_list_last_pick = &p->ready_list;
+    p->ready_list_last_pick = p->ready_list_last_pick->prev;
   }
 
   /* removing last item */
@@ -239,12 +265,12 @@ static void rr_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while ((pp = p->pending_picks)) {
     p->pending_picks = pp->next;
     *pp->target = NULL;
-    grpc_exec_ctx_push(exec_ctx, pp->on_complete,
-                       GRPC_ERROR_CREATE("Channel Shutdown"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
+                        GRPC_ERROR_CREATE("Channel Shutdown"), NULL);
     gpr_free(pp);
   }
   grpc_connectivity_state_set(
-      exec_ctx, &p->state_tracker, GRPC_CHANNEL_FATAL_FAILURE,
+      exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
       GRPC_ERROR_CREATE("Channel Shutdown"), "shutdown");
   for (i = 0; i < p->num_subchannels; i++) {
     subchannel_data *sd = p->subchannels[i];
@@ -264,10 +290,11 @@ static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   while (pp != NULL) {
     pending_pick *next = pp->next;
     if (pp->target == target) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
       *target = NULL;
-      grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_CANCELLED, NULL);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_CANCELLED,
+                          NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -290,10 +317,11 @@ static void rr_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
       *pp->target = NULL;
-      grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_CANCELLED, NULL);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_CANCELLED,
+                          NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -309,7 +337,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) {
   p->started_picking = 1;
 
   if (grpc_lb_round_robin_trace) {
-    gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%d", p,
+    gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%" PRIuPTR, p,
             p->num_subchannels);
   }
 
@@ -333,7 +361,8 @@ static void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                   grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
+                   grpc_polling_entity *pollent,
+                   grpc_metadata_batch *initial_metadata,
                    uint32_t initial_metadata_flags,
                    grpc_connected_subchannel **target,
                    grpc_closure *on_complete) {
@@ -346,8 +375,8 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     *target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
     if (grpc_lb_round_robin_trace) {
       gpr_log(GPR_DEBUG,
-              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)",
-              selected->subchannel, selected);
+              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", *target,
+              selected);
     }
     /* only advance the last picked pointer if the selection was used */
     advance_last_picked_locked(p);
@@ -356,10 +385,11 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset);
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+                                           p->base.interested_parties);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
-    pp->pollset = pollset;
+    pp->pollent = pollent;
     pp->target = target;
     pp->on_complete = on_complete;
     pp->initial_metadata_flags = initial_metadata_flags;
@@ -410,9 +440,9 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                     "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)",
                     selected->subchannel, selected);
           }
-          grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                       pp->pollset);
-          grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+          grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                                   p->base.interested_parties);
+          grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
           gpr_free(pp);
         }
         grpc_subchannel_notify_on_state_change(
@@ -443,7 +473,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
             exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
             GRPC_ERROR_REF(error), "connecting_transient_failure");
         break;
-      case GRPC_CHANNEL_FATAL_FAILURE:
+      case GRPC_CHANNEL_SHUTDOWN:
         if (sd->ready_list_node != NULL) {
           remove_disconnected_sc_locked(p, sd->ready_list_node);
           sd->ready_list_node = NULL;
@@ -459,15 +489,15 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         unref = 1;
         if (p->num_subchannels == 0) {
           grpc_connectivity_state_set(
-              exec_ctx, &p->state_tracker, GRPC_CHANNEL_FATAL_FAILURE,
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
               GRPC_ERROR_CREATE_REFERENCING("Round Robin Channels Exhausted",
                                             &error, 1),
               "no_more_channels");
           while ((pp = p->pending_picks)) {
             p->pending_picks = pp->next;
             *pp->target = NULL;
-            grpc_exec_ctx_push(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
-                               NULL);
+            grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
+                                NULL);
             gpr_free(pp);
           }
         } else {
@@ -521,8 +551,8 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     grpc_connected_subchannel_ping(exec_ctx, target, closure);
   } else {
     gpr_mu_unlock(&p->mu);
-    grpc_exec_ctx_push(exec_ctx, closure,
-                       GRPC_ERROR_CREATE("Round Robin not connected"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure,
+                        GRPC_ERROR_CREATE("Round Robin not connected"), NULL);
   }
 }
 
@@ -535,7 +565,7 @@ static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {}
 
 static void round_robin_factory_unref(grpc_lb_policy_factory *factory) {}
 
-static grpc_lb_policy *create_round_robin(grpc_exec_ctx *exec_ctx,
+static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
                                           grpc_lb_policy_factory *factory,
                                           grpc_lb_policy_args *args) {
   GPR_ASSERT(args->addresses != NULL);
@@ -591,7 +621,7 @@ static grpc_lb_policy *create_round_robin(grpc_exec_ctx *exec_ctx,
 }
 
 static const grpc_lb_policy_factory_vtable round_robin_factory_vtable = {
-    round_robin_factory_ref, round_robin_factory_unref, create_round_robin,
+    round_robin_factory_ref, round_robin_factory_unref, round_robin_create,
     "round_robin"};
 
 static grpc_lb_policy_factory round_robin_lb_policy_factory = {
diff --git a/src/core/ext/load_reporting/load_reporting.c b/src/core/ext/load_reporting/load_reporting.c
new file mode 100644
index 0000000000..9e4d32676f
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting.c
@@ -0,0 +1,133 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <limits.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/sync.h>
+
+#include "src/core/ext/load_reporting/load_reporting.h"
+#include "src/core/ext/load_reporting/load_reporting_filter.h"
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_init.h"
+
+struct grpc_load_reporting_config {
+  grpc_load_reporting_fn fn;
+  void *user_data;
+};
+
+grpc_load_reporting_config *grpc_load_reporting_config_create(
+    grpc_load_reporting_fn fn, void *user_data) {
+  GPR_ASSERT(fn != NULL);
+  grpc_load_reporting_config *lrc =
+      gpr_malloc(sizeof(grpc_load_reporting_config));
+  lrc->fn = fn;
+  lrc->user_data = user_data;
+  return lrc;
+}
+
+grpc_load_reporting_config *grpc_load_reporting_config_copy(
+    grpc_load_reporting_config *src) {
+  return grpc_load_reporting_config_create(src->fn, src->user_data);
+}
+
+void grpc_load_reporting_config_destroy(grpc_load_reporting_config *lrc) {
+  gpr_free(lrc);
+}
+
+void grpc_load_reporting_config_call(
+    grpc_load_reporting_config *lrc,
+    const grpc_load_reporting_call_data *call_data) {
+  lrc->fn(call_data, lrc->user_data);
+}
+
+static bool is_load_reporting_enabled(const grpc_channel_args *a) {
+  if (a == NULL) return false;
+  for (size_t i = 0; i < a->num_args; i++) {
+    if (0 == strcmp(a->args[i].key, GRPC_ARG_ENABLE_LOAD_REPORTING)) {
+      return a->args[i].type == GRPC_ARG_POINTER &&
+             a->args[i].value.pointer.p != NULL;
+    }
+  }
+  return false;
+}
+
+static bool maybe_add_load_reporting_filter(grpc_channel_stack_builder *builder,
+                                            void *arg) {
+  const grpc_channel_args *args =
+      grpc_channel_stack_builder_get_channel_arguments(builder);
+  if (is_load_reporting_enabled(args)) {
+    return grpc_channel_stack_builder_prepend_filter(
+        builder, (const grpc_channel_filter *)arg, NULL, NULL);
+  }
+  return true;
+}
+
+static void lrd_arg_destroy(void *p) { grpc_load_reporting_config_destroy(p); }
+
+static void *lrd_arg_copy(void *p) {
+  return grpc_load_reporting_config_copy(p);
+}
+
+static int lrd_arg_cmp(void *a, void *b) {
+  grpc_load_reporting_config *lhs = a;
+  grpc_load_reporting_config *rhs = b;
+  return !(lhs->fn == rhs->fn && lhs->user_data == rhs->user_data);
+}
+
+static const grpc_arg_pointer_vtable lrd_ptr_vtable = {
+    lrd_arg_copy, lrd_arg_destroy, lrd_arg_cmp};
+
+grpc_arg grpc_load_reporting_config_create_arg(
+    grpc_load_reporting_config *lrc) {
+  grpc_arg arg;
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_ARG_ENABLE_LOAD_REPORTING;
+  arg.value.pointer.p = lrc;
+  arg.value.pointer.vtable = &lrd_ptr_vtable;
+  return arg;
+}
+
+/* Plugin registration */
+
+void grpc_load_reporting_plugin_init(void) {
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
+                                   maybe_add_load_reporting_filter,
+                                   (void *)&grpc_load_reporting_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_load_reporting_filter,
+                                   (void *)&grpc_load_reporting_filter);
+}
+
+void grpc_load_reporting_plugin_shutdown() {}
diff --git a/src/core/ext/load_reporting/load_reporting.h b/src/core/ext/load_reporting/load_reporting.h
new file mode 100644
index 0000000000..316cd89bd7
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting.h
@@ -0,0 +1,75 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H
+#define GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H
+
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/surface/call.h"
+
+typedef struct grpc_load_reporting_config grpc_load_reporting_config;
+
+/** Call information to be passed to the provided load reporting function upon
+ * completion of the call */
+typedef struct grpc_load_reporting_call_data {
+  const grpc_call_stats *stats;   /**< Stats for the call */
+  const char *trailing_md_string; /**< LR trailing metadata info */
+} grpc_load_reporting_call_data;
+
+/** Custom function to be called by the load reporting filter. */
+typedef void (*grpc_load_reporting_fn)(
+    const grpc_load_reporting_call_data *call_data, void *user_data);
+
+/** Register \a fn as the function to be invoked by the load reporting filter.
+ * \a fn will be invoked at the beginning and at the end of the call.
+ *
+ * For the first invocation, \a fn's first argument
+ * (grpc_load_reporting_call_data*) will be NULL. \a user_data is always passed
+ * as-is. */
+grpc_load_reporting_config *grpc_load_reporting_config_create(
+    grpc_load_reporting_fn fn, void *user_data);
+
+grpc_load_reporting_config *grpc_load_reporting_config_copy(
+    grpc_load_reporting_config *src);
+
+void grpc_load_reporting_config_destroy(grpc_load_reporting_config *lrc);
+
+/** Invoke the function registered by \a grpc_load_reporting_init. */
+void grpc_load_reporting_config_call(
+    grpc_load_reporting_config *lrc,
+    const grpc_load_reporting_call_data *call_data);
+
+/** Return a \a grpc_arg enabling load reporting */
+grpc_arg grpc_load_reporting_config_create_arg(grpc_load_reporting_config *lrc);
+
+#endif /* GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H */
diff --git a/src/core/ext/load_reporting/load_reporting_filter.c b/src/core/ext/load_reporting/load_reporting_filter.c
new file mode 100644
index 0000000000..f372f88c3a
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting_filter.c
@@ -0,0 +1,151 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <string.h>
+
+#include "src/core/ext/load_reporting/load_reporting.h"
+#include "src/core/ext/load_reporting/load_reporting_filter.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+typedef struct call_data { const char *trailing_md_string; } call_data;
+typedef struct channel_data {
+  gpr_mu mu;
+  grpc_load_reporting_config *lrc;
+} channel_data;
+
+static void invoke_lr_fn_locked(grpc_load_reporting_config *lrc,
+                                grpc_load_reporting_call_data *lr_call_data) {
+  GPR_TIMER_BEGIN("load_reporting_config_fn", 0);
+  grpc_load_reporting_config_call(lrc, lr_call_data);
+  GPR_TIMER_END("load_reporting_config_fn", 0);
+}
+
+/* Constructor for call_data */
+static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                           grpc_call_element_args *args) {
+  call_data *calld = elem->call_data;
+  memset(calld, 0, sizeof(call_data));
+}
+
+/* Destructor for call_data */
+static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats, void *ignored) {
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+
+  grpc_load_reporting_call_data lr_call_data = {stats,
+                                                calld->trailing_md_string};
+
+  gpr_mu_lock(&chand->mu);
+  invoke_lr_fn_locked(chand->lrc, &lr_call_data);
+  gpr_mu_unlock(&chand->mu);
+}
+
+/* Constructor for channel_data */
+static void init_channel_elem(grpc_exec_ctx *exec_ctx,
+                              grpc_channel_element *elem,
+                              grpc_channel_element_args *args) {
+  GPR_ASSERT(!args->is_last);
+
+  channel_data *chand = elem->channel_data;
+  memset(chand, 0, sizeof(channel_data));
+
+  gpr_mu_init(&chand->mu);
+  for (size_t i = 0; i < args->channel_args->num_args; i++) {
+    if (0 == strcmp(args->channel_args->args[i].key,
+                    GRPC_ARG_ENABLE_LOAD_REPORTING)) {
+      grpc_load_reporting_config *arg_lrc =
+          args->channel_args->args[i].value.pointer.p;
+      chand->lrc = grpc_load_reporting_config_copy(arg_lrc);
+      GPR_ASSERT(chand->lrc != NULL);
+      break;
+    }
+  }
+  GPR_ASSERT(chand->lrc != NULL); /* arg actually found */
+
+  gpr_mu_lock(&chand->mu);
+  invoke_lr_fn_locked(chand->lrc, NULL);
+  gpr_mu_unlock(&chand->mu);
+}
+
+/* Destructor for channel data */
+static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_element *elem) {
+  channel_data *chand = elem->channel_data;
+  gpr_mu_destroy(&chand->mu);
+  grpc_load_reporting_config_destroy(chand->lrc);
+}
+
+static grpc_mdelem *lr_trailing_md_filter(void *user_data, grpc_mdelem *md) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+
+  if (md->key == GRPC_MDSTR_LOAD_REPORTING) {
+    calld->trailing_md_string = gpr_strdup(grpc_mdstr_as_c_string(md->value));
+    return NULL;
+  }
+
+  return md;
+}
+
+static void lr_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
+                                         grpc_call_element *elem,
+                                         grpc_transport_stream_op *op) {
+  GPR_TIMER_BEGIN("lr_start_transport_stream_op", 0);
+
+  if (op->send_trailing_metadata) {
+    grpc_metadata_batch_filter(op->send_trailing_metadata,
+                               lr_trailing_md_filter, elem);
+  }
+  grpc_call_next_op(exec_ctx, elem, op);
+
+  GPR_TIMER_END("lr_start_transport_stream_op", 0);
+}
+
+const grpc_channel_filter grpc_load_reporting_filter = {
+    lr_start_transport_stream_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "load_reporting"};
diff --git a/src/core/ext/load_reporting/load_reporting_filter.h b/src/core/ext/load_reporting/load_reporting_filter.h
new file mode 100644
index 0000000000..f69cd6fdc6
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting_filter.h
@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_FILTER_H
+#define GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_FILTER_H
+
+#include "src/core/lib/channel/channel_stack.h"
+
+extern const grpc_channel_filter grpc_load_reporting_filter;
+
+#endif /* GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_FILTER_H */
diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c
index adb9c3af84..31ac968670 100644
--- a/src/core/ext/resolver/dns/native/dns_resolver.c
+++ b/src/core/ext/resolver/dns/native/dns_resolver.c
@@ -111,8 +111,8 @@ static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
   }
   if (r->next_completion != NULL) {
     *r->target_config = NULL;
-    grpc_exec_ctx_push(exec_ctx, r->next_completion,
-                       GRPC_ERROR_CREATE("Resolver Shutdown"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion,
+                        GRPC_ERROR_CREATE("Resolver Shutdown"), NULL);
     r->next_completion = NULL;
   }
   gpr_mu_unlock(&r->mu);
@@ -189,12 +189,17 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
     gpr_timespec next_try = gpr_backoff_step(&r->backoff_state, now);
     gpr_timespec timeout = gpr_time_sub(next_try, now);
     const char *msg = grpc_error_string(error);
-    gpr_log(GPR_DEBUG, "dns resolution failed: retrying in %d.%09d seconds: %s",
-            timeout.tv_sec, timeout.tv_nsec, msg);
+    gpr_log(GPR_DEBUG, "dns resolution failed: %s", msg);
     grpc_error_free_string(msg);
     GPR_ASSERT(!r->have_retry_timer);
     r->have_retry_timer = true;
     GRPC_RESOLVER_REF(&r->base, "retry-timer");
+    if (gpr_time_cmp(timeout, gpr_time_0(timeout.clock_type)) <= 0) {
+      gpr_log(GPR_DEBUG, "retrying in %" PRId64 ".%09d seconds", timeout.tv_sec,
+              timeout.tv_nsec);
+    } else {
+      gpr_log(GPR_DEBUG, "retrying immediately");
+    }
     grpc_timer_init(exec_ctx, &r->retry_timer, next_try, dns_on_retry_timer, r,
                     now);
   }
@@ -227,7 +232,7 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
     if (r->resolved_config) {
       grpc_client_config_ref(r->resolved_config);
     }
-    grpc_exec_ctx_push(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
     r->published_version = r->resolved_version;
   }
diff --git a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
index 90da6a93a0..1f7cce2f43 100644
--- a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
+++ b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
@@ -92,7 +92,7 @@ static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&r->mu);
   if (r->next_completion != NULL) {
     *r->target_config = NULL;
-    grpc_exec_ctx_push(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
   gpr_mu_unlock(&r->mu);
@@ -133,7 +133,7 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "sockaddr");
     r->published = 1;
     *r->target_config = cfg;
-    grpc_exec_ctx_push(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
 }
diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c
index 28d1be878f..85f9efb3b6 100644
--- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c
+++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c
@@ -103,13 +103,13 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
     grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL,
                                         0);
     GPR_ASSERT(c->result->transport);
-    c->result->channel_args = c->args.channel_args;
+    c->result->channel_args = grpc_channel_args_copy(c->args.channel_args);
   } else {
     memset(c->result, 0, sizeof(*c->result));
   }
   notify = c->notify;
   c->notify = NULL;
-  notify->cb(exec_ctx, notify->cb_arg, error);
+  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
 }
 
 static void connector_shutdown(grpc_exec_ctx *exec_ctx, grpc_connector *con) {}
diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
new file mode 100644
index 0000000000..ca435c25ce
--- /dev/null
+++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
@@ -0,0 +1,95 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_SUPPORT_CHANNELS_FROM_FD
+
+#include <fcntl.h>
+
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/transport.h"
+
+grpc_channel *grpc_insecure_channel_create_from_fd(
+    const char *target, int fd, const grpc_channel_args *args) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GRPC_API_TRACE("grpc_insecure_channel_create(target=%p, fd=%d, args=%p)", 3,
+                 (target, fd, args));
+
+  grpc_arg default_authority_arg;
+  default_authority_arg.type = GRPC_ARG_STRING;
+  default_authority_arg.key = GRPC_ARG_DEFAULT_AUTHORITY;
+  default_authority_arg.value.string = "test.authority";
+  grpc_channel_args *final_args =
+      grpc_channel_args_copy_and_add(args, &default_authority_arg, 1);
+
+  int flags = fcntl(fd, F_GETFL, 0);
+  GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0);
+
+  grpc_endpoint *client =
+      grpc_tcp_create(grpc_fd_create(fd, "client"),
+                      GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "fd-client");
+
+  grpc_transport *transport =
+      grpc_create_chttp2_transport(&exec_ctx, final_args, client, 1);
+  GPR_ASSERT(transport);
+  grpc_channel *channel = grpc_channel_create(
+      &exec_ctx, target, final_args, GRPC_CLIENT_DIRECT_CHANNEL, transport);
+  grpc_channel_args_destroy(final_args);
+  grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0);
+
+  grpc_exec_ctx_finish(&exec_ctx);
+
+  return channel != NULL ? channel : grpc_lame_client_channel_create(
+                                         target, GRPC_STATUS_INTERNAL,
+                                         "Failed to create client channel");
+}
+
+#else  // !GPR_SUPPORT_CHANNELS_FROM_FD
+
+grpc_channel *grpc_insecure_channel_create_from_fd(
+    const char *target, int fd, const grpc_channel_args *args) {
+  GPR_ASSERT(0);
+  return NULL;
+}
+
+#endif  // GPR_SUPPORT_CHANNELS_FROM_FD
diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
index bceef152be..721ba82d8f 100644
--- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
+++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
@@ -90,7 +90,6 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
                                      grpc_auth_context *auth_context) {
   connector *c = arg;
   grpc_closure *notify;
-  grpc_channel_args *args_copy = NULL;
   gpr_mu_lock(&c->mu);
   if (c->connecting_endpoint == NULL) {
     memset(c->result, 0, sizeof(*c->result));
@@ -109,23 +108,20 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL,
                                         0);
     auth_context_arg = grpc_auth_context_to_arg(auth_context);
-    args_copy = grpc_channel_args_copy_and_add(c->args.channel_args,
-                                               &auth_context_arg, 1);
-    c->result->channel_args = args_copy;
+    c->result->channel_args = grpc_channel_args_copy_and_add(
+        c->args.channel_args, &auth_context_arg, 1);
   }
   notify = c->notify;
   c->notify = NULL;
-  /* look at c->args which are connector args. */
-  notify->cb(exec_ctx, notify->cb_arg, GRPC_ERROR_NONE);
-  if (args_copy != NULL) grpc_channel_args_destroy(args_copy);
+  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_NONE, NULL);
 }
 
 static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
                                            grpc_error *error) {
   connector *c = arg;
-  grpc_channel_security_connector_do_handshake(exec_ctx, c->security_connector,
-                                               c->connecting_endpoint,
-                                               on_secure_handshake_done, c);
+  grpc_channel_security_connector_do_handshake(
+      exec_ctx, c->security_connector, c->connecting_endpoint, c->args.deadline,
+      on_secure_handshake_done, c);
 }
 
 static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
@@ -147,13 +143,14 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
                           &c->initial_string_sent);
     } else {
       grpc_channel_security_connector_do_handshake(
-          exec_ctx, c->security_connector, tcp, on_secure_handshake_done, c);
+          exec_ctx, c->security_connector, tcp, c->args.deadline,
+          on_secure_handshake_done, c);
     }
   } else {
     memset(c->result, 0, sizeof(*c->result));
     notify = c->notify;
     c->notify = NULL;
-    notify->cb(exec_ctx, notify->cb_arg, GRPC_ERROR_NONE);
+    grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
   }
 }
 
@@ -175,7 +172,6 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con,
                               grpc_closure *notify) {
   connector *c = (connector *)con;
   GPR_ASSERT(c->notify == NULL);
-  GPR_ASSERT(notify->cb);
   c->notify = notify;
   c->args = *args;
   c->result = result;
diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
index 9efc922a8d..e5c987925c 100644
--- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -75,14 +75,14 @@ static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
                     grpc_closure *destroy_done) {
   grpc_tcp_server *tcp = tcpp;
   grpc_tcp_server_unref(exec_ctx, tcp);
-  grpc_exec_ctx_push(exec_ctx, destroy_done, GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, destroy_done, GRPC_ERROR_NONE, NULL);
 }
 
 int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
   size_t i;
-  unsigned count = 0;
+  size_t count = 0;
   int port_num = -1;
   int port_temp;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -120,13 +120,15 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
   }
   if (count == 0) {
     char *msg;
-    gpr_asprintf(&msg, "No address added out of total %d resolved", naddrs);
+    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
+                 naddrs);
     err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
     gpr_free(msg);
     goto error;
   } else if (count != naddrs) {
     char *msg;
-    gpr_asprintf(&msg, "Only %d addresses added out of total %d resolved",
+    gpr_asprintf(&msg, "Only %" PRIuPTR
+                       " addresses added out of total %" PRIuPTR " resolved",
                  count, naddrs);
     err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
     gpr_free(msg);
@@ -153,6 +155,11 @@ error:
   }
   port_num = 0;
 
+  const char *msg = grpc_error_string(err);
+  gpr_log(GPR_ERROR, "%s", msg);
+  grpc_error_free_string(msg);
+  GRPC_ERROR_UNREF(err);
+
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   if (errors != NULL) {
@@ -160,7 +167,6 @@ done:
       GRPC_ERROR_UNREF(errors[i]);
     }
   }
-  GRPC_ERROR_UNREF(err);
   gpr_free(errors);
   return port_num;
 }
diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
new file mode 100644
index 0000000000..96bf4d6f30
--- /dev/null
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
@@ -0,0 +1,82 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_SUPPORT_CHANNELS_FROM_FD
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/server.h"
+
+void grpc_server_add_insecure_channel_from_fd(grpc_server *server,
+                                              grpc_completion_queue *cq,
+                                              int fd) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+  char *name;
+  gpr_asprintf(&name, "fd:%d", fd);
+
+  grpc_endpoint *server_endpoint = grpc_tcp_create(
+      grpc_fd_create(fd, name), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name);
+
+  gpr_free(name);
+
+  const grpc_channel_args *server_args = grpc_server_get_channel_args(server);
+  grpc_transport *transport = grpc_create_chttp2_transport(
+      &exec_ctx, server_args, server_endpoint, 0 /* is_client */);
+  grpc_endpoint_add_to_pollset(&exec_ctx, server_endpoint, grpc_cq_pollset(cq));
+  grpc_server_setup_transport(&exec_ctx, server, transport, NULL, server_args);
+  grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+#else  // !GPR_SUPPORT_CHANNELS_FROM_FD
+
+void grpc_server_add_insecure_channel_from_fd(grpc_server *server,
+                                              grpc_completion_queue *cq,
+                                              int fd) {
+  GPR_ASSERT(0);
+}
+
+#endif  // GPR_SUPPORT_CHANNELS_FROM_FD
diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index 3286b220d4..c42810e913 100644
--- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -129,9 +129,11 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
   state->state = statep;
   state_ref(state->state);
   state->accepting_pollset = accepting_pollset;
-  grpc_server_security_connector_do_handshake(exec_ctx, state->state->sc,
-                                              acceptor, tcp,
-                                              on_secure_handshake_done, state);
+  grpc_server_security_connector_do_handshake(
+      exec_ctx, state->state->sc, acceptor, tcp,
+      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                   gpr_time_from_seconds(120, GPR_TIMESPAN)),
+      on_secure_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
@@ -173,7 +175,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   grpc_tcp_server *tcp = NULL;
   server_secure_state *state = NULL;
   size_t i;
-  unsigned count = 0;
+  size_t count = 0;
   int port_num = -1;
   int port_temp;
   grpc_security_status status = GRPC_SECURITY_ERROR;
@@ -188,7 +190,11 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
       3, (server, addr, creds));
 
   /* create security context */
-  if (creds == NULL) goto error;
+  if (creds == NULL) {
+    err = GRPC_ERROR_CREATE(
+        "No credentials specified for secure server port (creds==NULL)");
+    goto error;
+  }
   status = grpc_server_credentials_create_security_connector(creds, &sc);
   if (status != GRPC_SECURITY_OK) {
     char *msg;
@@ -240,14 +246,15 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   }
   if (count == 0) {
     char *msg;
-    gpr_asprintf(&msg, "No address added out of total %d resolved",
+    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
                  resolved->naddrs);
     err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, resolved->naddrs);
     gpr_free(msg);
     goto error;
   } else if (count != resolved->naddrs) {
     char *msg;
-    gpr_asprintf(&msg, "Only %d addresses added out of total %d resolved",
+    gpr_asprintf(&msg, "Only %" PRIuPTR
+                       " addresses added out of total %" PRIuPTR " resolved",
                  count, resolved->naddrs);
     err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, resolved->naddrs);
     gpr_free(msg);
diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.c b/src/core/ext/transport/chttp2/transport/bin_decoder.c
new file mode 100644
index 0000000000..2d90b01cd8
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/bin_decoder.c
@@ -0,0 +1,232 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/ext/transport/chttp2/transport/bin_decoder.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include "src/core/lib/support/string.h"
+
+static uint8_t decode_table[] = {
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 62,   0x40, 0x40, 0x40, 63,
+    52,   53,   54,   55,   56,   57,   58,   59,   60,   61,   0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0,    1,    2,    3,    4,    5,    6,
+    7,    8,    9,    10,   11,   12,   13,   14,   15,   16,   17,   18,
+    19,   20,   21,   22,   23,   24,   25,   0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 26,   27,   28,   29,   30,   31,   32,   33,   34,   35,   36,
+    37,   38,   39,   40,   41,   42,   43,   44,   45,   46,   47,   48,
+    49,   50,   51,   0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40};
+
+static const uint8_t tail_xtra[4] = {0, 0, 1, 2};
+
+static bool input_is_valid(uint8_t *input_ptr, size_t length) {
+  size_t i;
+
+  for (i = 0; i < length; ++i) {
+    if ((decode_table[input_ptr[i]] & 0xC0) != 0) {
+      gpr_log(GPR_ERROR,
+              "Base64 decoding failed, invalid character '%c' in base64 "
+              "input.\n",
+              (char)(*input_ptr));
+      return false;
+    }
+  }
+  return true;
+}
+
+#define COMPOSE_OUTPUT_BYTE_0(input_ptr)        \
+  (uint8_t)((decode_table[input_ptr[0]] << 2) | \
+            (decode_table[input_ptr[1]] >> 4))
+
+#define COMPOSE_OUTPUT_BYTE_1(input_ptr)        \
+  (uint8_t)((decode_table[input_ptr[1]] << 4) | \
+            (decode_table[input_ptr[2]] >> 2))
+
+#define COMPOSE_OUTPUT_BYTE_2(input_ptr) \
+  (uint8_t)((decode_table[input_ptr[2]] << 6) | decode_table[input_ptr[3]])
+
+bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx) {
+  size_t input_tail;
+
+  if (ctx->input_cur > ctx->input_end || ctx->output_cur > ctx->output_end) {
+    return false;
+  }
+
+  // Process a block of 4 input characters and 3 output bytes
+  while (ctx->input_end >= ctx->input_cur + 4 &&
+         ctx->output_end >= ctx->output_cur + 3) {
+    if (!input_is_valid(ctx->input_cur, 4)) return false;
+    ctx->output_cur[0] = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+    ctx->output_cur[1] = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+    ctx->output_cur[2] = COMPOSE_OUTPUT_BYTE_2(ctx->input_cur);
+    ctx->output_cur += 3;
+    ctx->input_cur += 4;
+  }
+
+  // Process the tail of input data
+  input_tail = (size_t)(ctx->input_end - ctx->input_cur);
+  if (input_tail == 4) {
+    // Process the input data with pad chars
+    if (ctx->input_cur[3] == '=') {
+      if (ctx->input_cur[2] == '=' && ctx->output_end >= ctx->output_cur + 1) {
+        if (!input_is_valid(ctx->input_cur, 2)) return false;
+        *(ctx->output_cur++) = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+        ctx->input_cur += 4;
+      } else if (ctx->output_end >= ctx->output_cur + 2) {
+        if (!input_is_valid(ctx->input_cur, 3)) return false;
+        *(ctx->output_cur++) = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+        *(ctx->output_cur++) = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+        ;
+        ctx->input_cur += 4;
+      }
+    }
+
+  } else if (ctx->contains_tail && input_tail > 1) {
+    // Process the input data without pad chars, but constains_tail is set
+    if (ctx->output_end >= ctx->output_cur + tail_xtra[input_tail]) {
+      if (!input_is_valid(ctx->input_cur, input_tail)) return false;
+      switch (input_tail) {
+        case 3:
+          ctx->output_cur[1] = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+        case 2:
+          ctx->output_cur[0] = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+      }
+      ctx->output_cur += tail_xtra[input_tail];
+      ctx->input_cur += input_tail;
+    }
+  }
+
+  return true;
+}
+
+gpr_slice grpc_chttp2_base64_decode(gpr_slice input) {
+  size_t input_length = GPR_SLICE_LENGTH(input);
+  size_t output_length = input_length / 4 * 3;
+  struct grpc_base64_decode_context ctx;
+  gpr_slice output;
+
+  if (input_length % 4 != 0) {
+    gpr_log(GPR_ERROR,
+            "Base64 decoding failed, input of "
+            "grpc_chttp2_base64_decode has a length of %d, which is not a "
+            "multiple of 4.\n",
+            (int)input_length);
+    return gpr_empty_slice();
+  }
+
+  if (input_length > 0) {
+    uint8_t *input_end = GPR_SLICE_END_PTR(input);
+    if (*(--input_end) == '=') {
+      output_length--;
+      if (*(--input_end) == '=') {
+        output_length--;
+      }
+    }
+  }
+  output = gpr_slice_malloc(output_length);
+
+  ctx.input_cur = GPR_SLICE_START_PTR(input);
+  ctx.input_end = GPR_SLICE_END_PTR(input);
+  ctx.output_cur = GPR_SLICE_START_PTR(output);
+  ctx.output_end = GPR_SLICE_END_PTR(output);
+  ctx.contains_tail = false;
+
+  if (!grpc_base64_decode_partial(&ctx)) {
+    char *s = gpr_dump_slice(input, GPR_DUMP_ASCII);
+    gpr_log(GPR_ERROR, "Base64 decoding failed, input string:\n%s\n", s);
+    gpr_free(s);
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+  GPR_ASSERT(ctx.output_cur == GPR_SLICE_END_PTR(output));
+  GPR_ASSERT(ctx.input_cur == GPR_SLICE_END_PTR(input));
+  return output;
+}
+
+gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input,
+                                                size_t output_length) {
+  size_t input_length = GPR_SLICE_LENGTH(input);
+  gpr_slice output = gpr_slice_malloc(output_length);
+  struct grpc_base64_decode_context ctx;
+
+  // The length of a base64 string cannot be 4 * n + 1
+  if (input_length % 4 == 1) {
+    gpr_log(GPR_ERROR,
+            "Base64 decoding failed, input of "
+            "grpc_chttp2_base64_decode_with_length has a length of %d, which "
+            "has a tail of 1 byte.\n",
+            (int)input_length);
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+
+  if (output_length > input_length / 4 * 3 + tail_xtra[input_length % 4]) {
+    gpr_log(GPR_ERROR,
+            "Base64 decoding failed, output_length %d is longer "
+            "than the max possible output length %d.\n",
+            (int)output_length,
+            (int)(input_length / 4 * 3 + tail_xtra[input_length % 4]));
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+
+  ctx.input_cur = GPR_SLICE_START_PTR(input);
+  ctx.input_end = GPR_SLICE_END_PTR(input);
+  ctx.output_cur = GPR_SLICE_START_PTR(output);
+  ctx.output_end = GPR_SLICE_END_PTR(output);
+  ctx.contains_tail = true;
+
+  if (!grpc_base64_decode_partial(&ctx)) {
+    char *s = gpr_dump_slice(input, GPR_DUMP_ASCII);
+    gpr_log(GPR_ERROR, "Base64 decoding failed, input string:\n%s\n", s);
+    gpr_free(s);
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+  GPR_ASSERT(ctx.output_cur == GPR_SLICE_END_PTR(output));
+  GPR_ASSERT(ctx.input_cur <= GPR_SLICE_END_PTR(input));
+  return output;
+}
diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.h b/src/core/ext/transport/chttp2/transport/bin_decoder.h
new file mode 100644
index 0000000000..b9d40c9b74
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/bin_decoder.h
@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H
+
+#include <grpc/support/slice.h>
+#include <stdbool.h>
+
+struct grpc_base64_decode_context {
+  /* input/output: */
+  uint8_t *input_cur;
+  uint8_t *input_end;
+  uint8_t *output_cur;
+  uint8_t *output_end;
+  /* Indicate if the decoder should handle the tail of input data*/
+  bool contains_tail;
+};
+
+/* base64 decode a grpc_base64_decode_context util either input_end is reached
+   or output_end is reached. When input_end is reached, (input_end - input_cur)
+   is less than 4. When output_end is reached, (output_end - output_cur) is less
+   than 3. Returns false if decoding is failed. */
+bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx);
+
+/* base64 decode a slice with pad chars. Returns a new slice, does not take
+   ownership of the input. Returns an empty slice if decoding is failed. */
+gpr_slice grpc_chttp2_base64_decode(gpr_slice input);
+
+/* base64 decode a slice without pad chars, data length is needed. Returns a new
+   slice, does not take ownership of the input. Returns an empty slice if
+   decoding is failed. */
+gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input,
+                                                size_t output_length);
+
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H */
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
index 5264b9e1f8..9aa39ba26c 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
@@ -47,6 +47,7 @@
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/ext/transport/chttp2/transport/status_conversion.h"
 #include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
+#include "src/core/lib/http/parser.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
@@ -105,7 +106,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
 static void cancel_from_api(grpc_exec_ctx *exec_ctx,
                             grpc_chttp2_transport_global *transport_global,
                             grpc_chttp2_stream_global *stream_global,
-                            grpc_status_code status);
+                            grpc_status_code status,
+                            gpr_slice *optional_message);
 
 static void close_from_api(grpc_exec_ctx *exec_ctx,
                            grpc_chttp2_transport_global *transport_global,
@@ -161,6 +163,8 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx,
 
   GPR_ASSERT(t->ep == NULL);
 
+  gpr_slice_unref(t->optional_drop_message);
+
   gpr_slice_buffer_destroy(&t->global.qbuf);
 
   gpr_slice_buffer_destroy(&t->writing.outbuf);
@@ -190,8 +194,8 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx,
      and maybe they hold resources that need to be freed */
   while (t->global.pings.next != &t->global.pings) {
     grpc_chttp2_outstanding_ping *ping = t->global.pings.next;
-    grpc_exec_ctx_push(exec_ctx, ping->on_recv,
-                       GRPC_ERROR_CREATE("Transport closed"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, ping->on_recv,
+                        GRPC_ERROR_CREATE("Transport closed"), NULL);
     ping->next->prev = ping->prev;
     ping->prev->next = ping->next;
     gpr_free(ping);
@@ -261,6 +265,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   t->parsing.deframe_state =
       is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0;
   t->writing.is_client = is_client;
+  t->optional_drop_message = gpr_empty_slice();
   grpc_connectivity_state_init(
       &t->channel_callback.state_tracker, GRPC_CHANNEL_READY,
       is_client ? "client_transport" : "server_transport");
@@ -444,7 +449,7 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_error *error) {
   if (!t->closed) {
     t->closed = 1;
-    connectivity_state_set(exec_ctx, &t->global, GRPC_CHANNEL_FATAL_FAILURE,
+    connectivity_state_set(exec_ctx, &t->global, GRPC_CHANNEL_SHUTDOWN,
                            GRPC_ERROR_REF(error), "close_transport");
     if (t->ep) {
       allow_endpoint_shutdown_locked(exec_ctx, t);
@@ -643,7 +648,7 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
       t->executor.writing_active = 1;
       REF_TRANSPORT(t, "writing");
       prevent_endpoint_shutdown(t);
-      grpc_exec_ctx_push(exec_ctx, &t->writing_action, GRPC_ERROR_NONE, NULL);
+      grpc_exec_ctx_sched(exec_ctx, &t->writing_action, GRPC_ERROR_NONE, NULL);
     }
     check_read_ops(exec_ctx, &t->global);
 
@@ -807,8 +812,10 @@ void grpc_chttp2_add_incoming_goaway(
       gpr_log(GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg));
   gpr_slice_unref(goaway_text);
   transport_global->seen_goaway = 1;
+  /* lie: use transient failure from the transport to indicate goaway has been
+   * received */
   connectivity_state_set(
-      exec_ctx, transport_global, GRPC_CHANNEL_FATAL_FAILURE,
+      exec_ctx, transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE,
       grpc_error_set_str(
           grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"),
                              GRPC_ERROR_INT_HTTP2_ERROR,
@@ -869,7 +876,7 @@ static void maybe_start_some_streams(
          grpc_chttp2_list_pop_waiting_for_concurrency(transport_global,
                                                       &stream_global)) {
     cancel_from_api(exec_ctx, transport_global, stream_global,
-                    GRPC_STATUS_UNAVAILABLE);
+                    GRPC_STATUS_UNAVAILABLE, NULL);
   }
 }
 
@@ -907,7 +914,7 @@ void grpc_chttp2_complete_closure_step(
                                 stream_global->collecting_stats);
       stream_global->collecting_stats = NULL;
     }
-    grpc_exec_ctx_push(exec_ctx, closure, closure->error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, closure->error, NULL);
   }
   *pclosure = NULL;
 }
@@ -953,7 +960,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
 
   if (op->cancel_with_status != GRPC_STATUS_OK) {
     cancel_from_api(exec_ctx, transport_global, stream_global,
-                    op->cancel_with_status);
+                    op->cancel_with_status, op->optional_close_message);
   }
 
   if (op->close_with_status != GRPC_STATUS_OK) {
@@ -974,10 +981,10 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     if (metadata_size > metadata_peer_limit) {
       gpr_log(GPR_DEBUG,
               "to-be-sent initial metadata size exceeds peer limit "
-              "(%lu vs. %lu)",
+              "(%" PRIuPTR " vs. %" PRIuPTR ")",
               metadata_size, metadata_peer_limit);
       cancel_from_api(exec_ctx, transport_global, stream_global,
-                      GRPC_STATUS_RESOURCE_EXHAUSTED);
+                      GRPC_STATUS_RESOURCE_EXHAUSTED, NULL);
     } else {
       if (contains_non_ok_status(transport_global, op->send_initial_metadata)) {
         stream_global->seen_error = true;
@@ -1033,10 +1040,10 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     if (metadata_size > metadata_peer_limit) {
       gpr_log(GPR_DEBUG,
               "to-be-sent trailing metadata size exceeds peer limit "
-              "(%lu vs. %lu)",
+              "(%" PRIuPTR " vs. %" PRIuPTR ")",
               metadata_size, metadata_peer_limit);
       cancel_from_api(exec_ctx, transport_global, stream_global,
-                      GRPC_STATUS_RESOURCE_EXHAUSTED);
+                      GRPC_STATUS_RESOURCE_EXHAUSTED, NULL);
     } else {
       if (contains_non_ok_status(transport_global,
                                  op->send_trailing_metadata)) {
@@ -1127,7 +1134,7 @@ static void ack_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   for (ping = transport_global->pings.next; ping != &transport_global->pings;
        ping = ping->next) {
     if (0 == memcmp(opaque_8bytes, ping->id, 8)) {
-      grpc_exec_ctx_push(exec_ctx, ping->on_recv, GRPC_ERROR_NONE, NULL);
+      grpc_exec_ctx_sched(exec_ctx, ping->on_recv, GRPC_ERROR_NONE, NULL);
       ping->next->prev = ping->prev;
       ping->prev->next = ping->next;
       gpr_free(ping);
@@ -1160,7 +1167,7 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
     return;
   }
 
-  grpc_exec_ctx_push(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
 
   if (op->on_connectivity_state_change != NULL) {
     grpc_connectivity_state_notify_on_state_change(
@@ -1228,14 +1235,14 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
         }
         if (stream_global->exceeded_metadata_size) {
           cancel_from_api(exec_ctx, transport_global, stream_global,
-                          GRPC_STATUS_RESOURCE_EXHAUSTED);
+                          GRPC_STATUS_RESOURCE_EXHAUSTED, NULL);
         }
       }
       grpc_chttp2_incoming_metadata_buffer_publish(
           &stream_global->received_initial_metadata,
           stream_global->recv_initial_metadata);
-      grpc_exec_ctx_push(exec_ctx, stream_global->recv_initial_metadata_ready,
-                         GRPC_ERROR_NONE, NULL);
+      grpc_exec_ctx_sched(exec_ctx, stream_global->recv_initial_metadata_ready,
+                          GRPC_ERROR_NONE, NULL);
       stream_global->recv_initial_metadata_ready = NULL;
     }
     if (stream_global->recv_message_ready != NULL) {
@@ -1248,13 +1255,13 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
         *stream_global->recv_message = grpc_chttp2_incoming_frame_queue_pop(
             &stream_global->incoming_frames);
         GPR_ASSERT(*stream_global->recv_message != NULL);
-        grpc_exec_ctx_push(exec_ctx, stream_global->recv_message_ready,
-                           GRPC_ERROR_NONE, NULL);
+        grpc_exec_ctx_sched(exec_ctx, stream_global->recv_message_ready,
+                            GRPC_ERROR_NONE, NULL);
         stream_global->recv_message_ready = NULL;
       } else if (stream_global->published_trailing_metadata) {
         *stream_global->recv_message = NULL;
-        grpc_exec_ctx_push(exec_ctx, stream_global->recv_message_ready,
-                           GRPC_ERROR_NONE, NULL);
+        grpc_exec_ctx_sched(exec_ctx, stream_global->recv_message_ready,
+                            GRPC_ERROR_NONE, NULL);
         stream_global->recv_message_ready = NULL;
       }
     }
@@ -1267,7 +1274,7 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
         }
         if (stream_global->exceeded_metadata_size) {
           cancel_from_api(exec_ctx, transport_global, stream_global,
-                          GRPC_STATUS_RESOURCE_EXHAUSTED);
+                          GRPC_STATUS_RESOURCE_EXHAUSTED, NULL);
         }
       }
       if (stream_global->all_incoming_byte_streams_finished) {
@@ -1334,7 +1341,8 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
 static void cancel_from_api(grpc_exec_ctx *exec_ctx,
                             grpc_chttp2_transport_global *transport_global,
                             grpc_chttp2_stream_global *stream_global,
-                            grpc_status_code status) {
+                            grpc_status_code status,
+                            gpr_slice *optional_message) {
   if (!stream_global->read_closed || !stream_global->write_closed) {
     if (stream_global->id != 0) {
       gpr_slice_buffer_add(
@@ -1344,8 +1352,12 @@ static void cancel_from_api(grpc_exec_ctx *exec_ctx,
               (uint32_t)grpc_chttp2_grpc_status_to_http2_error(status),
               &stream_global->stats.outgoing));
     }
+
+    if (optional_message) {
+      gpr_slice_ref(*optional_message);
+    }
     grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status,
-                            NULL);
+                            optional_message);
   }
   if (status != GRPC_STATUS_OK && !stream_global->seen_error) {
     stream_global->seen_error = true;
@@ -1465,93 +1477,95 @@ static void close_from_api(grpc_exec_ctx *exec_ctx,
 
   GPR_ASSERT(status >= 0 && (int)status < 100);
 
-  GPR_ASSERT(stream_global->id != 0);
-
-  /* Hand roll a header block.
-     This is unnecessarily ugly - at some point we should find a more elegant
-     solution.
-     It's complicated by the fact that our send machinery would be dead by the
-     time we got around to sending this, so instead we ignore HPACK compression
-     and just write the uncompressed bytes onto the wire. */
-  status_hdr = gpr_slice_malloc(15 + (status >= 10));
-  p = GPR_SLICE_START_PTR(status_hdr);
-  *p++ = 0x40; /* literal header */
-  *p++ = 11;   /* len(grpc-status) */
-  *p++ = 'g';
-  *p++ = 'r';
-  *p++ = 'p';
-  *p++ = 'c';
-  *p++ = '-';
-  *p++ = 's';
-  *p++ = 't';
-  *p++ = 'a';
-  *p++ = 't';
-  *p++ = 'u';
-  *p++ = 's';
-  if (status < 10) {
-    *p++ = 1;
-    *p++ = (uint8_t)('0' + status);
-  } else {
-    *p++ = 2;
-    *p++ = (uint8_t)('0' + (status / 10));
-    *p++ = (uint8_t)('0' + (status % 10));
-  }
-  GPR_ASSERT(p == GPR_SLICE_END_PTR(status_hdr));
-  len += (uint32_t)GPR_SLICE_LENGTH(status_hdr);
-
-  if (optional_message) {
-    GPR_ASSERT(GPR_SLICE_LENGTH(*optional_message) < 127);
-    message_pfx = gpr_slice_malloc(15);
-    p = GPR_SLICE_START_PTR(message_pfx);
-    *p++ = 0x40;
-    *p++ = 12; /* len(grpc-message) */
+  if (stream_global->id != 0 && !transport_global->is_client) {
+    /* Hand roll a header block.
+       This is unnecessarily ugly - at some point we should find a more elegant
+       solution.
+       It's complicated by the fact that our send machinery would be dead by the
+       time we got around to sending this, so instead we ignore HPACK
+       compression
+       and just write the uncompressed bytes onto the wire. */
+    status_hdr = gpr_slice_malloc(15 + (status >= 10));
+    p = GPR_SLICE_START_PTR(status_hdr);
+    *p++ = 0x40; /* literal header */
+    *p++ = 11;   /* len(grpc-status) */
     *p++ = 'g';
     *p++ = 'r';
     *p++ = 'p';
     *p++ = 'c';
     *p++ = '-';
-    *p++ = 'm';
-    *p++ = 'e';
-    *p++ = 's';
     *p++ = 's';
+    *p++ = 't';
     *p++ = 'a';
-    *p++ = 'g';
-    *p++ = 'e';
-    *p++ = (uint8_t)GPR_SLICE_LENGTH(*optional_message);
-    GPR_ASSERT(p == GPR_SLICE_END_PTR(message_pfx));
-    len += (uint32_t)GPR_SLICE_LENGTH(message_pfx);
-    len += (uint32_t)GPR_SLICE_LENGTH(*optional_message);
-  }
-
-  hdr = gpr_slice_malloc(9);
-  p = GPR_SLICE_START_PTR(hdr);
-  *p++ = (uint8_t)(len >> 16);
-  *p++ = (uint8_t)(len >> 8);
-  *p++ = (uint8_t)(len);
-  *p++ = GRPC_CHTTP2_FRAME_HEADER;
-  *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
-  *p++ = (uint8_t)(stream_global->id >> 24);
-  *p++ = (uint8_t)(stream_global->id >> 16);
-  *p++ = (uint8_t)(stream_global->id >> 8);
-  *p++ = (uint8_t)(stream_global->id);
-  GPR_ASSERT(p == GPR_SLICE_END_PTR(hdr));
-
-  gpr_slice_buffer_add(&transport_global->qbuf, hdr);
-  gpr_slice_buffer_add(&transport_global->qbuf, status_hdr);
-  if (optional_message) {
-    gpr_slice_buffer_add(&transport_global->qbuf, message_pfx);
-    gpr_slice_buffer_add(&transport_global->qbuf,
-                         gpr_slice_ref(*optional_message));
-  }
+    *p++ = 't';
+    *p++ = 'u';
+    *p++ = 's';
+    if (status < 10) {
+      *p++ = 1;
+      *p++ = (uint8_t)('0' + status);
+    } else {
+      *p++ = 2;
+      *p++ = (uint8_t)('0' + (status / 10));
+      *p++ = (uint8_t)('0' + (status % 10));
+    }
+    GPR_ASSERT(p == GPR_SLICE_END_PTR(status_hdr));
+    len += (uint32_t)GPR_SLICE_LENGTH(status_hdr);
+
+    if (optional_message) {
+      GPR_ASSERT(GPR_SLICE_LENGTH(*optional_message) < 127);
+      message_pfx = gpr_slice_malloc(15);
+      p = GPR_SLICE_START_PTR(message_pfx);
+      *p++ = 0x40;
+      *p++ = 12; /* len(grpc-message) */
+      *p++ = 'g';
+      *p++ = 'r';
+      *p++ = 'p';
+      *p++ = 'c';
+      *p++ = '-';
+      *p++ = 'm';
+      *p++ = 'e';
+      *p++ = 's';
+      *p++ = 's';
+      *p++ = 'a';
+      *p++ = 'g';
+      *p++ = 'e';
+      *p++ = (uint8_t)GPR_SLICE_LENGTH(*optional_message);
+      GPR_ASSERT(p == GPR_SLICE_END_PTR(message_pfx));
+      len += (uint32_t)GPR_SLICE_LENGTH(message_pfx);
+      len += (uint32_t)GPR_SLICE_LENGTH(*optional_message);
+    }
 
-  gpr_slice_buffer_add(
-      &transport_global->qbuf,
-      grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR,
-                                    &stream_global->stats.outgoing));
+    hdr = gpr_slice_malloc(9);
+    p = GPR_SLICE_START_PTR(hdr);
+    *p++ = (uint8_t)(len >> 16);
+    *p++ = (uint8_t)(len >> 8);
+    *p++ = (uint8_t)(len);
+    *p++ = GRPC_CHTTP2_FRAME_HEADER;
+    *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
+    *p++ = (uint8_t)(stream_global->id >> 24);
+    *p++ = (uint8_t)(stream_global->id >> 16);
+    *p++ = (uint8_t)(stream_global->id >> 8);
+    *p++ = (uint8_t)(stream_global->id);
+    GPR_ASSERT(p == GPR_SLICE_END_PTR(hdr));
+
+    gpr_slice_buffer_add(&transport_global->qbuf, hdr);
+    gpr_slice_buffer_add(&transport_global->qbuf, status_hdr);
+    if (optional_message) {
+      gpr_slice_buffer_add(&transport_global->qbuf, message_pfx);
+      gpr_slice_buffer_add(&transport_global->qbuf,
+                           gpr_slice_ref(*optional_message));
+    }
 
-  if (optional_message) {
-    gpr_slice_ref(*optional_message);
+    gpr_slice_buffer_add(
+        &transport_global->qbuf,
+        grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR,
+                                      &stream_global->stats.outgoing));
+
+    if (optional_message) {
+      gpr_slice_ref(*optional_message);
+    }
   }
+
   grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status,
                           optional_message);
   grpc_error *err = GRPC_ERROR_CREATE("Stream closed");
@@ -1569,8 +1583,12 @@ static void close_from_api(grpc_exec_ctx *exec_ctx,
 static void cancel_stream_cb(grpc_chttp2_transport_global *transport_global,
                              void *user_data,
                              grpc_chttp2_stream_global *stream_global) {
+  grpc_chttp2_transport *transport = TRANSPORT_FROM_GLOBAL(transport_global);
   cancel_from_api(user_data, transport_global, stream_global,
-                  GRPC_STATUS_UNAVAILABLE);
+                  GRPC_STATUS_UNAVAILABLE,
+                  GPR_SLICE_IS_EMPTY(transport->optional_drop_message)
+                      ? NULL
+                      : &transport->optional_drop_message);
 }
 
 static void end_all_the_calls(grpc_exec_ctx *exec_ctx,
@@ -1643,24 +1661,64 @@ static void reading_action_locked(grpc_exec_ctx *exec_ctx,
     grpc_chttp2_stream_map_move_into(&t->new_stream_map,
                                      &t->parsing_stream_map);
     grpc_chttp2_prepare_to_read(transport_global, transport_parsing);
-    grpc_exec_ctx_push(exec_ctx, &t->parsing_action, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &t->parsing_action, error, NULL);
   } else {
     post_reading_action_locked(exec_ctx, t, s_unused, arg);
   }
 }
 
+static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx,
+                                    grpc_chttp2_transport *t) {
+  grpc_http_parser parser;
+  size_t i = 0;
+  grpc_error *error = GRPC_ERROR_NONE;
+  grpc_http_response response;
+  memset(&response, 0, sizeof(response));
+
+  grpc_http_parser_init(&parser, GRPC_HTTP_RESPONSE, &response);
+
+  grpc_error *parse_error = GRPC_ERROR_NONE;
+  for (; i < t->read_buffer.count && parse_error == GRPC_ERROR_NONE; i++) {
+    parse_error = grpc_http_parser_parse(&parser, t->read_buffer.slices[i]);
+  }
+  if (parse_error == GRPC_ERROR_NONE &&
+      (parse_error = grpc_http_parser_eof(&parser)) == GRPC_ERROR_NONE) {
+    error = grpc_error_set_int(
+        GRPC_ERROR_CREATE("Trying to connect an http1.x server"),
+        GRPC_ERROR_INT_HTTP_STATUS, response.status);
+  }
+  GRPC_ERROR_UNREF(parse_error);
+
+  grpc_http_parser_destroy(&parser);
+  grpc_http_response_destroy(&response);
+  return error;
+}
+
 static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg,
                            grpc_error *error) {
   grpc_chttp2_transport *t = arg;
   GPR_TIMER_BEGIN("reading_action.parse", 0);
   size_t i = 0;
-  grpc_error *errors[2] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE};
+  grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE,
+                           GRPC_ERROR_NONE};
   for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) {
     errors[1] = grpc_chttp2_perform_read(exec_ctx, &t->parsing,
                                          t->read_buffer.slices[i]);
   };
+  if (i != t->read_buffer.count) {
+    gpr_slice_unref(t->optional_drop_message);
+    errors[2] = try_http_parsing(exec_ctx, t);
+    if (errors[2] != GRPC_ERROR_NONE) {
+      t->optional_drop_message = gpr_slice_from_copied_string(
+          "Connection dropped: received http1.x response");
+    } else {
+      t->optional_drop_message = gpr_slice_from_copied_string(
+          "Connection dropped: received unparseable response");
+    }
+  }
   grpc_error *err =
-      errors[0] == GRPC_ERROR_NONE && errors[1] == GRPC_ERROR_NONE
+      errors[0] == GRPC_ERROR_NONE && errors[1] == GRPC_ERROR_NONE &&
+              errors[2] == GRPC_ERROR_NONE
           ? GRPC_ERROR_NONE
           : GRPC_ERROR_CREATE_REFERENCING("Failed parsing HTTP/2", errors,
                                           GPR_ARRAY_SIZE(errors));
@@ -1794,6 +1852,13 @@ static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                                    add_to_pollset_locked, pollset, 0);
 }
 
+static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                            grpc_stream *gs, grpc_pollset_set *pollset_set) {
+  grpc_chttp2_run_with_global_lock(exec_ctx, (grpc_chttp2_transport *)gt,
+                                   (grpc_chttp2_stream *)gs,
+                                   add_to_pollset_set_locked, pollset_set, 0);
+}
+
 /*******************************************************************************
  * BYTE STREAM
  */
@@ -1870,10 +1935,10 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx,
   }
   if (bs->slices.count > 0) {
     *arg->slice = gpr_slice_buffer_take_first(&bs->slices);
-    grpc_exec_ctx_push(exec_ctx, arg->on_complete, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, arg->on_complete, GRPC_ERROR_NONE, NULL);
   } else if (bs->error != GRPC_ERROR_NONE) {
-    grpc_exec_ctx_push(exec_ctx, arg->on_complete, GRPC_ERROR_REF(bs->error),
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, arg->on_complete, GRPC_ERROR_REF(bs->error),
+                        NULL);
   } else {
     bs->on_next = arg->on_complete;
     bs->next = arg->slice;
@@ -1930,7 +1995,7 @@ static void incoming_byte_stream_push_locked(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_incoming_byte_stream *bs = arg->byte_stream;
   if (bs->on_next != NULL) {
     *bs->next = arg->slice;
-    grpc_exec_ctx_push(exec_ctx, bs->on_next, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE, NULL);
     bs->on_next = NULL;
   } else {
     gpr_slice_buffer_add(&bs->slices, arg->slice);
@@ -1968,7 +2033,7 @@ static void incoming_byte_stream_finished_failed_locked(
   grpc_chttp2_incoming_byte_stream *bs = a->bs;
   grpc_error *error = a->error;
   gpr_free(a);
-  grpc_exec_ctx_push(exec_ctx, bs->on_next, GRPC_ERROR_REF(error), NULL);
+  grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error), NULL);
   bs->on_next = NULL;
   GRPC_ERROR_UNREF(bs->error);
   bs->error = error;
@@ -2045,10 +2110,13 @@ static char *format_flowctl_context_var(const char *context, const char *var,
                                         int64_t val, uint32_t id,
                                         char **scope) {
   char *underscore_pos;
+  char *buf;
   char *result;
   if (context == NULL) {
     *scope = NULL;
-    gpr_asprintf(&result, "%s(%lld)", var, val);
+    gpr_asprintf(&buf, "%s(%" PRId64 ")", var, val);
+    result = gpr_leftpad(buf, ' ', 40);
+    gpr_free(buf);
     return result;
   }
   underscore_pos = strchr(context, '_');
@@ -2059,7 +2127,9 @@ static char *format_flowctl_context_var(const char *context, const char *var,
     gpr_asprintf(scope, "%s[%d]", tmp, id);
     gpr_free(tmp);
   }
-  gpr_asprintf(&result, "%s.%s(%lld)", underscore_pos + 1, var, val);
+  gpr_asprintf(&buf, "%s.%s(%" PRId64 ")", underscore_pos + 1, var, val);
+  result = gpr_leftpad(buf, ' ', 40);
+  gpr_free(buf);
   return result;
 }
 
@@ -2080,6 +2150,8 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
                                uint32_t stream_id, int64_t val1, int64_t val2) {
   char *scope1;
   char *scope2;
+  char *tmp_phase;
+  char *tmp_scope1;
   char *label1 =
       format_flowctl_context_var(context1, var1, val1, stream_id, &scope1);
   char *label2 =
@@ -2087,14 +2159,18 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
   char *clisvr = is_client ? "client" : "server";
   char *prefix;
 
-  gpr_asprintf(&prefix, "FLOW % 8s: %s % 11s ", phase, clisvr, scope1);
+  tmp_phase = gpr_leftpad(phase, ' ', 8);
+  tmp_scope1 = gpr_leftpad(scope1, ' ', 11);
+  gpr_asprintf(&prefix, "FLOW %s: %s %s ", phase, clisvr, scope1);
+  gpr_free(tmp_phase);
+  gpr_free(tmp_scope1);
 
   switch (op) {
     case GRPC_CHTTP2_FLOWCTL_MOVE:
       GPR_ASSERT(samestr(scope1, scope2));
       if (val2 != 0) {
         gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sMOVE   % 40s <- % 40s giving %d", prefix, label1, label2,
+                "%sMOVE   %s <- %s giving %" PRId64, prefix, label1, label2,
                 val1 + val2);
       }
       break;
@@ -2102,7 +2178,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
       GPR_ASSERT(val2 >= 0);
       if (val2 != 0) {
         gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sCREDIT % 40s by % 40s giving %d", prefix, label1, label2,
+                "%sCREDIT %s by %s giving %" PRId64, prefix, label1, label2,
                 val1 + val2);
       }
       break;
@@ -2110,7 +2186,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
       GPR_ASSERT(val2 >= 0);
       if (val2 != 0) {
         gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sDEBIT  % 40s by % 40s giving %d", prefix, label1, label2,
+                "%sDEBIT  %s by %s giving %" PRId64, prefix, label1, label2,
                 val1 - val2);
       }
       break;
@@ -2135,6 +2211,7 @@ static const grpc_transport_vtable vtable = {sizeof(grpc_chttp2_stream),
                                              "chttp2",
                                              init_stream,
                                              set_pollset,
+                                             set_pollset_set,
                                              perform_stream_op,
                                              perform_transport_op,
                                              destroy_stream,
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 7e5d58380e..54e72ebd24 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -383,6 +383,9 @@ struct grpc_chttp2_transport {
 
   /** Transport op to be applied post-parsing */
   grpc_transport_op *post_parsing_op;
+
+  /** Message explaining the reason of dropping connection */
+  gpr_slice optional_drop_message;
 };
 
 typedef struct {
diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c
index 72b3131d7b..785134091f 100644
--- a/src/core/ext/transport/chttp2/transport/parsing.c
+++ b/src/core/ext/transport/chttp2/transport/parsing.c
@@ -559,7 +559,7 @@ static grpc_error *update_incoming_window(
   uint32_t incoming_frame_size = transport_parsing->incoming_frame_size;
   if (incoming_frame_size > transport_parsing->incoming_window) {
     char *msg;
-    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %d",
+    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64,
                  transport_parsing->incoming_frame_size,
                  transport_parsing->incoming_window);
     grpc_error *err = GRPC_ERROR_CREATE(msg);
@@ -569,7 +569,7 @@ static grpc_error *update_incoming_window(
 
   if (incoming_frame_size > stream_parsing->incoming_window) {
     char *msg;
-    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %d",
+    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64,
                  transport_parsing->incoming_frame_size,
                  stream_parsing->incoming_window);
     grpc_error *err = GRPC_ERROR_CREATE(msg);
@@ -678,7 +678,8 @@ static void on_initial_header(void *tp, grpc_mdelem *md) {
     if (new_size > metadata_size_limit) {
       if (!stream_parsing->exceeded_metadata_size) {
         gpr_log(GPR_DEBUG,
-                "received initial metadata size exceeds limit (%lu vs. %lu)",
+                "received initial metadata size exceeds limit (%" PRIuPTR
+                " vs. %" PRIuPTR ")",
                 new_size, metadata_size_limit);
         stream_parsing->seen_error = true;
         stream_parsing->exceeded_metadata_size = true;
@@ -724,7 +725,8 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) {
   if (new_size > metadata_size_limit) {
     if (!stream_parsing->exceeded_metadata_size) {
       gpr_log(GPR_DEBUG,
-              "received trailing metadata size exceeds limit (%lu vs. %lu)",
+              "received trailing metadata size exceeds limit (%" PRIuPTR
+              " vs. %" PRIuPTR ")",
               new_size, metadata_size_limit);
       stream_parsing->seen_error = true;
       stream_parsing->exceeded_metadata_size = true;
diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c
index ca56debc84..add7678182 100644
--- a/src/core/ext/transport/chttp2/transport/writing.c
+++ b/src/core/ext/transport/chttp2/transport/writing.c
@@ -187,8 +187,8 @@ void grpc_chttp2_perform_writes(
     grpc_endpoint_write(exec_ctx, endpoint, &transport_writing->outbuf,
                         &transport_writing->done_cb);
   } else {
-    grpc_exec_ctx_push(exec_ctx, &transport_writing->done_cb, GRPC_ERROR_NONE,
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, &transport_writing->done_cb, GRPC_ERROR_NONE,
+                        NULL);
   }
 }
 
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c
index ea0f2bcba4..25d8aca250 100644
--- a/src/core/ext/transport/cronet/transport/cronet_transport.c
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.c
@@ -152,14 +152,18 @@ static void next_recv_step(stream_obj *s, enum e_caller caller);
 static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                                    grpc_stream *gs, grpc_pollset *pollset) {}
 
+static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx,
+                                       grpc_transport *gt, grpc_stream *gs,
+                                       grpc_pollset_set *pollset_set) {}
+
 static void enqueue_callbacks(grpc_closure *callback_list[]) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   if (callback_list[0]) {
-    grpc_exec_ctx_push(&exec_ctx, callback_list[0], GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(&exec_ctx, callback_list[0], GRPC_ERROR_NONE, NULL);
     callback_list[0] = NULL;
   }
   if (callback_list[1]) {
-    grpc_exec_ctx_push(&exec_ctx, callback_list[1], GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(&exec_ctx, callback_list[1], GRPC_ERROR_NONE, NULL);
     callback_list[1] = NULL;
   }
   grpc_exec_ctx_finish(&exec_ctx);
@@ -218,8 +222,11 @@ static void on_write_completed(cronet_bidirectional_stream *stream,
 static void process_recv_message(stream_obj *s, const uint8_t *recv_data) {
   gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes);
   uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice);
-  memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
-  gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  if (s->total_read_bytes > 0) {
+    // Only copy if there is non-zero number of bytes
+    memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
+    gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  }
   grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0);
   *s->recv_message = (grpc_byte_buffer *)&s->sbs;
 }
@@ -347,8 +354,17 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) {
           if (grpc_cronet_trace) {
             gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");
           }
-          cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
-                                           s->remaining_read_bytes);
+          if (s->remaining_read_bytes > 0) {
+            cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
+                                             s->remaining_read_bytes);
+          } else {
+            // Calling the closing callback directly since this is a 0 byte read
+            // for an empty message.
+            process_recv_message(s, NULL);
+            enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]);
+            invoke_closing_callback(s);
+            set_recv_state(s, CRONET_RECV_CLOSED);
+          }
         }
       }
       break;
@@ -634,7 +650,13 @@ static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
   }
 }
 
-const grpc_transport_vtable grpc_cronet_vtable = {
-    sizeof(stream_obj),     "cronet_http",     init_stream,
-    set_pollset_do_nothing, perform_stream_op, NULL,
-    destroy_stream,         destroy_transport, NULL};
+const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj),
+                                                  "cronet_http",
+                                                  init_stream,
+                                                  set_pollset_do_nothing,
+                                                  set_pollset_set_do_nothing,
+                                                  perform_stream_op,
+                                                  NULL,
+                                                  destroy_stream,
+                                                  destroy_transport,
+                                                  NULL};
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index 569be4dc28..79ceeb66b3 100644
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -35,6 +35,7 @@
 #include <grpc/grpc.h>
 #include "src/core/lib/support/string.h"
 
+#include <grpc/compression.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
@@ -148,6 +149,7 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
 
 void grpc_channel_args_destroy(grpc_channel_args *a) {
   size_t i;
+  if (!a) return;
   for (i = 0; i < a->num_args; i++) {
     switch (a->args[i].type) {
       case GRPC_ARG_STRING:
@@ -181,6 +183,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
 
 grpc_channel_args *grpc_channel_args_set_compression_algorithm(
     grpc_channel_args *a, grpc_compression_algorithm algorithm) {
+  GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT);
   grpc_arg tmp;
   tmp.type = GRPC_ARG_INTEGER;
   tmp.key = GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM;
@@ -200,7 +203,8 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a,
           !strcmp(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET,
                   a->args[i].key)) {
         *states_arg = &a->args[i].value.integer;
-        return 1; /* GPR_TRUE */
+        **states_arg |= 0x1; /* forcefully enable support for no compression */
+        return 1;
       }
     }
   }
@@ -209,15 +213,23 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a,
 
 grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
     grpc_channel_args **a, grpc_compression_algorithm algorithm, int state) {
-  int *states_arg;
+  int *states_arg = NULL;
   grpc_channel_args *result = *a;
   const int states_arg_found =
       find_compression_algorithm_states_bitset(*a, &states_arg);
 
-  if (states_arg_found) {
+  if (grpc_channel_args_get_compression_algorithm(*a) == algorithm &&
+      state == 0) {
+    char *algo_name = NULL;
+    GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algo_name) != 0);
+    gpr_log(GPR_ERROR,
+            "Tried to disable default compression algorithm '%s'. The "
+            "operation has been ignored.",
+            algo_name);
+  } else if (states_arg_found) {
     if (state != 0) {
       GPR_BITSET((unsigned *)states_arg, algorithm);
-    } else {
+    } else if (algorithm != GRPC_COMPRESS_NONE) {
       GPR_BITCLEAR((unsigned *)states_arg, algorithm);
     }
   } else {
@@ -229,7 +241,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
     tmp.value.integer = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
     if (state != 0) {
       GPR_BITSET((unsigned *)&tmp.value.integer, algorithm);
-    } else {
+    } else if (algorithm != GRPC_COMPRESS_NONE) {
       GPR_BITCLEAR((unsigned *)&tmp.value.integer, algorithm);
     }
     result = grpc_channel_args_copy_and_add(*a, &tmp, 1);
@@ -239,11 +251,11 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
   return result;
 }
 
-int grpc_channel_args_compression_algorithm_get_states(
+uint32_t grpc_channel_args_compression_algorithm_get_states(
     const grpc_channel_args *a) {
   int *states_arg;
   if (find_compression_algorithm_states_bitset(a, &states_arg)) {
-    return *states_arg;
+    return (uint32_t)*states_arg;
   } else {
     return (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; /* All algs. enabled */
   }
diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h
index 23c7b7b897..653d04f427 100644
--- a/src/core/lib/channel/channel_args.h
+++ b/src/core/lib/channel/channel_args.h
@@ -81,7 +81,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
  *
  * The i-th bit of the returned bitset corresponds to the i-th entry in the
  * grpc_compression_algorithm enum. */
-int grpc_channel_args_compression_algorithm_get_states(
+uint32_t grpc_channel_args_compression_algorithm_get_states(
     const grpc_channel_args *a);
 
 int grpc_channel_args_compare(const grpc_channel_args *a,
diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c
index ad182d1f69..bbba85d80b 100644
--- a/src/core/lib/channel/channel_stack.c
+++ b/src/core/lib/channel/channel_stack.c
@@ -106,6 +106,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
                              const grpc_channel_filter **filters,
                              size_t filter_count,
                              const grpc_channel_args *channel_args,
+                             grpc_transport *optional_transport,
                              const char *name, grpc_channel_stack *stack) {
   size_t call_size =
       ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call_stack)) +
@@ -127,6 +128,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
   for (i = 0; i < filter_count; i++) {
     args.channel_stack = stack;
     args.channel_args = channel_args;
+    args.optional_transport = optional_transport;
     args.is_first = i == 0;
     args.is_last = i == (filter_count - 1);
     elems[i].filter = filters[i];
@@ -189,9 +191,9 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
-                                 grpc_call_stack *call_stack,
-                                 grpc_pollset *pollset) {
+void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                                grpc_call_stack *call_stack,
+                                                grpc_polling_entity *pollent) {
   size_t count = call_stack->count;
   grpc_call_element *call_elems;
   char *user_data;
@@ -203,17 +205,19 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
 
   /* init per-filter data */
   for (i = 0; i < count; i++) {
-    call_elems[i].filter->set_pollset(exec_ctx, &call_elems[i], pollset);
+    call_elems[i].filter->set_pollset_or_pollset_set(exec_ctx, &call_elems[i],
+                                                     pollent);
     user_data +=
         ROUND_UP_TO_ALIGNMENT_SIZE(call_elems[i].filter->sizeof_call_data);
   }
 }
 
-void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_element *elem,
-                                        grpc_pollset *pollset) {}
+void grpc_call_stack_ignore_set_pollset_or_pollset_set(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_polling_entity *pollent) {}
 
 void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
+                             const grpc_call_stats *call_stats,
                              void *and_free_memory) {
   grpc_call_element *elems = CALL_ELEMS_FROM_STACK(stack);
   size_t count = stack->count;
@@ -221,7 +225,7 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
 
   /* destroy per-filter data */
   for (i = 0; i < count; i++) {
-    elems[i].filter->destroy_call_elem(exec_ctx, &elems[i],
+    elems[i].filter->destroy_call_elem(exec_ctx, &elems[i], call_stats,
                                        i == count - 1 ? and_free_memory : NULL);
   }
 }
diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index 36c17cb467..41dd4a0d8a 100644
--- a/src/core/lib/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -45,7 +45,10 @@
 
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/time.h>
+
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/transport.h"
 
 typedef struct grpc_channel_element grpc_channel_element;
@@ -57,6 +60,8 @@ typedef struct grpc_call_stack grpc_call_stack;
 typedef struct {
   grpc_channel_stack *channel_stack;
   const grpc_channel_args *channel_args;
+  /** Transport, iff it is known */
+  grpc_transport *optional_transport;
   int is_first;
   int is_last;
 } grpc_channel_element_args;
@@ -67,6 +72,12 @@ typedef struct {
   grpc_call_context_element *context;
 } grpc_call_element_args;
 
+typedef struct {
+  grpc_transport_stream_stats transport_stream_stats;
+  gpr_timespec latency; /* From call creating to enqueing of received status */
+  grpc_status_code final_status;
+} grpc_call_stats;
+
 /* Channel filters specify:
    1. the amount of memory needed in the channel & call (via the sizeof_XXX
       members)
@@ -101,14 +112,16 @@ typedef struct {
      argument. */
   void (*init_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                          grpc_call_element_args *args);
-  void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                      grpc_pollset *pollset);
+  void (*set_pollset_or_pollset_set)(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_element *elem,
+                                     grpc_polling_entity *pollent);
   /* Destroy per call data.
      The filter does not need to do any chaining.
      The bottom filter of a stack will be passed a non-NULL pointer to
      \a and_free_memory that should be passed to gpr_free when destruction
      is complete. */
   void (*destroy_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                            const grpc_call_stats *stats,
                             void *and_free_memory);
 
   /* sizeof(per channel data) */
@@ -187,6 +200,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
                              grpc_iomgr_cb_func destroy, void *destroy_arg,
                              const grpc_channel_filter **filters,
                              size_t filter_count, const grpc_channel_args *args,
+                             grpc_transport *optional_transport,
                              const char *name, grpc_channel_stack *stack);
 /* Destroy a channel stack */
 void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx,
@@ -201,10 +215,11 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx,
                           grpc_call_context_element *context,
                           const void *transport_server_data,
                           grpc_call_stack *call_stack);
-/* Set a pollset for a call stack: must occur before the first op is started */
-void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
-                                 grpc_call_stack *call_stack,
-                                 grpc_pollset *pollset);
+/* Set a pollset or a pollset_set for a call stack: must occur before the first
+ * op is started */
+void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                                grpc_call_stack *call_stack,
+                                                grpc_polling_entity *pollent);
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 #define GRPC_CALL_STACK_REF(call_stack, reason) \
@@ -228,13 +243,14 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
 
 /* Destroy a call stack */
 void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
+                             const grpc_call_stats *call_stats,
                              void *and_free_memory);
 
-/* Ignore set pollset - used by filters to implement the set_pollset method
-   if they don't care about pollsets at all. Does nothing. */
-void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_element *elem,
-                                        grpc_pollset *pollset);
+/* Ignore set pollset{_set} - used by filters if they don't care about pollsets
+ * at all. Does nothing. */
+void grpc_call_stack_ignore_set_pollset_or_pollset_set(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_polling_entity *pollent);
 /* Call the next operation in a call stack */
 void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                        grpc_transport_stream_op *op);
diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c
index a8646c9565..eda4968f48 100644
--- a/src/core/lib/channel/channel_stack_builder.c
+++ b/src/core/lib/channel/channel_stack_builder.c
@@ -257,8 +257,8 @@ void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
   // and initialize it
   grpc_channel_stack_init(exec_ctx, initial_refs, destroy,
                           destroy_arg == NULL ? result : destroy_arg, filters,
-                          num_filters, builder->args, builder->name,
-                          channel_stack);
+                          num_filters, builder->args, builder->transport,
+                          builder->name, channel_stack);
 
   // run post-initialization functions
   i = 0;
diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c
index 27a87db0c2..32ebe53ee6 100644
--- a/src/core/lib/channel/compress_filter.c
+++ b/src/core/lib/channel/compress_filter.c
@@ -73,8 +73,8 @@ typedef struct call_data {
 typedef struct channel_data {
   /** The default, channel-level, compression algorithm */
   grpc_compression_algorithm default_compression_algorithm;
-  /** Compression options for the channel */
-  grpc_compression_options compression_options;
+  /** Bitset of enabled algorithms */
+  uint32_t enabled_algorithms_bitset;
   /** Supported compression algorithms */
   uint32_t supported_compression_algorithms;
 } channel_data;
@@ -96,9 +96,8 @@ static grpc_mdelem *compression_md_filter(void *user_data, grpc_mdelem *md) {
               md_c_str);
       calld->compression_algorithm = GRPC_COMPRESS_NONE;
     }
-    if (grpc_compression_options_is_algorithm_enabled(
-            &channeld->compression_options, calld->compression_algorithm) ==
-        0) {
+    if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
+                    calld->compression_algorithm)) {
       gpr_log(GPR_ERROR,
               "Invalid compression algorithm: '%s' (previously disabled). "
               "Ignoring.",
@@ -178,8 +177,8 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
       const float savings_ratio = 1.0f - (float)after_size / (float)before_size;
       GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm,
                                                  &algo_name));
-      gpr_log(GPR_DEBUG,
-              "Compressed[%s] %d bytes vs. %d bytes (%.2f%% savings)",
+      gpr_log(GPR_DEBUG, "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR
+                         " bytes (%.2f%% savings)",
               algo_name, before_size, after_size, 100 * savings_ratio);
     }
     gpr_slice_buffer_swap(&calld->slices, &tmp);
@@ -189,10 +188,10 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
       char *algo_name;
       GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm,
                                                  &algo_name));
-      gpr_log(
-          GPR_DEBUG,
-          "Algorithm '%s' enabled but decided not to compress. Input size: %d",
-          algo_name, calld->slices.length);
+      gpr_log(GPR_DEBUG,
+              "Algorithm '%s' enabled but decided not to compress. Input size: "
+              "%" PRIuPTR,
+              algo_name, calld->slices.length);
     }
   }
 
@@ -271,7 +270,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {
+                              const grpc_call_stats *stats, void *ignored) {
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   gpr_slice_buffer_destroy(&calld->slices);
@@ -282,32 +281,26 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               grpc_channel_element *elem,
                               grpc_channel_element_args *args) {
   channel_data *channeld = elem->channel_data;
-  grpc_compression_algorithm algo_idx;
 
-  grpc_compression_options_init(&channeld->compression_options);
-  channeld->compression_options.enabled_algorithms_bitset =
-      (uint32_t)grpc_channel_args_compression_algorithm_get_states(
-          args->channel_args);
+  channeld->enabled_algorithms_bitset =
+      grpc_channel_args_compression_algorithm_get_states(args->channel_args);
 
   channeld->default_compression_algorithm =
       grpc_channel_args_get_compression_algorithm(args->channel_args);
   /* Make sure the default isn't disabled. */
-  if (!grpc_compression_options_is_algorithm_enabled(
-          &channeld->compression_options,
-          channeld->default_compression_algorithm)) {
+  if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
+                  channeld->default_compression_algorithm)) {
     gpr_log(GPR_DEBUG,
             "compression algorithm %d not enabled: switching to none",
             channeld->default_compression_algorithm);
     channeld->default_compression_algorithm = GRPC_COMPRESS_NONE;
   }
-  channeld->compression_options.default_compression_algorithm =
-      channeld->default_compression_algorithm;
 
-  channeld->supported_compression_algorithms = 0;
-  for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
+  channeld->supported_compression_algorithms = 1; /* always support identity */
+  for (grpc_compression_algorithm algo_idx = 1;
+       algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
     /* skip disabled algorithms */
-    if (grpc_compression_options_is_algorithm_enabled(
-            &channeld->compression_options, algo_idx) == 0) {
+    if (!GPR_BITGET(channeld->enabled_algorithms_bitset, algo_idx)) {
       continue;
     }
     channeld->supported_compression_algorithms |= 1u << algo_idx;
@@ -325,7 +318,7 @@ const grpc_channel_filter grpc_compress_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/channel/compress_filter.h b/src/core/lib/channel/compress_filter.h
index 0ce5d08837..e4a2a829d5 100644
--- a/src/core/lib/channel/compress_filter.h
+++ b/src/core/lib/channel/compress_filter.h
@@ -34,9 +34,9 @@
 #ifndef GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H
 #define GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H
 
-#include "src/core/lib/channel/channel_stack.h"
+#include <grpc/impl/codegen/compression_types.h>
 
-#define GRPC_COMPRESS_REQUEST_ALGORITHM_KEY "grpc-internal-encoding-request"
+#include "src/core/lib/channel/channel_stack.h"
 
 extern int grpc_compression_trace;
 
@@ -48,7 +48,7 @@ extern int grpc_compression_trace;
  *  - Channel configuration, as established at channel creation time.
  *  - The metadata accompanying the outgoing data to be compressed. This is
  *    taken as a request only. We may choose not to honor it. The metadata key
- *    is given by \a GRPC_COMPRESS_REQUEST_ALGORITHM_KEY.
+ *    is given by \a GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY.
  *
  * Compression can be disabled for concrete messages (for instance in order to
  * prevent CRIME/BEAST type attacks) by having the GRPC_WRITE_NO_COMPRESS set in
diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c
index 68a3a7d6fd..0a7d27a1dc 100644
--- a/src/core/lib/channel/connected_channel.c
+++ b/src/core/lib/channel/connected_channel.c
@@ -93,16 +93,18 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   GPR_ASSERT(r == 0);
 }
 
-static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                        grpc_pollset *pollset) {
+static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                       grpc_call_element *elem,
+                                       grpc_polling_entity *pollent) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  grpc_transport_set_pollset(exec_ctx, chand->transport,
-                             TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollset);
+  grpc_transport_set_pops(exec_ctx, chand->transport,
+                          TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollent);
 }
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats,
                               void *and_free_memory) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
@@ -137,7 +139,7 @@ static const grpc_channel_filter connected_channel_filter = {
     con_start_transport_op,
     sizeof(call_data),
     init_call_elem,
-    set_pollset,
+    set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c
index ced3d75cd2..ab6c6c9ef0 100644
--- a/src/core/lib/channel/http_client_filter.c
+++ b/src/core/lib/channel/http_client_filter.c
@@ -38,6 +38,10 @@
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport_impl.h"
+
+#define EXPECTED_CONTENT_TYPE "application/grpc"
+#define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
 typedef struct call_data {
   grpc_linked_mdelem method;
@@ -74,7 +78,24 @@ static grpc_mdelem *client_recv_filter(void *user_data, grpc_mdelem *md) {
   } else if (md->key == GRPC_MDSTR_STATUS) {
     grpc_call_element_send_cancel(a->exec_ctx, a->elem);
     return NULL;
+  } else if (md == GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC) {
+    return NULL;
   } else if (md->key == GRPC_MDSTR_CONTENT_TYPE) {
+    const char *value_str = grpc_mdstr_as_c_string(md->value);
+    if (strncmp(value_str, EXPECTED_CONTENT_TYPE,
+                EXPECTED_CONTENT_TYPE_LENGTH) == 0 &&
+        (value_str[EXPECTED_CONTENT_TYPE_LENGTH] == '+' ||
+         value_str[EXPECTED_CONTENT_TYPE_LENGTH] == ';')) {
+      /* Although the C implementation doesn't (currently) generate them,
+         any custom +-suffix is explicitly valid. */
+      /* TODO(klempner): We should consider preallocating common values such
+         as +proto or +json, or at least stashing them if we see them. */
+      /* TODO(klempner): Should we be surfacing this to application code? */
+    } else {
+      /* TODO(klempner): We're currently allowing this, but we shouldn't
+         see it without a proxy so log for now. */
+      gpr_log(GPR_INFO, "Unexpected content-type '%s'", value_str);
+    }
     return NULL;
   }
   return md;
@@ -157,7 +178,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {}
+                              const grpc_call_stats *stats, void *ignored) {}
 
 static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) {
   unsigned i;
@@ -180,7 +201,8 @@ static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) {
   return GRPC_MDELEM_SCHEME_HTTP;
 }
 
-static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args) {
+static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args,
+                                        const char *transport_name) {
   gpr_strvec v;
   size_t i;
   int is_first = 1;
@@ -202,8 +224,8 @@ static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args) {
     }
   }
 
-  gpr_asprintf(&tmp, "%sgrpc-c/%s (%s)", is_first ? "" : " ",
-               grpc_version_string(), GPR_PLATFORM_STRING);
+  gpr_asprintf(&tmp, "%sgrpc-c/%s (%s; %s)", is_first ? "" : " ",
+               grpc_version_string(), GPR_PLATFORM_STRING, transport_name);
   is_first = 0;
   gpr_strvec_add(&v, tmp);
 
@@ -234,9 +256,12 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               grpc_channel_element_args *args) {
   channel_data *chand = elem->channel_data;
   GPR_ASSERT(!args->is_last);
+  GPR_ASSERT(args->optional_transport != NULL);
   chand->static_scheme = scheme_from_args(args->channel_args);
   chand->user_agent = grpc_mdelem_from_metadata_strings(
-      GRPC_MDSTR_USER_AGENT, user_agent_from_args(args->channel_args));
+      GRPC_MDSTR_USER_AGENT,
+      user_agent_from_args(args->channel_args,
+                           args->optional_transport->vtable->name));
 }
 
 /* Destructor for channel data */
@@ -251,7 +276,7 @@ const grpc_channel_filter grpc_http_client_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/channel/http_client_filter.h b/src/core/lib/channel/http_client_filter.h
index a884b36318..47081175ea 100644
--- a/src/core/lib/channel/http_client_filter.h
+++ b/src/core/lib/channel/http_client_filter.h
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2015, Google Inc.
  * All rights reserved.
  *
@@ -39,6 +38,7 @@
 /* Processes metadata on the client side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_client_filter;
 
+/* Channel arg to override the http2 :scheme header */
 #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme"
 
 #endif /* GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H */
diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c
index 876eaddc1e..d0beebd817 100644
--- a/src/core/lib/channel/http_server_filter.c
+++ b/src/core/lib/channel/http_server_filter.c
@@ -108,7 +108,7 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
     } else {
       /* TODO(klempner): We're currently allowing this, but we shouldn't
          see it without a proxy so log for now. */
-      gpr_log(GPR_INFO, "Unexpected content-type %s", value_str);
+      gpr_log(GPR_INFO, "Unexpected content-type '%s'", value_str);
     }
     return NULL;
   } else if (md->key == GRPC_MDSTR_TE || md->key == GRPC_MDSTR_METHOD ||
@@ -235,7 +235,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {}
+                              const grpc_call_stats *stats, void *ignored) {}
 
 /* Constructor for channel_data */
 static void init_channel_elem(grpc_exec_ctx *exec_ctx,
@@ -253,7 +253,7 @@ const grpc_channel_filter grpc_http_server_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/compression/compression_algorithm.c b/src/core/lib/compression/compression.c
index 820871d579..54efb5e855 100644
--- a/src/core/lib/compression/compression_algorithm.c
+++ b/src/core/lib/compression/compression.c
@@ -125,6 +125,28 @@ grpc_mdelem *grpc_compression_encoding_mdelem(
   return NULL;
 }
 
+void grpc_compression_options_init(grpc_compression_options *opts) {
+  memset(opts, 0, sizeof(*opts));
+  /* all enabled by default */
+  opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
+}
+
+void grpc_compression_options_enable_algorithm(
+    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
+  GPR_BITSET(&opts->enabled_algorithms_bitset, algorithm);
+}
+
+void grpc_compression_options_disable_algorithm(
+    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
+  GPR_BITCLEAR(&opts->enabled_algorithms_bitset, algorithm);
+}
+
+int grpc_compression_options_is_algorithm_enabled(
+    const grpc_compression_options *opts,
+    grpc_compression_algorithm algorithm) {
+  return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
+}
+
 /* TODO(dgq): Add the ability to specify parameters to the individual
  * compression algorithms */
 grpc_compression_algorithm grpc_compression_algorithm_for_level(
@@ -180,25 +202,3 @@ grpc_compression_algorithm grpc_compression_algorithm_for_level(
       abort();
   };
 }
-
-void grpc_compression_options_init(grpc_compression_options *opts) {
-  opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
-  opts->default_compression_algorithm = GRPC_COMPRESS_NONE;
-}
-
-void grpc_compression_options_enable_algorithm(
-    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
-  GPR_BITSET(&opts->enabled_algorithms_bitset, algorithm);
-}
-
-void grpc_compression_options_disable_algorithm(
-    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
-  GPR_BITCLEAR(&opts->enabled_algorithms_bitset, algorithm);
-}
-
-int grpc_compression_options_is_algorithm_enabled(
-    const grpc_compression_options *opts,
-    grpc_compression_algorithm algorithm) {
-  if (algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) return 0;
-  return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
-}
diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c
index 4bda734bfa..18135bcb58 100644
--- a/src/core/lib/http/httpcli.c
+++ b/src/core/lib/http/httpcli.c
@@ -63,7 +63,7 @@ typedef struct {
   const grpc_httpcli_handshaker *handshaker;
   grpc_closure *on_done;
   grpc_httpcli_context *context;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   grpc_iomgr_object iomgr_obj;
   gpr_slice_buffer incoming;
   gpr_slice_buffer outgoing;
@@ -78,6 +78,7 @@ static grpc_httpcli_post_override g_post_override = NULL;
 
 static void plaintext_handshake(grpc_exec_ctx *exec_ctx, void *arg,
                                 grpc_endpoint *endpoint, const char *host,
+                                gpr_timespec deadline,
                                 void (*on_done)(grpc_exec_ctx *exec_ctx,
                                                 void *arg,
                                                 grpc_endpoint *endpoint)) {
@@ -100,9 +101,9 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req,
 
 static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
                    grpc_error *error) {
-  grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set,
-                               req->pollset);
-  grpc_exec_ctx_push(exec_ctx, req->on_done, error, NULL);
+  grpc_polling_entity_del_from_pollset_set(exec_ctx, req->pollent,
+                                           req->context->pollset_set);
+  grpc_exec_ctx_sched(exec_ctx, req->on_done, error, NULL);
   grpc_http_parser_destroy(&req->parser);
   if (req->addresses != NULL) {
     grpc_resolved_addresses_destroy(req->addresses);
@@ -206,7 +207,7 @@ static void on_connected(grpc_exec_ctx *exec_ctx, void *arg,
   req->handshaker->handshake(
       exec_ctx, req, req->ep,
       req->ssl_host_override ? req->ssl_host_override : req->host,
-      on_handshake_done);
+      req->deadline, on_handshake_done);
 }
 
 static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req,
@@ -240,7 +241,7 @@ static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 
 static void internal_request_begin(grpc_exec_ctx *exec_ctx,
                                    grpc_httpcli_context *context,
-                                   grpc_pollset *pollset,
+                                   grpc_polling_entity *pollent,
                                    const grpc_httpcli_request *request,
                                    gpr_timespec deadline, grpc_closure *on_done,
                                    grpc_httpcli_response *response,
@@ -254,7 +255,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx,
   req->handshaker =
       request->handshaker ? request->handshaker : &grpc_httpcli_plaintext;
   req->context = context;
-  req->pollset = pollset;
+  req->pollent = pollent;
   req->overall_error = GRPC_ERROR_NONE;
   grpc_closure_init(&req->on_read, on_read, req);
   grpc_closure_init(&req->done_write, done_write, req);
@@ -264,14 +265,15 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx,
   req->host = gpr_strdup(request->host);
   req->ssl_host_override = gpr_strdup(request->ssl_host_override);
 
-  grpc_pollset_set_add_pollset(exec_ctx, req->context->pollset_set,
-                               req->pollset);
+  GPR_ASSERT(pollent);
+  grpc_polling_entity_add_to_pollset_set(exec_ctx, req->pollent,
+                                         req->context->pollset_set);
   grpc_resolve_address(exec_ctx, request->host, req->handshaker->default_port,
                        grpc_closure_create(on_resolved, req), &req->addresses);
 }
 
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                      grpc_pollset *pollset,
+                      grpc_polling_entity *pollent,
                       const grpc_httpcli_request *request,
                       gpr_timespec deadline, grpc_closure *on_done,
                       grpc_httpcli_response *response) {
@@ -281,14 +283,14 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
     return;
   }
   gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->http.path);
-  internal_request_begin(exec_ctx, context, pollset, request, deadline, on_done,
+  internal_request_begin(exec_ctx, context, pollent, request, deadline, on_done,
                          response, name,
                          grpc_httpcli_format_get_request(request));
   gpr_free(name);
 }
 
 void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                       grpc_pollset *pollset,
+                       grpc_polling_entity *pollent,
                        const grpc_httpcli_request *request,
                        const char *body_bytes, size_t body_size,
                        gpr_timespec deadline, grpc_closure *on_done,
@@ -301,7 +303,7 @@ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
   }
   gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->http.path);
   internal_request_begin(
-      exec_ctx, context, pollset, request, deadline, on_done, response, name,
+      exec_ctx, context, pollent, request, deadline, on_done, response, name,
       grpc_httpcli_format_post_request(request, body_bytes, body_size));
   gpr_free(name);
 }
diff --git a/src/core/lib/http/httpcli.h b/src/core/lib/http/httpcli.h
index 2d57864a1f..662e176f4c 100644
--- a/src/core/lib/http/httpcli.h
+++ b/src/core/lib/http/httpcli.h
@@ -41,6 +41,7 @@
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 
 /* User agent this library reports */
@@ -56,7 +57,7 @@ typedef struct grpc_httpcli_context {
 typedef struct {
   const char *default_port;
   void (*handshake)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint,
-                    const char *host,
+                    const char *host, gpr_timespec deadline,
                     void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg,
                                     grpc_endpoint *endpoint));
 } grpc_httpcli_handshaker;
@@ -95,7 +96,7 @@ void grpc_httpcli_context_destroy(grpc_httpcli_context *context);
    'on_response' is a callback to report results to (and 'user_data' is a user
      supplied pointer to pass to said call) */
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                      grpc_pollset *pollset,
+                      grpc_polling_entity *pollent,
                       const grpc_httpcli_request *request,
                       gpr_timespec deadline, grpc_closure *on_complete,
                       grpc_httpcli_response *response);
@@ -116,7 +117,7 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
      supplied pointer to pass to said call)
    Does not support ?var1=val1&var2=val2 in the path. */
 void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                       grpc_pollset *pollset,
+                       grpc_polling_entity *pollent,
                        const grpc_httpcli_request *request,
                        const char *body_bytes, size_t body_size,
                        gpr_timespec deadline, grpc_closure *on_complete,
diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c
index 5590928968..a57d93bb7b 100644
--- a/src/core/lib/http/httpcli_security_connector.c
+++ b/src/core/lib/http/httpcli_security_connector.c
@@ -61,6 +61,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) {
 static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_security_connector *sc,
                                      grpc_endpoint *nonsecure_endpoint,
+                                     gpr_timespec deadline,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
   grpc_httpcli_ssl_channel_security_connector *c =
@@ -79,7 +80,7 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
     grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, cb, user_data);
+                               nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
@@ -163,6 +164,7 @@ static void on_secure_transport_setup_done(grpc_exec_ctx *exec_ctx, void *rp,
 
 static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
                           grpc_endpoint *tcp, const char *host,
+                          gpr_timespec deadline,
                           void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg,
                                           grpc_endpoint *endpoint)) {
   grpc_channel_security_connector *sc = NULL;
@@ -181,7 +183,7 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
                  pem_root_certs, pem_root_certs_size, host, &sc) ==
              GRPC_SECURITY_OK);
   grpc_channel_security_connector_do_handshake(
-      exec_ctx, sc, tcp, on_secure_transport_setup_done, c);
+      exec_ctx, sc, tcp, deadline, on_secure_transport_setup_done, c);
   GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
 }
 
diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h
index 344b7d3c3e..08e59a168e 100644
--- a/src/core/lib/iomgr/closure.h
+++ b/src/core/lib/iomgr/closure.h
@@ -53,8 +53,8 @@ typedef struct grpc_closure_list {
 /** gRPC Callback definition.
  *
  * \param arg Arbitrary input.
- * \param success An indication on the state of the iomgr. On false, cleanup
- * actions should be taken (eg, shutdown). */
+ * \param error GRPC_ERROR_NONE if no error occurred, otherwise some grpc_error
+ *              describing what went wrong */
 typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_error *error);
 
@@ -66,8 +66,11 @@ struct grpc_closure {
   /** Arguments to be passed to "cb". */
   void *cb_arg;
 
+  /** Once queued, the result of the closure. Before then: scratch space */
   grpc_error *error;
 
+  /** Once queued, next indicates the next queued closure; before then, scratch
+   *  space */
   union {
     grpc_closure *next;
     uintptr_t scratch;
@@ -84,8 +87,8 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg);
 #define GRPC_CLOSURE_LIST_INIT \
   { NULL, NULL }
 
-/** add \a closure to the end of \a list and set \a closure's success to \a
- * success */
+/** add \a closure to the end of \a list
+    and set \a closure's result to \a error */
 void grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure,
                               grpc_error *error);
 
diff --git a/src/core/lib/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h
index 3877ceb1e2..f9808bbda1 100644
--- a/src/core/lib/iomgr/endpoint.h
+++ b/src/core/lib/iomgr/endpoint.h
@@ -82,7 +82,7 @@ char *grpc_endpoint_get_peer(grpc_endpoint *ep);
 void grpc_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                          gpr_slice_buffer *slices, grpc_closure *cb);
 
-/* Causes any pending read/write callbacks to run immediately with
+/* Causes any pending and future read/write callbacks to run immediately with
    success==0 */
 void grpc_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
 void grpc_endpoint_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c
index 6da0b00582..540fb4fa7e 100644
--- a/src/core/lib/iomgr/error.c
+++ b/src/core/lib/iomgr/error.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,7 @@
 
 #include "src/core/lib/iomgr/error.h"
 
+#include <inttypes.h>
 #include <stdbool.h>
 #include <string.h>
 
@@ -42,6 +43,10 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
+#ifdef GPR_WINDOWS
+#include <grpc/support/log_windows.h>
+#endif
+
 static void destroy_integer(void *key) {}
 
 static void *copy_integer(void *key) { return key; }
@@ -84,14 +89,10 @@ static const gpr_avl_vtable avl_vtable_errs = {
 
 static const char *error_int_name(grpc_error_ints key) {
   switch (key) {
-    case GRPC_ERROR_INT_STATUS_CODE:
-      return "status_code";
     case GRPC_ERROR_INT_ERRNO:
       return "errno";
     case GRPC_ERROR_INT_FILE_LINE:
       return "file_line";
-    case GRPC_ERROR_INT_WARNING:
-      return "warning";
     case GRPC_ERROR_INT_STREAM_ID:
       return "stream_id";
     case GRPC_ERROR_INT_GRPC_STATUS:
@@ -112,6 +113,8 @@ static const char *error_int_name(grpc_error_ints key) {
       return "fd";
     case GRPC_ERROR_INT_WSA_ERROR:
       return "wsa_error";
+    case GRPC_ERROR_INT_HTTP_STATUS:
+      return "http_status";
   }
   GPR_UNREACHABLE_CODE(return "unknown");
 }
@@ -336,7 +339,7 @@ static char *key_time(void *p) {
 
 static char *fmt_int(void *p) {
   char *s;
-  gpr_asprintf(&s, "%lld", (intptr_t)p);
+  gpr_asprintf(&s, "%" PRIdPTR, (intptr_t)p);
   return s;
 }
 
@@ -418,7 +421,7 @@ static char *fmt_time(void *p) {
       pfx = "";
       break;
   }
-  gpr_asprintf(&out, "\"%s%d.%09d\"", pfx, tm.tv_sec, tm.tv_nsec);
+  gpr_asprintf(&out, "\"%s%" PRId64 ".%09d\"", pfx, tm.tv_sec, tm.tv_nsec);
   return out;
 }
 
@@ -506,7 +509,7 @@ grpc_error *grpc_os_error(const char *file, int line, int err,
       GRPC_ERROR_STR_SYSCALL, call_name);
 }
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 grpc_error *grpc_wsa_error(const char *file, int line, int err,
                            const char *call_name) {
   char *utf8_message = gpr_format_message(err);
diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h
index 695724c0be..69cdf3028e 100644
--- a/src/core/lib/iomgr/error.h
+++ b/src/core/lib/iomgr/error.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,49 +39,84 @@
 
 #include <grpc/support/time.h>
 
-// Opaque representation of an error.
-// Errors are refcounted objects that represent the result of an operation.
-// Ownership laws:
-//  if a grpc_error is returned by a function, the caller owns a ref to that
-//    instance
-//  if a grpc_error is passed to a grpc_closure callback function (functions
-//    with the signature:
-//      void (*f)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error))
-//    then those functions do not automatically own a ref to error
-//  if a grpc_error is passed to *ANY OTHER FUNCTION* then that function takes
-//    ownership of the error
+/// Opaque representation of an error.
+/// Errors are refcounted objects that represent the result of an operation.
+/// Ownership laws:
+///  if a grpc_error is returned by a function, the caller owns a ref to that
+///    instance
+///  if a grpc_error is passed to a grpc_closure callback function (functions
+///    with the signature:
+///      void (*f)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error))
+///    then those functions do not automatically own a ref to error
+///  if a grpc_error is passed to *ANY OTHER FUNCTION* then that function takes
+///    ownership of the error
+/// Errors have:
+///  a set of ints, strings, and timestamps that describe the error
+///  always present are:
+///    GRPC_ERROR_STR_FILE, GRPC_ERROR_INT_FILE_LINE - source location the error
+///      was generated
+///    GRPC_ERROR_STR_DESCRIPTION - a human readable description of the error
+///    GRPC_ERROR_TIME_CREATED - a timestamp indicating when the error happened
+///  an error can also have children; these are other errors that are believed
+///    to have contributed to this one. By accumulating children, we can begin
+///    to root cause high level failures from low level failures, without having
+///    to derive execution paths from log lines
 typedef struct grpc_error grpc_error;
 
 typedef enum {
+  /// 'errno' from the operating system
   GRPC_ERROR_INT_ERRNO,
+  /// __LINE__ from the call site creating the error
   GRPC_ERROR_INT_FILE_LINE,
-  GRPC_ERROR_INT_STATUS_CODE,
-  GRPC_ERROR_INT_WARNING,
+  /// stream identifier: for errors that are associated with an individual
+  /// wire stream
   GRPC_ERROR_INT_STREAM_ID,
+  /// grpc status code representing this error
   GRPC_ERROR_INT_GRPC_STATUS,
+  /// offset into some binary blob (usually represented by
+  /// GRPC_ERROR_STR_RAW_BYTES) where the error occurred
   GRPC_ERROR_INT_OFFSET,
+  /// context sensitive index associated with the error
   GRPC_ERROR_INT_INDEX,
+  /// context sensitive size associated with the error
   GRPC_ERROR_INT_SIZE,
+  /// http2 error code associated with the error (see the HTTP2 RFC)
   GRPC_ERROR_INT_HTTP2_ERROR,
+  /// TSI status code associated with the error
   GRPC_ERROR_INT_TSI_CODE,
+  /// grpc_security_status associated with the error
   GRPC_ERROR_INT_SECURITY_STATUS,
+  /// WSAGetLastError() reported when this error occurred
   GRPC_ERROR_INT_WSA_ERROR,
+  /// File descriptor associated with this error
   GRPC_ERROR_INT_FD,
+  /// HTTP status (i.e. 404)
+  GRPC_ERROR_INT_HTTP_STATUS,
 } grpc_error_ints;
 
 typedef enum {
+  /// top-level textual description of this error
   GRPC_ERROR_STR_DESCRIPTION,
+  /// source file in which this error occurred
   GRPC_ERROR_STR_FILE,
+  /// operating system description of this error
   GRPC_ERROR_STR_OS_ERROR,
+  /// syscall that generated this error
   GRPC_ERROR_STR_SYSCALL,
+  /// peer that we were trying to communicate when this error occurred
   GRPC_ERROR_STR_TARGET_ADDRESS,
+  /// grpc status message associated with this error
   GRPC_ERROR_STR_GRPC_MESSAGE,
+  /// hex dump (or similar) with the data that generated this error
   GRPC_ERROR_STR_RAW_BYTES,
+  /// tsi error string associated with this error
   GRPC_ERROR_STR_TSI_ERROR,
+  /// filename that we were trying to read/write when this error occurred
   GRPC_ERROR_STR_FILENAME,
 } grpc_error_strs;
 
 typedef enum {
+  /// timestamp of error creation
   GRPC_ERROR_TIME_CREATED,
 } grpc_error_times;
 
@@ -92,8 +127,17 @@ typedef enum {
 const char *grpc_error_string(grpc_error *error);
 void grpc_error_free_string(const char *str);
 
+/// Create an error - but use GRPC_ERROR_CREATE instead
 grpc_error *grpc_error_create(const char *file, int line, const char *desc,
                               grpc_error **referencing, size_t num_referencing);
+/// Create an error (this is the preferred way of generating an error that is
+///   not due to a system call - for system calls, use GRPC_OS_ERROR or
+///   GRPC_WSA_ERROR as appropriate)
+/// \a referencing is an array of num_referencing elements indicating one or
+/// more errors that are believed to have contributed to this one
+/// err = grpc_error_create(x, y, z, r, nr) is equivalent to:
+///   err = grpc_error_create(x, y, z, NULL, 0);
+///   for (i=0; i<nr; i++) err = grpc_error_add_child(err, r[i]);
 #define GRPC_ERROR_CREATE(desc) \
   grpc_error_create(__FILE__, __LINE__, desc, NULL, 0)
 
@@ -125,13 +169,18 @@ grpc_error *grpc_error_set_time(grpc_error *src, grpc_error_times which,
                                 gpr_timespec value);
 grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which,
                                const char *value);
+/// Add a child error: an error that is believed to have contributed to this
+/// error occurring. Allows root causing high level errors from lower level
+/// errors that contributed to them.
 grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child);
 grpc_error *grpc_os_error(const char *file, int line, int err,
                           const char *call_name);
+/// create an error associated with errno!=0 (an 'operating system' error)
 #define GRPC_OS_ERROR(err, call_name) \
   grpc_os_error(__FILE__, __LINE__, err, call_name)
 grpc_error *grpc_wsa_error(const char *file, int line, int err,
                            const char *call_name);
+/// windows only: create an error associated with WSAGetLastError()!=0
 #define GRPC_WSA_ERROR(err, call_name) \
   grpc_wsa_error(__FILE__, __LINE__, err, call_name)
 
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
new file mode 100644
index 0000000000..9e306af5fa
--- /dev/null
+++ b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
@@ -0,0 +1,2043 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* This file will be removed shortly: it's here to keep refactoring
+ * steps simple and auditable.
+ * It's the combination of the old files:
+ *  - fd_posix.{h,c}
+ *  - pollset_posix.{h,c}
+ *  - pullset_multipoller_with_{poll,epoll}.{h,c}
+ * The new version will be split into:
+ *  - ev_poll_posix.{h,c}
+ *  - ev_epoll_posix.{h,c}
+ */
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_POSIX_SOCKET
+
+#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <poll.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/tls.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/block_annotate.h"
+
+/*******************************************************************************
+ * FD declarations
+ */
+
+typedef struct grpc_fd_watcher {
+  struct grpc_fd_watcher *next;
+  struct grpc_fd_watcher *prev;
+  grpc_pollset *pollset;
+  grpc_pollset_worker *worker;
+  grpc_fd *fd;
+} grpc_fd_watcher;
+
+struct grpc_fd {
+  int fd;
+  /* refst format:
+     bit0:   1=active/0=orphaned
+     bit1-n: refcount
+     meaning that mostly we ref by two to avoid altering the orphaned bit,
+     and just unref by 1 when we're ready to flag the object as orphaned */
+  gpr_atm refst;
+
+  gpr_mu mu;
+  int shutdown;
+  int closed;
+  int released;
+
+  /* The watcher list.
+
+     The following watcher related fields are protected by watcher_mu.
+
+     An fd_watcher is an ephemeral object created when an fd wants to
+     begin polling, and destroyed after the poll.
+
+     It denotes the fd's interest in whether to read poll or write poll
+     or both or neither on this fd.
+
+     If a watcher is asked to poll for reads or writes, the read_watcher
+     or write_watcher fields are set respectively. A watcher may be asked
+     to poll for both, in which case both fields will be set.
+
+     read_watcher and write_watcher may be NULL if no watcher has been
+     asked to poll for reads or writes.
+
+     If an fd_watcher is not asked to poll for reads or writes, it's added
+     to a linked list of inactive watchers, rooted at inactive_watcher_root.
+     If at a later time there becomes need of a poller to poll, one of
+     the inactive pollers may be kicked out of their poll loops to take
+     that responsibility. */
+  grpc_fd_watcher inactive_watcher_root;
+  grpc_fd_watcher *read_watcher;
+  grpc_fd_watcher *write_watcher;
+
+  grpc_closure *read_closure;
+  grpc_closure *write_closure;
+
+  struct grpc_fd *freelist_next;
+
+  grpc_closure *on_done_closure;
+
+  grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
+};
+
+/* Begin polling on an fd.
+   Registers that the given pollset is interested in this fd - so that if read
+   or writability interest changes, the pollset can be kicked to pick up that
+   new interest.
+   Return value is:
+     (fd_needs_read? read_mask : 0) | (fd_needs_write? write_mask : 0)
+   i.e. a combination of read_mask and write_mask determined by the fd's current
+   interest in said events.
+   Polling strategies that do not need to alter their behavior depending on the
+   fd's current interest (such as epoll) do not need to call this function.
+   MUST NOT be called with a pollset lock taken */
+static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
+                              grpc_pollset_worker *worker, uint32_t read_mask,
+                              uint32_t write_mask, grpc_fd_watcher *rec);
+/* Complete polling previously started with fd_begin_poll
+   MUST NOT be called with a pollset lock taken
+   if got_read or got_write are 1, also does the become_{readable,writable} as
+   appropriate. */
+static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
+
+/* Return 1 if this fd is orphaned, 0 otherwise */
+static bool fd_is_orphaned(grpc_fd *fd);
+
+/* Reference counting for fds */
+/*#define GRPC_FD_REF_COUNT_DEBUG*/
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line);
+static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
+                     int line);
+#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__)
+#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__)
+#else
+static void fd_ref(grpc_fd *fd);
+static void fd_unref(grpc_fd *fd);
+#define GRPC_FD_REF(fd, reason) fd_ref(fd)
+#define GRPC_FD_UNREF(fd, reason) fd_unref(fd)
+#endif
+
+static void fd_global_init(void);
+static void fd_global_shutdown(void);
+
+#define CLOSURE_NOT_READY ((grpc_closure *)0)
+#define CLOSURE_READY ((grpc_closure *)1)
+
+/*******************************************************************************
+ * pollset declarations
+ */
+
+typedef struct grpc_pollset_vtable grpc_pollset_vtable;
+
+typedef struct grpc_cached_wakeup_fd {
+  grpc_wakeup_fd fd;
+  struct grpc_cached_wakeup_fd *next;
+} grpc_cached_wakeup_fd;
+
+struct grpc_pollset_worker {
+  grpc_cached_wakeup_fd *wakeup_fd;
+  int reevaluate_polling_on_wakeup;
+  int kicked_specifically;
+  struct grpc_pollset_worker *next;
+  struct grpc_pollset_worker *prev;
+};
+
+struct grpc_pollset {
+  /* pollsets under posix can mutate representation as fds are added and
+     removed.
+     For example, we may choose a poll() based implementation on linux for
+     few fds, and an epoll() based implementation for many fds */
+  const grpc_pollset_vtable *vtable;
+  gpr_mu mu;
+  grpc_pollset_worker root_worker;
+  int in_flight_cbs;
+  int shutting_down;
+  int called_shutdown;
+  int kicked_without_pollers;
+  grpc_closure *shutdown_done;
+  grpc_closure_list idle_jobs;
+  union {
+    int fd;
+    void *ptr;
+  } data;
+  /* Local cache of eventfds for workers */
+  grpc_cached_wakeup_fd *local_wakeup_cache;
+};
+
+struct grpc_pollset_vtable {
+  void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                 struct grpc_fd *fd, int and_unlock_pollset);
+  grpc_error *(*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx,
+                                       grpc_pollset *pollset,
+                                       grpc_pollset_worker *worker,
+                                       gpr_timespec deadline, gpr_timespec now);
+  void (*finish_shutdown)(grpc_pollset *pollset);
+  void (*destroy)(grpc_pollset *pollset);
+};
+
+/* Add an fd to a pollset */
+static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           struct grpc_fd *fd);
+
+static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd);
+
+/* Convert a timespec to milliseconds:
+   - very small or negative poll times are clamped to zero to do a
+     non-blocking poll (which becomes spin polling)
+   - other small values are rounded up to one millisecond
+   - longer than a millisecond polls are rounded up to the next nearest
+     millisecond to avoid spinning
+   - infinite timeouts are converted to -1 */
+static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
+                                           gpr_timespec now);
+
+/* Allow kick to wakeup the currently polling worker */
+#define GRPC_POLLSET_CAN_KICK_SELF 1
+/* Force the wakee to repoll when awoken */
+#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2
+/* As per pollset_kick, with an extended set of flags (defined above)
+   -- mostly for fd_posix's use. */
+static grpc_error *pollset_kick_ext(grpc_pollset *p,
+                                    grpc_pollset_worker *specific_worker,
+                                    uint32_t flags) GRPC_MUST_USE_RESULT;
+
+/* turn a pollset into a multipoller: platform specific */
+typedef void (*platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx,
+                                                 grpc_pollset *pollset,
+                                                 struct grpc_fd **fds,
+                                                 size_t fd_count);
+static platform_become_multipoller_type platform_become_multipoller;
+
+/* Return 1 if the pollset has active threads in pollset_work (pollset must
+ * be locked) */
+static int pollset_has_workers(grpc_pollset *pollset);
+
+static void remove_fd_from_all_epoll_sets(int fd);
+
+/*******************************************************************************
+ * pollset_set definitions
+ */
+
+struct grpc_pollset_set {
+  gpr_mu mu;
+
+  size_t pollset_count;
+  size_t pollset_capacity;
+  grpc_pollset **pollsets;
+
+  size_t pollset_set_count;
+  size_t pollset_set_capacity;
+  struct grpc_pollset_set **pollset_sets;
+
+  size_t fd_count;
+  size_t fd_capacity;
+  grpc_fd **fds;
+};
+
+/*******************************************************************************
+ * fd_posix.c
+ */
+
+/* We need to keep a freelist not because of any concerns of malloc performance
+ * but instead so that implementations with multiple threads in (for example)
+ * epoll_wait deal with the race between pollset removal and incoming poll
+ * notifications.
+ *
+ * The problem is that the poller ultimately holds a reference to this
+ * object, so it is very difficult to know when is safe to free it, at least
+ * without some expensive synchronization.
+ *
+ * If we keep the object freelisted, in the worst case losing this race just
+ * becomes a spurious read notification on a reused fd.
+ */
+/* TODO(klempner): We could use some form of polling generation count to know
+ * when these are safe to free. */
+/* TODO(klempner): Consider disabling freelisting if we don't have multiple
+ * threads in poll on the same fd */
+/* TODO(klempner): Batch these allocations to reduce fragmentation */
+static grpc_fd *fd_freelist = NULL;
+static gpr_mu fd_freelist_mu;
+
+static void freelist_fd(grpc_fd *fd) {
+  gpr_mu_lock(&fd_freelist_mu);
+  fd->freelist_next = fd_freelist;
+  fd_freelist = fd;
+  grpc_iomgr_unregister_object(&fd->iomgr_object);
+  gpr_mu_unlock(&fd_freelist_mu);
+}
+
+static grpc_fd *alloc_fd(int fd) {
+  grpc_fd *r = NULL;
+  gpr_mu_lock(&fd_freelist_mu);
+  if (fd_freelist != NULL) {
+    r = fd_freelist;
+    fd_freelist = fd_freelist->freelist_next;
+  }
+  gpr_mu_unlock(&fd_freelist_mu);
+  if (r == NULL) {
+    r = gpr_malloc(sizeof(grpc_fd));
+    gpr_mu_init(&r->mu);
+  }
+
+  gpr_mu_lock(&r->mu);
+  gpr_atm_rel_store(&r->refst, 1);
+  r->shutdown = 0;
+  r->read_closure = CLOSURE_NOT_READY;
+  r->write_closure = CLOSURE_NOT_READY;
+  r->fd = fd;
+  r->inactive_watcher_root.next = r->inactive_watcher_root.prev =
+      &r->inactive_watcher_root;
+  r->freelist_next = NULL;
+  r->read_watcher = r->write_watcher = NULL;
+  r->on_done_closure = NULL;
+  r->closed = 0;
+  r->released = 0;
+  r->read_notifier_pollset = NULL;
+  gpr_mu_unlock(&r->mu);
+  return r;
+}
+
+static void destroy(grpc_fd *fd) {
+  gpr_mu_destroy(&fd->mu);
+  gpr_free(fd);
+}
+
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__)
+#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__)
+static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file,
+                   int line) {
+  gpr_log(GPR_DEBUG, "FD %d %p   ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
+          gpr_atm_no_barrier_load(&fd->refst),
+          gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line);
+#else
+#define REF_BY(fd, n, reason) ref_by(fd, n)
+#define UNREF_BY(fd, n, reason) unref_by(fd, n)
+static void ref_by(grpc_fd *fd, int n) {
+#endif
+  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
+}
+
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file,
+                     int line) {
+  gpr_atm old;
+  gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
+          gpr_atm_no_barrier_load(&fd->refst),
+          gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line);
+#else
+static void unref_by(grpc_fd *fd, int n) {
+  gpr_atm old;
+#endif
+  old = gpr_atm_full_fetch_add(&fd->refst, -n);
+  if (old == n) {
+    freelist_fd(fd);
+  } else {
+    GPR_ASSERT(old > n);
+  }
+}
+
+static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); }
+
+static void fd_global_shutdown(void) {
+  gpr_mu_lock(&fd_freelist_mu);
+  gpr_mu_unlock(&fd_freelist_mu);
+  while (fd_freelist != NULL) {
+    grpc_fd *fd = fd_freelist;
+    fd_freelist = fd_freelist->freelist_next;
+    destroy(fd);
+  }
+  gpr_mu_destroy(&fd_freelist_mu);
+}
+
+static grpc_fd *fd_create(int fd, const char *name) {
+  grpc_fd *r = alloc_fd(fd);
+  char *name2;
+  gpr_asprintf(&name2, "%s fd=%d", name, fd);
+  grpc_iomgr_register_object(&r->iomgr_object, name2);
+  gpr_free(name2);
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+  gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name);
+#endif
+  return r;
+}
+
+static bool fd_is_orphaned(grpc_fd *fd) {
+  return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
+}
+
+static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) {
+  gpr_mu_lock(&watcher->pollset->mu);
+  GPR_ASSERT(watcher->worker);
+  grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker,
+                                     GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
+  gpr_mu_unlock(&watcher->pollset->mu);
+  return err;
+}
+
+static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
+  if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) {
+    pollset_kick_locked(fd->inactive_watcher_root.next);
+  } else if (fd->read_watcher) {
+    pollset_kick_locked(fd->read_watcher);
+  } else if (fd->write_watcher) {
+    pollset_kick_locked(fd->write_watcher);
+  }
+}
+
+static void wake_all_watchers_locked(grpc_fd *fd) {
+  grpc_fd_watcher *watcher;
+  for (watcher = fd->inactive_watcher_root.next;
+       watcher != &fd->inactive_watcher_root; watcher = watcher->next) {
+    pollset_kick_locked(watcher);
+  }
+  if (fd->read_watcher) {
+    pollset_kick_locked(fd->read_watcher);
+  }
+  if (fd->write_watcher && fd->write_watcher != fd->read_watcher) {
+    pollset_kick_locked(fd->write_watcher);
+  }
+}
+
+static int has_watchers(grpc_fd *fd) {
+  return fd->read_watcher != NULL || fd->write_watcher != NULL ||
+         fd->inactive_watcher_root.next != &fd->inactive_watcher_root;
+}
+
+static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  fd->closed = 1;
+  if (!fd->released) {
+    close(fd->fd);
+  } else {
+    remove_fd_from_all_epoll_sets(fd->fd);
+  }
+  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL);
+}
+
+static int fd_wrapped_fd(grpc_fd *fd) {
+  if (fd->released || fd->closed) {
+    return -1;
+  } else {
+    return fd->fd;
+  }
+}
+
+static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                      grpc_closure *on_done, int *release_fd,
+                      const char *reason) {
+  fd->on_done_closure = on_done;
+  fd->released = release_fd != NULL;
+  if (!fd->released) {
+    shutdown(fd->fd, SHUT_RDWR);
+  } else {
+    *release_fd = fd->fd;
+  }
+  gpr_mu_lock(&fd->mu);
+  REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
+  if (!has_watchers(fd)) {
+    close_fd_locked(exec_ctx, fd);
+  } else {
+    wake_all_watchers_locked(fd);
+  }
+  gpr_mu_unlock(&fd->mu);
+  UNREF_BY(fd, 2, reason); /* drop the reference */
+}
+
+/* increment refcount by two to avoid changing the orphan bit */
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void fd_ref(grpc_fd *fd, const char *reason, const char *file,
+                   int line) {
+  ref_by(fd, 2, reason, file, line);
+}
+
+static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
+                     int line) {
+  unref_by(fd, 2, reason, file, line);
+}
+#else
+static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
+
+static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
+#endif
+
+static grpc_error *fd_shutdown_error(bool shutdown) {
+  if (!shutdown) {
+    return GRPC_ERROR_NONE;
+  } else {
+    return GRPC_ERROR_CREATE("FD shutdown");
+  }
+}
+
+static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                             grpc_closure **st, grpc_closure *closure) {
+  if (fd->shutdown) {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"),
+                        NULL);
+  } else if (*st == CLOSURE_NOT_READY) {
+    /* not ready ==> switch to a waiting state by setting the closure */
+    *st = closure;
+  } else if (*st == CLOSURE_READY) {
+    /* already ready ==> queue the closure to run immediately */
+    *st = CLOSURE_NOT_READY;
+    grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
+                        NULL);
+    maybe_wake_one_watcher_locked(fd);
+  } else {
+    /* upcallptr was set to a different closure.  This is an error! */
+    gpr_log(GPR_ERROR,
+            "User called a notify_on function with a previous callback still "
+            "pending");
+    abort();
+  }
+}
+
+/* returns 1 if state becomes not ready */
+static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                            grpc_closure **st) {
+  if (*st == CLOSURE_READY) {
+    /* duplicate ready ==> ignore */
+    return 0;
+  } else if (*st == CLOSURE_NOT_READY) {
+    /* not ready, and not waiting ==> flag ready */
+    *st = CLOSURE_READY;
+    return 0;
+  } else {
+    /* waiting ==> queue closure */
+    grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
+    *st = CLOSURE_NOT_READY;
+    return 1;
+  }
+}
+
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
+static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  /* only shutdown once */
+  if (!fd->shutdown) {
+    fd->shutdown = 1;
+    /* signal read/write closed to OS so that future operations fail */
+    shutdown(fd->fd, SHUT_RDWR);
+    set_ready_locked(exec_ctx, fd, &fd->read_closure);
+    set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  }
+  gpr_mu_unlock(&fd->mu);
+}
+
+static bool fd_is_shutdown(grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  bool r = fd->shutdown;
+  gpr_mu_unlock(&fd->mu);
+  return r;
+}
+
+static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                              grpc_closure *closure) {
+  gpr_mu_lock(&fd->mu);
+  notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_closure *closure) {
+  gpr_mu_lock(&fd->mu);
+  notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
+static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
+                              grpc_pollset_worker *worker, uint32_t read_mask,
+                              uint32_t write_mask, grpc_fd_watcher *watcher) {
+  uint32_t mask = 0;
+  grpc_closure *cur;
+  int requested;
+  /* keep track of pollers that have requested our events, in case they change
+   */
+  GRPC_FD_REF(fd, "poll");
+
+  gpr_mu_lock(&fd->mu);
+
+  /* if we are shutdown, then don't add to the watcher set */
+  if (fd->shutdown) {
+    watcher->fd = NULL;
+    watcher->pollset = NULL;
+    watcher->worker = NULL;
+    gpr_mu_unlock(&fd->mu);
+    GRPC_FD_UNREF(fd, "poll");
+    return 0;
+  }
+
+  /* if there is nobody polling for read, but we need to, then start doing so */
+  cur = fd->read_closure;
+  requested = cur != CLOSURE_READY;
+  if (read_mask && fd->read_watcher == NULL && requested) {
+    fd->read_watcher = watcher;
+    mask |= read_mask;
+  }
+  /* if there is nobody polling for write, but we need to, then start doing so
+   */
+  cur = fd->write_closure;
+  requested = cur != CLOSURE_READY;
+  if (write_mask && fd->write_watcher == NULL && requested) {
+    fd->write_watcher = watcher;
+    mask |= write_mask;
+  }
+  /* if not polling, remember this watcher in case we need someone to later */
+  if (mask == 0 && worker != NULL) {
+    watcher->next = &fd->inactive_watcher_root;
+    watcher->prev = watcher->next->prev;
+    watcher->next->prev = watcher->prev->next = watcher;
+  }
+  watcher->pollset = pollset;
+  watcher->worker = worker;
+  watcher->fd = fd;
+  gpr_mu_unlock(&fd->mu);
+
+  return mask;
+}
+
+static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
+  int was_polling = 0;
+  int kick = 0;
+  grpc_fd *fd = watcher->fd;
+
+  if (fd == NULL) {
+    return;
+  }
+
+  gpr_mu_lock(&fd->mu);
+
+  if (watcher == fd->read_watcher) {
+    /* remove read watcher, kick if we still need a read */
+    was_polling = 1;
+    if (!got_read) {
+      kick = 1;
+    }
+    fd->read_watcher = NULL;
+  }
+  if (watcher == fd->write_watcher) {
+    /* remove write watcher, kick if we still need a write */
+    was_polling = 1;
+    if (!got_write) {
+      kick = 1;
+    }
+    fd->write_watcher = NULL;
+  }
+  if (!was_polling && watcher->worker != NULL) {
+    /* remove from inactive list */
+    watcher->next->prev = watcher->prev;
+    watcher->prev->next = watcher->next;
+  }
+  if (got_read) {
+    if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
+      kick = 1;
+    }
+
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
+  }
+  if (got_write) {
+    if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
+      kick = 1;
+    }
+  }
+  if (kick) {
+    maybe_wake_one_watcher_locked(fd);
+  }
+  if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) {
+    close_fd_locked(exec_ctx, fd);
+  }
+  gpr_mu_unlock(&fd->mu);
+
+  GRPC_FD_UNREF(fd, "poll");
+}
+
+/*******************************************************************************
+ * pollset_posix.c
+ */
+
+GPR_TLS_DECL(g_current_thread_poller);
+GPR_TLS_DECL(g_current_thread_worker);
+
+/** The alarm system needs to be able to wakeup 'some poller' sometimes
+ *  (specifically when a new alarm needs to be triggered earlier than the next
+ *  alarm 'epoch').
+ *  This wakeup_fd gives us something to alert on when such a case occurs. */
+grpc_wakeup_fd grpc_global_wakeup_fd;
+
+static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev->next = worker->next;
+  worker->next->prev = worker->prev;
+}
+
+static int pollset_has_workers(grpc_pollset *p) {
+  return p->root_worker.next != &p->root_worker;
+}
+
+static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
+  if (pollset_has_workers(p)) {
+    grpc_pollset_worker *w = p->root_worker.next;
+    remove_worker(p, w);
+    return w;
+  } else {
+    return NULL;
+  }
+}
+
+static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->next = &p->root_worker;
+  worker->prev = worker->next->prev;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev = &p->root_worker;
+  worker->next = worker->prev->next;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void kick_append_error(grpc_error **composite, grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE("Kick Failure");
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+static grpc_error *pollset_kick_ext(grpc_pollset *p,
+                                    grpc_pollset_worker *specific_worker,
+                                    uint32_t flags) {
+  GPR_TIMER_BEGIN("pollset_kick_ext", 0);
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  /* pollset->mu already held */
+  if (specific_worker != NULL) {
+    if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
+      GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0);
+      GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
+      for (specific_worker = p->root_worker.next;
+           specific_worker != &p->root_worker;
+           specific_worker = specific_worker->next) {
+        kick_append_error(
+            &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+      }
+      p->kicked_without_pollers = true;
+      GPR_TIMER_END("pollset_kick_ext.broadcast", 0);
+    } else if (gpr_tls_get(&g_current_thread_worker) !=
+               (intptr_t)specific_worker) {
+      GPR_TIMER_MARK("different_thread_worker", 0);
+      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
+        specific_worker->reevaluate_polling_on_wakeup = true;
+      }
+      specific_worker->kicked_specifically = true;
+      kick_append_error(&error,
+                        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+    } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) {
+      GPR_TIMER_MARK("kick_yoself", 0);
+      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
+        specific_worker->reevaluate_polling_on_wakeup = true;
+      }
+      specific_worker->kicked_specifically = true;
+      kick_append_error(&error,
+                        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+    }
+  } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) {
+    GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
+    GPR_TIMER_MARK("kick_anonymous", 0);
+    specific_worker = pop_front_worker(p);
+    if (specific_worker != NULL) {
+      if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) {
+        GPR_TIMER_MARK("kick_anonymous_not_self", 0);
+        push_back_worker(p, specific_worker);
+        specific_worker = pop_front_worker(p);
+        if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 &&
+            gpr_tls_get(&g_current_thread_worker) ==
+                (intptr_t)specific_worker) {
+          push_back_worker(p, specific_worker);
+          specific_worker = NULL;
+        }
+      }
+      if (specific_worker != NULL) {
+        GPR_TIMER_MARK("finally_kick", 0);
+        push_back_worker(p, specific_worker);
+        kick_append_error(
+            &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+      }
+    } else {
+      GPR_TIMER_MARK("kicked_no_pollers", 0);
+      p->kicked_without_pollers = true;
+    }
+  }
+
+  GPR_TIMER_END("pollset_kick_ext", 0);
+  return error;
+}
+
+static grpc_error *pollset_kick(grpc_pollset *p,
+                                grpc_pollset_worker *specific_worker) {
+  return pollset_kick_ext(p, specific_worker, 0);
+}
+
+/* global state management */
+
+static grpc_error *pollset_global_init(void) {
+  gpr_tls_init(&g_current_thread_poller);
+  gpr_tls_init(&g_current_thread_worker);
+  return grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
+}
+
+static void pollset_global_shutdown(void) {
+  grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd);
+  gpr_tls_destroy(&g_current_thread_poller);
+  gpr_tls_destroy(&g_current_thread_worker);
+}
+
+static grpc_error *kick_poller(void) {
+  return grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd);
+}
+
+/* main interface */
+
+static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null);
+
+static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
+  gpr_mu_init(&pollset->mu);
+  *mu = &pollset->mu;
+  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
+  pollset->in_flight_cbs = 0;
+  pollset->shutting_down = 0;
+  pollset->called_shutdown = 0;
+  pollset->kicked_without_pollers = 0;
+  pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL;
+  pollset->local_wakeup_cache = NULL;
+  pollset->kicked_without_pollers = 0;
+  become_basic_pollset(pollset, NULL);
+}
+
+static void pollset_destroy(grpc_pollset *pollset) {
+  GPR_ASSERT(pollset->in_flight_cbs == 0);
+  GPR_ASSERT(!pollset_has_workers(pollset));
+  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
+  pollset->vtable->destroy(pollset);
+  while (pollset->local_wakeup_cache) {
+    grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next;
+    grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd);
+    gpr_free(pollset->local_wakeup_cache);
+    pollset->local_wakeup_cache = next;
+  }
+  gpr_mu_destroy(&pollset->mu);
+}
+
+static void pollset_reset(grpc_pollset *pollset) {
+  GPR_ASSERT(pollset->shutting_down);
+  GPR_ASSERT(pollset->in_flight_cbs == 0);
+  GPR_ASSERT(!pollset_has_workers(pollset));
+  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
+  pollset->vtable->destroy(pollset);
+  pollset->shutting_down = 0;
+  pollset->called_shutdown = 0;
+  pollset->kicked_without_pollers = 0;
+  become_basic_pollset(pollset, NULL);
+}
+
+static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           grpc_fd *fd) {
+  gpr_mu_lock(&pollset->mu);
+  pollset->vtable->add_fd(exec_ctx, pollset, fd, 1);
+/* the following (enabled only in debug) will reacquire and then release
+   our lock - meaning that if the unlocking flag passed to add_fd above is
+   not respected, the code will deadlock (in a way that we have a chance of
+   debugging) */
+#ifndef NDEBUG
+  gpr_mu_lock(&pollset->mu);
+  gpr_mu_unlock(&pollset->mu);
+#endif
+}
+
+static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
+  GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs));
+  pollset->vtable->finish_shutdown(pollset);
+  grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
+}
+
+static void work_combine_error(grpc_error **composite, grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE("pollset_work");
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                                grpc_pollset_worker **worker_hdl,
+                                gpr_timespec now, gpr_timespec deadline) {
+  grpc_pollset_worker worker;
+  *worker_hdl = &worker;
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  /* pollset->mu already held */
+  int added_worker = 0;
+  int locked = 1;
+  int queued_work = 0;
+  int keep_polling = 0;
+  GPR_TIMER_BEGIN("pollset_work", 0);
+  /* this must happen before we (potentially) drop pollset->mu */
+  worker.next = worker.prev = NULL;
+  worker.reevaluate_polling_on_wakeup = 0;
+  if (pollset->local_wakeup_cache != NULL) {
+    worker.wakeup_fd = pollset->local_wakeup_cache;
+    pollset->local_wakeup_cache = worker.wakeup_fd->next;
+  } else {
+    worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
+    error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
+    if (error != GRPC_ERROR_NONE) {
+      return error;
+    }
+  }
+  worker.kicked_specifically = 0;
+  /* If there's work waiting for the pollset to be idle, and the
+     pollset is idle, then do that work */
+  if (!pollset_has_workers(pollset) &&
+      !grpc_closure_list_empty(pollset->idle_jobs)) {
+    GPR_TIMER_MARK("pollset_work.idle_jobs", 0);
+    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+    goto done;
+  }
+  /* If we're shutting down then we don't execute any extended work */
+  if (pollset->shutting_down) {
+    GPR_TIMER_MARK("pollset_work.shutting_down", 0);
+    goto done;
+  }
+  /* Give do_promote priority so we don't starve it out */
+  if (pollset->in_flight_cbs) {
+    GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0);
+    gpr_mu_unlock(&pollset->mu);
+    locked = 0;
+    goto done;
+  }
+  /* Start polling, and keep doing so while we're being asked to
+     re-evaluate our pollers (this allows poll() based pollers to
+     ensure they don't miss wakeups) */
+  keep_polling = 1;
+  while (keep_polling) {
+    keep_polling = 0;
+    if (!pollset->kicked_without_pollers) {
+      if (!added_worker) {
+        push_front_worker(pollset, &worker);
+        added_worker = 1;
+        gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
+      }
+      gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
+      GPR_TIMER_BEGIN("maybe_work_and_unlock", 0);
+      work_combine_error(&error,
+                         pollset->vtable->maybe_work_and_unlock(
+                             exec_ctx, pollset, &worker, deadline, now));
+      GPR_TIMER_END("maybe_work_and_unlock", 0);
+      locked = 0;
+      gpr_tls_set(&g_current_thread_poller, 0);
+    } else {
+      GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0);
+      pollset->kicked_without_pollers = 0;
+    }
+  /* Finished execution - start cleaning up.
+     Note that we may arrive here from outside the enclosing while() loop.
+     In that case we won't loop though as we haven't added worker to the
+     worker list, which means nobody could ask us to re-evaluate polling). */
+  done:
+    if (!locked) {
+      queued_work |= grpc_exec_ctx_flush(exec_ctx);
+      gpr_mu_lock(&pollset->mu);
+      locked = 1;
+    }
+    /* If we're forced to re-evaluate polling (via pollset_kick with
+       GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force
+       a loop */
+    if (worker.reevaluate_polling_on_wakeup) {
+      worker.reevaluate_polling_on_wakeup = 0;
+      pollset->kicked_without_pollers = 0;
+      if (queued_work || worker.kicked_specifically) {
+        /* If there's queued work on the list, then set the deadline to be
+           immediate so we get back out of the polling loop quickly */
+        deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
+      }
+      keep_polling = 1;
+    }
+  }
+  if (added_worker) {
+    remove_worker(pollset, &worker);
+    gpr_tls_set(&g_current_thread_worker, 0);
+  }
+  /* release wakeup fd to the local pool */
+  worker.wakeup_fd->next = pollset->local_wakeup_cache;
+  pollset->local_wakeup_cache = worker.wakeup_fd;
+  /* check shutdown conditions */
+  if (pollset->shutting_down) {
+    if (pollset_has_workers(pollset)) {
+      pollset_kick(pollset, NULL);
+    } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
+      pollset->called_shutdown = 1;
+      gpr_mu_unlock(&pollset->mu);
+      finish_shutdown(exec_ctx, pollset);
+      grpc_exec_ctx_flush(exec_ctx);
+      /* Continuing to access pollset here is safe -- it is the caller's
+       * responsibility to not destroy when it has outstanding calls to
+       * pollset_work.
+       * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */
+      gpr_mu_lock(&pollset->mu);
+    } else if (!grpc_closure_list_empty(pollset->idle_jobs)) {
+      grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+      gpr_mu_unlock(&pollset->mu);
+      grpc_exec_ctx_flush(exec_ctx);
+      gpr_mu_lock(&pollset->mu);
+    }
+  }
+  *worker_hdl = NULL;
+  GPR_TIMER_END("pollset_work", 0);
+  return error;
+}
+
+static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                             grpc_closure *closure) {
+  GPR_ASSERT(!pollset->shutting_down);
+  pollset->shutting_down = 1;
+  pollset->shutdown_done = closure;
+  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
+  if (!pollset_has_workers(pollset)) {
+    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+  }
+  if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
+      !pollset_has_workers(pollset)) {
+    pollset->called_shutdown = 1;
+    finish_shutdown(exec_ctx, pollset);
+  }
+}
+
+static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
+                                           gpr_timespec now) {
+  gpr_timespec timeout;
+  static const int64_t max_spin_polling_us = 10;
+  if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) {
+    return -1;
+  }
+  if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros(
+                                                   max_spin_polling_us,
+                                                   GPR_TIMESPAN))) <= 0) {
+    return 0;
+  }
+  timeout = gpr_time_sub(deadline, now);
+  return gpr_time_to_millis(gpr_time_add(
+      timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN)));
+}
+
+/*
+ * basic_pollset - a vtable that provides polling for zero or one file
+ *                 descriptor via poll()
+ */
+
+typedef struct grpc_unary_promote_args {
+  const grpc_pollset_vtable *original_vtable;
+  grpc_pollset *pollset;
+  grpc_fd *fd;
+  grpc_closure promotion_closure;
+} grpc_unary_promote_args;
+
+static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args,
+                             grpc_error *error) {
+  grpc_unary_promote_args *up_args = args;
+  const grpc_pollset_vtable *original_vtable = up_args->original_vtable;
+  grpc_pollset *pollset = up_args->pollset;
+  grpc_fd *fd = up_args->fd;
+
+  /*
+   * This is quite tricky. There are a number of cases to keep in mind here:
+   * 1. fd may have been orphaned
+   * 2. The pollset may no longer be a unary poller (and we can't let case #1
+   * leak to other pollset types!)
+   * 3. pollset's fd (which may have changed) may have been orphaned
+   * 4. The pollset may be shutting down.
+   */
+
+  gpr_mu_lock(&pollset->mu);
+  /* First we need to ensure that nobody is polling concurrently */
+  GPR_ASSERT(!pollset_has_workers(pollset));
+
+  gpr_free(up_args);
+  /* At this point the pollset may no longer be a unary poller. In that case
+   * we should just call the right add function and be done. */
+  /* TODO(klempner): If we're not careful this could cause infinite recursion.
+   * That's not a problem for now because empty_pollset has a trivial poller
+   * and we don't have any mechanism to unbecome multipoller. */
+  pollset->in_flight_cbs--;
+  if (pollset->shutting_down) {
+    /* We don't care about this pollset anymore. */
+    if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) {
+      pollset->called_shutdown = 1;
+      finish_shutdown(exec_ctx, pollset);
+    }
+  } else if (fd_is_orphaned(fd)) {
+    /* Don't try to add it to anything, we'll drop our ref on it below */
+  } else if (pollset->vtable != original_vtable) {
+    pollset->vtable->add_fd(exec_ctx, pollset, fd, 0);
+  } else if (fd != pollset->data.ptr) {
+    grpc_fd *fds[2];
+    fds[0] = pollset->data.ptr;
+    fds[1] = fd;
+
+    if (fds[0] && !fd_is_orphaned(fds[0])) {
+      platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds));
+      GRPC_FD_UNREF(fds[0], "basicpoll");
+    } else {
+      /* old fd is orphaned and we haven't cleaned it up until now, so remain a
+       * unary poller */
+      /* Note that it is possible that fds[1] is also orphaned at this point.
+       * That's okay, we'll correct it at the next add or poll. */
+      if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll");
+      pollset->data.ptr = fd;
+      GRPC_FD_REF(fd, "basicpoll");
+    }
+  }
+
+  gpr_mu_unlock(&pollset->mu);
+
+  /* Matching ref in basic_pollset_add_fd */
+  GRPC_FD_UNREF(fd, "basicpoll_add");
+}
+
+static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                                 grpc_fd *fd, int and_unlock_pollset) {
+  grpc_unary_promote_args *up_args;
+  GPR_ASSERT(fd);
+  if (fd == pollset->data.ptr) goto exit;
+
+  if (!pollset_has_workers(pollset)) {
+    /* Fast path -- no in flight cbs */
+    /* TODO(klempner): Comment this out and fix any test failures or establish
+     * they are due to timing issues */
+    grpc_fd *fds[2];
+    fds[0] = pollset->data.ptr;
+    fds[1] = fd;
+
+    if (fds[0] == NULL) {
+      pollset->data.ptr = fd;
+      GRPC_FD_REF(fd, "basicpoll");
+    } else if (!fd_is_orphaned(fds[0])) {
+      platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds));
+      GRPC_FD_UNREF(fds[0], "basicpoll");
+    } else {
+      /* old fd is orphaned and we haven't cleaned it up until now, so remain a
+       * unary poller */
+      GRPC_FD_UNREF(fds[0], "basicpoll");
+      pollset->data.ptr = fd;
+      GRPC_FD_REF(fd, "basicpoll");
+    }
+    goto exit;
+  }
+
+  /* Now we need to promote. This needs to happen when we're not polling. Since
+   * this may be called from poll, the wait needs to happen asynchronously. */
+  GRPC_FD_REF(fd, "basicpoll_add");
+  pollset->in_flight_cbs++;
+  up_args = gpr_malloc(sizeof(*up_args));
+  up_args->fd = fd;
+  up_args->original_vtable = pollset->vtable;
+  up_args->pollset = pollset;
+  up_args->promotion_closure.cb = basic_do_promote;
+  up_args->promotion_closure.cb_arg = up_args;
+
+  grpc_closure_list_append(&pollset->idle_jobs, &up_args->promotion_closure,
+                           GRPC_ERROR_NONE);
+  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
+
+exit:
+  if (and_unlock_pollset) {
+    gpr_mu_unlock(&pollset->mu);
+  }
+}
+
+static grpc_error *basic_pollset_maybe_work_and_unlock(
+    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
+    gpr_timespec deadline, gpr_timespec now) {
+#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
+#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
+
+  struct pollfd pfd[3];
+  grpc_fd *fd;
+  grpc_fd_watcher fd_watcher;
+  int timeout;
+  int r;
+  nfds_t nfds;
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  fd = pollset->data.ptr;
+  if (fd && fd_is_orphaned(fd)) {
+    GRPC_FD_UNREF(fd, "basicpoll");
+    fd = pollset->data.ptr = NULL;
+  }
+  timeout = poll_deadline_to_millis_timeout(deadline, now);
+  pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
+  pfd[0].events = POLLIN;
+  pfd[0].revents = 0;
+  pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
+  pfd[1].events = POLLIN;
+  pfd[1].revents = 0;
+  nfds = 2;
+  if (fd) {
+    pfd[2].fd = fd->fd;
+    pfd[2].revents = 0;
+    GRPC_FD_REF(fd, "basicpoll_begin");
+    gpr_mu_unlock(&pollset->mu);
+    pfd[2].events =
+        (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher);
+    if (pfd[2].events != 0) {
+      nfds++;
+    }
+  } else {
+    gpr_mu_unlock(&pollset->mu);
+  }
+
+  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
+     even going into the blocking annotation if possible */
+  /* poll fd count (argument 2) is shortened by one if we have no events
+     to poll on - such that it only includes the kicker */
+  GPR_TIMER_BEGIN("poll", 0);
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
+  r = grpc_poll_function(pfd, nfds, timeout);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
+  GPR_TIMER_END("poll", 0);
+
+  if (r < 0) {
+    if (errno != EINTR) {
+      work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
+    }
+    if (fd) {
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
+    }
+  } else if (r == 0) {
+    if (fd) {
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
+    }
+  } else {
+    if (pfd[0].revents & POLLIN_CHECK) {
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd));
+    }
+    if (pfd[1].revents & POLLIN_CHECK) {
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd));
+    }
+    if (nfds > 2) {
+      fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK,
+                  pfd[2].revents & POLLOUT_CHECK, pollset);
+    } else if (fd) {
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
+    }
+  }
+
+  if (fd) {
+    GRPC_FD_UNREF(fd, "basicpoll_begin");
+  }
+
+  return error;
+}
+
+static void basic_pollset_destroy(grpc_pollset *pollset) {
+  if (pollset->data.ptr != NULL) {
+    GRPC_FD_UNREF(pollset->data.ptr, "basicpoll");
+    pollset->data.ptr = NULL;
+  }
+}
+
+static const grpc_pollset_vtable basic_pollset = {
+    basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock,
+    basic_pollset_destroy, basic_pollset_destroy};
+
+static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) {
+  pollset->vtable = &basic_pollset;
+  pollset->data.ptr = fd_or_null;
+  if (fd_or_null != NULL) {
+    GRPC_FD_REF(fd_or_null, "basicpoll");
+  }
+}
+
+/*******************************************************************************
+ * pollset_multipoller_with_poll_posix.c
+ */
+
+#ifndef GPR_LINUX_MULTIPOLL_WITH_EPOLL
+
+typedef struct {
+  /* all polled fds */
+  size_t fd_count;
+  size_t fd_capacity;
+  grpc_fd **fds;
+  /* fds that have been removed from the pollset explicitly */
+  size_t del_count;
+  size_t del_capacity;
+  grpc_fd **dels;
+} poll_hdr;
+
+static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
+                                               grpc_pollset *pollset,
+                                               grpc_fd *fd,
+                                               int and_unlock_pollset) {
+  size_t i;
+  poll_hdr *h = pollset->data.ptr;
+  /* TODO(ctiller): this is O(num_fds^2); maybe switch to a hash set here */
+  for (i = 0; i < h->fd_count; i++) {
+    if (h->fds[i] == fd) goto exit;
+  }
+  if (h->fd_count == h->fd_capacity) {
+    h->fd_capacity = GPR_MAX(h->fd_capacity + 8, h->fd_count * 3 / 2);
+    h->fds = gpr_realloc(h->fds, sizeof(grpc_fd *) * h->fd_capacity);
+  }
+  h->fds[h->fd_count++] = fd;
+  GRPC_FD_REF(fd, "multipoller");
+exit:
+  if (and_unlock_pollset) {
+    gpr_mu_unlock(&pollset->mu);
+  }
+}
+
+static grpc_error *multipoll_with_poll_pollset_maybe_work_and_unlock(
+    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
+    gpr_timespec deadline, gpr_timespec now) {
+#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
+#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
+
+  int timeout;
+  int r;
+  size_t i, j, fd_count;
+  nfds_t pfd_count;
+  poll_hdr *h;
+  /* TODO(ctiller): inline some elements to avoid an allocation */
+  grpc_fd_watcher *watchers;
+  struct pollfd *pfds;
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  h = pollset->data.ptr;
+  timeout = poll_deadline_to_millis_timeout(deadline, now);
+  /* TODO(ctiller): perform just one malloc here if we exceed the inline case */
+  pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 2));
+  watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 2));
+  fd_count = 0;
+  pfd_count = 2;
+  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
+  pfds[0].events = POLLIN;
+  pfds[0].revents = 0;
+  pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
+  pfds[1].events = POLLIN;
+  pfds[1].revents = 0;
+  for (i = 0; i < h->fd_count; i++) {
+    int remove = fd_is_orphaned(h->fds[i]);
+    for (j = 0; !remove && j < h->del_count; j++) {
+      if (h->fds[i] == h->dels[j]) remove = 1;
+    }
+    if (remove) {
+      GRPC_FD_UNREF(h->fds[i], "multipoller");
+    } else {
+      h->fds[fd_count++] = h->fds[i];
+      watchers[pfd_count].fd = h->fds[i];
+      GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start");
+      pfds[pfd_count].fd = h->fds[i]->fd;
+      pfds[pfd_count].revents = 0;
+      pfd_count++;
+    }
+  }
+  for (j = 0; j < h->del_count; j++) {
+    GRPC_FD_UNREF(h->dels[j], "multipoller_del");
+  }
+  h->del_count = 0;
+  h->fd_count = fd_count;
+  gpr_mu_unlock(&pollset->mu);
+
+  for (i = 2; i < pfd_count; i++) {
+    grpc_fd *fd = watchers[i].fd;
+    pfds[i].events = (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT,
+                                          &watchers[i]);
+    GRPC_FD_UNREF(fd, "multipoller_start");
+  }
+
+  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
+     even going into the blocking annotation if possible */
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
+  r = grpc_poll_function(pfds, pfd_count, timeout);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
+
+  if (r < 0) {
+    if (errno != EINTR) {
+      work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
+    }
+    for (i = 2; i < pfd_count; i++) {
+      fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
+    }
+  } else if (r == 0) {
+    for (i = 2; i < pfd_count; i++) {
+      fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
+    }
+  } else {
+    if (pfds[0].revents & POLLIN_CHECK) {
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd));
+    }
+    if (pfds[1].revents & POLLIN_CHECK) {
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd));
+    }
+    for (i = 2; i < pfd_count; i++) {
+      if (watchers[i].fd == NULL) {
+        fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
+        continue;
+      }
+      fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
+                  pfds[i].revents & POLLOUT_CHECK, pollset);
+    }
+  }
+
+  gpr_free(pfds);
+  gpr_free(watchers);
+
+  return error;
+}
+
+static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) {
+  size_t i;
+  poll_hdr *h = pollset->data.ptr;
+  for (i = 0; i < h->fd_count; i++) {
+    GRPC_FD_UNREF(h->fds[i], "multipoller");
+  }
+  for (i = 0; i < h->del_count; i++) {
+    GRPC_FD_UNREF(h->dels[i], "multipoller_del");
+  }
+  h->fd_count = 0;
+  h->del_count = 0;
+}
+
+static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
+  poll_hdr *h = pollset->data.ptr;
+  multipoll_with_poll_pollset_finish_shutdown(pollset);
+  gpr_free(h->fds);
+  gpr_free(h->dels);
+  gpr_free(h);
+}
+
+static const grpc_pollset_vtable multipoll_with_poll_pollset = {
+    multipoll_with_poll_pollset_add_fd,
+    multipoll_with_poll_pollset_maybe_work_and_unlock,
+    multipoll_with_poll_pollset_finish_shutdown,
+    multipoll_with_poll_pollset_destroy};
+
+static void poll_become_multipoller(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset *pollset, grpc_fd **fds,
+                                    size_t nfds) {
+  size_t i;
+  poll_hdr *h = gpr_malloc(sizeof(poll_hdr));
+  pollset->vtable = &multipoll_with_poll_pollset;
+  pollset->data.ptr = h;
+  h->fd_count = nfds;
+  h->fd_capacity = nfds;
+  h->fds = gpr_malloc(nfds * sizeof(grpc_fd *));
+  h->del_count = 0;
+  h->del_capacity = 0;
+  h->dels = NULL;
+  for (i = 0; i < nfds; i++) {
+    h->fds[i] = fds[i];
+    GRPC_FD_REF(fds[i], "multipoller");
+  }
+}
+
+#endif /* !GPR_LINUX_MULTIPOLL_WITH_EPOLL */
+
+/*******************************************************************************
+ * pollset_multipoller_with_epoll_posix.c
+ */
+
+#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
+
+#include <errno.h>
+#include <poll.h>
+#include <string.h>
+#include <sys/epoll.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/block_annotate.h"
+
+static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st,
+                      grpc_pollset *read_notifier_pollset) {
+  /* only one set_ready can be active at once (but there may be a racing
+     notify_on) */
+  gpr_mu_lock(&fd->mu);
+  set_ready_locked(exec_ctx, fd, st);
+
+  /* A non-NULL read_notifier_pollset means that the fd is readable. */
+  if (read_notifier_pollset != NULL) {
+    /* Note: Since the fd might be a part of multiple pollsets, this might be
+     * called multiple times (for each time the fd becomes readable) and it is
+     * okay to set the fd's read-notifier pollset to anyone of these pollsets */
+    set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+  }
+
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_pollset *notifier_pollset) {
+  set_ready(exec_ctx, fd, &fd->read_closure, notifier_pollset);
+}
+
+static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  set_ready(exec_ctx, fd, &fd->write_closure, NULL);
+}
+
+struct epoll_fd_list {
+  int *epoll_fds;
+  size_t count;
+  size_t capacity;
+};
+
+static struct epoll_fd_list epoll_fd_global_list;
+static gpr_once init_epoll_fd_list_mu = GPR_ONCE_INIT;
+static gpr_mu epoll_fd_list_mu;
+
+static void init_mu(void) { gpr_mu_init(&epoll_fd_list_mu); }
+
+static void add_epoll_fd_to_global_list(int epoll_fd) {
+  gpr_once_init(&init_epoll_fd_list_mu, init_mu);
+
+  gpr_mu_lock(&epoll_fd_list_mu);
+  if (epoll_fd_global_list.count == epoll_fd_global_list.capacity) {
+    epoll_fd_global_list.capacity =
+        GPR_MAX((size_t)8, epoll_fd_global_list.capacity * 2);
+    epoll_fd_global_list.epoll_fds =
+        gpr_realloc(epoll_fd_global_list.epoll_fds,
+                    epoll_fd_global_list.capacity * sizeof(int));
+  }
+  epoll_fd_global_list.epoll_fds[epoll_fd_global_list.count++] = epoll_fd;
+  gpr_mu_unlock(&epoll_fd_list_mu);
+}
+
+static void remove_epoll_fd_from_global_list(int epoll_fd) {
+  gpr_mu_lock(&epoll_fd_list_mu);
+  GPR_ASSERT(epoll_fd_global_list.count > 0);
+  for (size_t i = 0; i < epoll_fd_global_list.count; i++) {
+    if (epoll_fd == epoll_fd_global_list.epoll_fds[i]) {
+      epoll_fd_global_list.epoll_fds[i] =
+          epoll_fd_global_list.epoll_fds[--(epoll_fd_global_list.count)];
+      break;
+    }
+  }
+  gpr_mu_unlock(&epoll_fd_list_mu);
+}
+
+static void remove_fd_from_all_epoll_sets(int fd) {
+  int err;
+  gpr_once_init(&init_epoll_fd_list_mu, init_mu);
+  gpr_mu_lock(&epoll_fd_list_mu);
+  if (epoll_fd_global_list.count == 0) {
+    gpr_mu_unlock(&epoll_fd_list_mu);
+    return;
+  }
+  for (size_t i = 0; i < epoll_fd_global_list.count; i++) {
+    err = epoll_ctl(epoll_fd_global_list.epoll_fds[i], EPOLL_CTL_DEL, fd, NULL);
+    if (err < 0 && errno != ENOENT) {
+      gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd,
+              strerror(errno));
+    }
+  }
+  gpr_mu_unlock(&epoll_fd_list_mu);
+}
+
+typedef struct {
+  grpc_pollset *pollset;
+  grpc_fd *fd;
+  grpc_closure closure;
+} delayed_add;
+
+typedef struct { int epoll_fd; } epoll_hdr;
+
+static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           grpc_fd *fd) {
+  epoll_hdr *h = pollset->data.ptr;
+  struct epoll_event ev;
+  int err;
+  grpc_fd_watcher watcher;
+
+  /* We pretend to be polling whilst adding an fd to keep the fd from being
+     closed during the add. This may result in a spurious wakeup being assigned
+     to this pollset whilst adding, but that should be benign. */
+  GPR_ASSERT(fd_begin_poll(fd, pollset, NULL, 0, 0, &watcher) == 0);
+  if (watcher.fd != NULL) {
+    ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET);
+    ev.data.ptr = fd;
+    err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev);
+    if (err < 0) {
+      /* FDs may be added to a pollset multiple times, so EEXIST is normal. */
+      if (errno != EEXIST) {
+        gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd,
+                strerror(errno));
+      }
+    }
+  }
+  fd_end_poll(exec_ctx, &watcher, 0, 0, NULL);
+}
+
+static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg,
+                                grpc_error *error) {
+  delayed_add *da = arg;
+
+  if (!fd_is_orphaned(da->fd)) {
+    finally_add_fd(exec_ctx, da->pollset, da->fd);
+  }
+
+  gpr_mu_lock(&da->pollset->mu);
+  da->pollset->in_flight_cbs--;
+  if (da->pollset->shutting_down) {
+    /* We don't care about this pollset anymore. */
+    if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) {
+      da->pollset->called_shutdown = 1;
+      grpc_exec_ctx_sched(exec_ctx, da->pollset->shutdown_done, GRPC_ERROR_NONE,
+                          NULL);
+    }
+  }
+  gpr_mu_unlock(&da->pollset->mu);
+
+  GRPC_FD_UNREF(da->fd, "delayed_add");
+
+  gpr_free(da);
+}
+
+static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
+                                                grpc_pollset *pollset,
+                                                grpc_fd *fd,
+                                                int and_unlock_pollset) {
+  if (and_unlock_pollset) {
+    gpr_mu_unlock(&pollset->mu);
+    finally_add_fd(exec_ctx, pollset, fd);
+  } else {
+    delayed_add *da = gpr_malloc(sizeof(*da));
+    da->pollset = pollset;
+    da->fd = fd;
+    GRPC_FD_REF(fd, "delayed_add");
+    grpc_closure_init(&da->closure, perform_delayed_add, da);
+    pollset->in_flight_cbs++;
+    grpc_exec_ctx_sched(exec_ctx, &da->closure, GRPC_ERROR_NONE, NULL);
+  }
+}
+
+/* TODO(klempner): We probably want to turn this down a bit */
+#define GRPC_EPOLL_MAX_EVENTS 1000
+
+static grpc_error *multipoll_with_epoll_pollset_maybe_work_and_unlock(
+    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
+    gpr_timespec deadline, gpr_timespec now) {
+  struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
+  int ep_rv;
+  int poll_rv;
+  epoll_hdr *h = pollset->data.ptr;
+  int timeout_ms;
+  struct pollfd pfds[2];
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  /* If you want to ignore epoll's ability to sanely handle parallel pollers,
+   * for a more apples-to-apples performance comparison with poll, add a
+   * if (pollset->counter != 0) { return 0; }
+   * here.
+   */
+
+  gpr_mu_unlock(&pollset->mu);
+
+  timeout_ms = poll_deadline_to_millis_timeout(deadline, now);
+
+  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
+  pfds[0].events = POLLIN;
+  pfds[0].revents = 0;
+  pfds[1].fd = h->epoll_fd;
+  pfds[1].events = POLLIN;
+  pfds[1].revents = 0;
+
+  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
+     even going into the blocking annotation if possible */
+  GPR_TIMER_BEGIN("poll", 0);
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
+  poll_rv = grpc_poll_function(pfds, 2, timeout_ms);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
+  GPR_TIMER_END("poll", 0);
+
+  if (poll_rv < 0) {
+    if (errno != EINTR) {
+      work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
+    }
+  } else if (poll_rv == 0) {
+    /* do nothing */
+  } else {
+    if (pfds[0].revents) {
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd));
+    }
+    if (pfds[1].revents) {
+      do {
+        /* The following epoll_wait never blocks; it has a timeout of 0 */
+        ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
+        if (ep_rv < 0) {
+          if (errno != EINTR) {
+            work_combine_error(&error, GRPC_OS_ERROR(errno, "epoll_wait"));
+          }
+        } else {
+          int i;
+          for (i = 0; i < ep_rv; ++i) {
+            grpc_fd *fd = ep_ev[i].data.ptr;
+            /* TODO(klempner): We might want to consider making err and pri
+             * separate events */
+            int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
+            int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
+            int write_ev = ep_ev[i].events & EPOLLOUT;
+            if (fd == NULL) {
+              work_combine_error(&error, grpc_wakeup_fd_consume_wakeup(
+                                             &grpc_global_wakeup_fd));
+            } else {
+              if (read_ev || cancel) {
+                fd_become_readable(exec_ctx, fd, pollset);
+              }
+              if (write_ev || cancel) {
+                fd_become_writable(exec_ctx, fd);
+              }
+            }
+          }
+        }
+      } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
+    }
+  }
+  return error;
+}
+
+static void multipoll_with_epoll_pollset_finish_shutdown(
+    grpc_pollset *pollset) {}
+
+static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) {
+  epoll_hdr *h = pollset->data.ptr;
+  close(h->epoll_fd);
+  remove_epoll_fd_from_global_list(h->epoll_fd);
+  gpr_free(h);
+}
+
+static const grpc_pollset_vtable multipoll_with_epoll_pollset = {
+    multipoll_with_epoll_pollset_add_fd,
+    multipoll_with_epoll_pollset_maybe_work_and_unlock,
+    multipoll_with_epoll_pollset_finish_shutdown,
+    multipoll_with_epoll_pollset_destroy};
+
+static void epoll_become_multipoller(grpc_exec_ctx *exec_ctx,
+                                     grpc_pollset *pollset, grpc_fd **fds,
+                                     size_t nfds) {
+  size_t i;
+  epoll_hdr *h = gpr_malloc(sizeof(epoll_hdr));
+  struct epoll_event ev;
+  int err;
+
+  pollset->vtable = &multipoll_with_epoll_pollset;
+  pollset->data.ptr = h;
+  h->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+  if (h->epoll_fd < 0) {
+    /* TODO(klempner): Fall back to poll here, especially on ENOSYS */
+    gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno));
+    abort();
+  }
+  add_epoll_fd_to_global_list(h->epoll_fd);
+
+  ev.events = (uint32_t)(EPOLLIN | EPOLLET);
+  ev.data.ptr = NULL;
+  err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD,
+                  GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), &ev);
+  if (err < 0) {
+    gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s",
+            GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd),
+            strerror(errno));
+  }
+
+  for (i = 0; i < nfds; i++) {
+    multipoll_with_epoll_pollset_add_fd(exec_ctx, pollset, fds[i], 0);
+  }
+}
+
+#else /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
+
+static void remove_fd_from_all_epoll_sets(int fd) {}
+
+#endif /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
+
+/*******************************************************************************
+ * pollset_set_posix.c
+ */
+
+static grpc_pollset_set *pollset_set_create(void) {
+  grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set));
+  memset(pollset_set, 0, sizeof(*pollset_set));
+  gpr_mu_init(&pollset_set->mu);
+  return pollset_set;
+}
+
+static void pollset_set_destroy(grpc_pollset_set *pollset_set) {
+  size_t i;
+  gpr_mu_destroy(&pollset_set->mu);
+  for (i = 0; i < pollset_set->fd_count; i++) {
+    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+  }
+  gpr_free(pollset_set->pollsets);
+  gpr_free(pollset_set->pollset_sets);
+  gpr_free(pollset_set->fds);
+  gpr_free(pollset_set);
+}
+
+static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset_set *pollset_set,
+                                    grpc_pollset *pollset) {
+  size_t i, j;
+  gpr_mu_lock(&pollset_set->mu);
+  if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
+    pollset_set->pollset_capacity =
+        GPR_MAX(8, 2 * pollset_set->pollset_capacity);
+    pollset_set->pollsets =
+        gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity *
+                                               sizeof(*pollset_set->pollsets));
+  }
+  pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
+  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
+    if (fd_is_orphaned(pollset_set->fds[i])) {
+      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+    } else {
+      pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
+      pollset_set->fds[j++] = pollset_set->fds[i];
+    }
+  }
+  pollset_set->fd_count = j;
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset_set *pollset_set,
+                                    grpc_pollset *pollset) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  for (i = 0; i < pollset_set->pollset_count; i++) {
+    if (pollset_set->pollsets[i] == pollset) {
+      pollset_set->pollset_count--;
+      GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i],
+               pollset_set->pollsets[pollset_set->pollset_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset_set *bag,
+                                        grpc_pollset_set *item) {
+  size_t i, j;
+  gpr_mu_lock(&bag->mu);
+  if (bag->pollset_set_count == bag->pollset_set_capacity) {
+    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
+    bag->pollset_sets =
+        gpr_realloc(bag->pollset_sets,
+                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
+  }
+  bag->pollset_sets[bag->pollset_set_count++] = item;
+  for (i = 0, j = 0; i < bag->fd_count; i++) {
+    if (fd_is_orphaned(bag->fds[i])) {
+      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
+    } else {
+      pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
+      bag->fds[j++] = bag->fds[i];
+    }
+  }
+  bag->fd_count = j;
+  gpr_mu_unlock(&bag->mu);
+}
+
+static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset_set *bag,
+                                        grpc_pollset_set *item) {
+  size_t i;
+  gpr_mu_lock(&bag->mu);
+  for (i = 0; i < bag->pollset_set_count; i++) {
+    if (bag->pollset_sets[i] == item) {
+      bag->pollset_set_count--;
+      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
+               bag->pollset_sets[bag->pollset_set_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&bag->mu);
+}
+
+static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  if (pollset_set->fd_count == pollset_set->fd_capacity) {
+    pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity);
+    pollset_set->fds = gpr_realloc(
+        pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds));
+  }
+  GRPC_FD_REF(fd, "pollset_set");
+  pollset_set->fds[pollset_set->fd_count++] = fd;
+  for (i = 0; i < pollset_set->pollset_count; i++) {
+    pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
+  }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  for (i = 0; i < pollset_set->fd_count; i++) {
+    if (pollset_set->fds[i] == fd) {
+      pollset_set->fd_count--;
+      GPR_SWAP(grpc_fd *, pollset_set->fds[i],
+               pollset_set->fds[pollset_set->fd_count]);
+      GRPC_FD_UNREF(fd, "pollset_set");
+      break;
+    }
+  }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+/*******************************************************************************
+ * event engine binding
+ */
+
+static void shutdown_engine(void) {
+  fd_global_shutdown();
+  pollset_global_shutdown();
+}
+
+static const grpc_event_engine_vtable vtable = {
+    .pollset_size = sizeof(grpc_pollset),
+
+    .fd_create = fd_create,
+    .fd_wrapped_fd = fd_wrapped_fd,
+    .fd_orphan = fd_orphan,
+    .fd_shutdown = fd_shutdown,
+    .fd_is_shutdown = fd_is_shutdown,
+    .fd_notify_on_read = fd_notify_on_read,
+    .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
+
+    .pollset_init = pollset_init,
+    .pollset_shutdown = pollset_shutdown,
+    .pollset_reset = pollset_reset,
+    .pollset_destroy = pollset_destroy,
+    .pollset_work = pollset_work,
+    .pollset_kick = pollset_kick,
+    .pollset_add_fd = pollset_add_fd,
+
+    .pollset_set_create = pollset_set_create,
+    .pollset_set_destroy = pollset_set_destroy,
+    .pollset_set_add_pollset = pollset_set_add_pollset,
+    .pollset_set_del_pollset = pollset_set_del_pollset,
+    .pollset_set_add_pollset_set = pollset_set_add_pollset_set,
+    .pollset_set_del_pollset_set = pollset_set_del_pollset_set,
+    .pollset_set_add_fd = pollset_set_add_fd,
+    .pollset_set_del_fd = pollset_set_del_fd,
+
+    .kick_poller = kick_poller,
+
+    .shutdown_engine = shutdown_engine,
+};
+
+const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void) {
+#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
+  platform_become_multipoller = epoll_become_multipoller;
+#else
+  platform_become_multipoller = poll_become_multipoller;
+#endif
+  fd_global_init();
+  pollset_global_init();
+  return &vtable;
+}
+
+#endif
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h b/src/core/lib/iomgr/ev_poll_and_epoll_posix.h
new file mode 100644
index 0000000000..06d6dbf29d
--- /dev/null
+++ b/src/core/lib/iomgr/ev_poll_and_epoll_posix.h
@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H
+
+#include "src/core/lib/iomgr/ev_posix.h"
+
+const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void);
+
+#endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H */
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index 52efb0d6b2..45c0a5e954 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -59,8 +59,6 @@
  * FD declarations
  */
 
-grpc_wakeup_fd grpc_global_wakeup_fd;
-
 typedef struct grpc_fd_watcher {
   struct grpc_fd_watcher *next;
   struct grpc_fd_watcher *prev;
@@ -373,7 +371,7 @@ static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   if (!fd->released) {
     close(fd->fd);
   }
-  grpc_exec_ctx_push(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL);
 }
 
 static int fd_wrapped_fd(grpc_fd *fd) {
@@ -432,14 +430,17 @@ static grpc_error *fd_shutdown_error(bool shutdown) {
 
 static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure **st, grpc_closure *closure) {
-  if (*st == CLOSURE_NOT_READY) {
+  if (fd->shutdown) {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"),
+                        NULL);
+  } else if (*st == CLOSURE_NOT_READY) {
     /* not ready ==> switch to a waiting state by setting the closure */
     *st = closure;
   } else if (*st == CLOSURE_READY) {
     /* already ready ==> queue the closure to run immediately */
     *st = CLOSURE_NOT_READY;
-    grpc_exec_ctx_push(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
+                        NULL);
     maybe_wake_one_watcher_locked(fd);
   } else {
     /* upcallptr was set to a different closure.  This is an error! */
@@ -462,7 +463,7 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
     return 0;
   } else {
     /* waiting ==> queue closure */
-    grpc_exec_ctx_push(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
+    grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
     *st = CLOSURE_NOT_READY;
     return 1;
   }
@@ -475,11 +476,22 @@ static void set_read_notifier_pollset_locked(
 
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_lock(&fd->mu);
-  GPR_ASSERT(!fd->shutdown);
-  fd->shutdown = 1;
-  set_ready_locked(exec_ctx, fd, &fd->read_closure);
-  set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  /* only shutdown once */
+  if (!fd->shutdown) {
+    fd->shutdown = 1;
+    /* signal read/write closed to OS so that future operations fail */
+    shutdown(fd->fd, SHUT_RDWR);
+    set_ready_locked(exec_ctx, fd, &fd->read_closure);
+    set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  }
+  gpr_mu_unlock(&fd->mu);
+}
+
+static bool fd_is_shutdown(grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  bool r = fd->shutdown;
   gpr_mu_unlock(&fd->mu);
+  return r;
 }
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
@@ -717,6 +729,7 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p,
   }
 
   GPR_TIMER_END("pollset_kick_ext", 0);
+  GRPC_LOG_IF_ERROR("pollset_kick_ext", GRPC_ERROR_REF(error));
   return error;
 }
 
@@ -811,7 +824,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
     GRPC_FD_UNREF(pollset->fds[i], "multipoller");
   }
   pollset->fd_count = 0;
-  grpc_exec_ctx_push(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
 }
 
 static void work_combine_error(grpc_error **composite, grpc_error *error) {
@@ -845,6 +858,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
     error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
     if (error != GRPC_ERROR_NONE) {
+      GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error));
       return error;
     }
   }
@@ -1024,6 +1038,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   }
   *worker_hdl = NULL;
   GPR_TIMER_END("pollset_work", 0);
+  GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error));
   return error;
 }
 
@@ -1215,6 +1230,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_wrapped_fd = fd_wrapped_fd,
     .fd_orphan = fd_orphan,
     .fd_shutdown = fd_shutdown,
+    .fd_is_shutdown = fd_is_shutdown,
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
     .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
@@ -1242,7 +1258,9 @@ static const grpc_event_engine_vtable vtable = {
 };
 
 const grpc_event_engine_vtable *grpc_init_poll_posix(void) {
-  pollset_global_init();
+  if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) {
+    return NULL;
+  }
   return &vtable;
 }
 
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index ad8eaab454..4c360f13a3 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -44,6 +44,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
 #include "src/core/lib/iomgr/ev_poll_posix.h"
 #include "src/core/lib/support/env.h"
 
@@ -61,7 +62,7 @@ typedef struct {
 } event_engine_factory;
 
 static const event_engine_factory g_factories[] = {
-    {"poll", grpc_init_poll_posix},
+    {"poll", grpc_init_poll_posix}, {"legacy", grpc_init_poll_and_epoll_posix},
 };
 
 static void add(const char *beg, const char *end, char ***ss, size_t *ns) {
@@ -152,6 +153,10 @@ void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   g_event_engine->fd_shutdown(exec_ctx, fd);
 }
 
+bool grpc_fd_is_shutdown(grpc_fd *fd) {
+  return g_event_engine->fd_is_shutdown(fd);
+}
+
 void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                             grpc_closure *closure) {
   g_event_engine->fd_notify_on_read(exec_ctx, fd, closure);
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 3058f22c91..87e9bc4d46 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -55,6 +55,7 @@ typedef struct grpc_event_engine_vtable {
                             grpc_closure *closure);
   void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
+  bool (*fd_is_shutdown)(grpc_fd *fd);
   grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
                                                 grpc_fd *fd);
 
@@ -116,7 +117,10 @@ int grpc_fd_wrapped_fd(grpc_fd *fd);
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
                     int *release_fd, const char *reason);
 
-/* Cause any current callbacks to error out with GRPC_CALLBACK_CANCELLED. */
+/* Has grpc_fd_shutdown been called on an fd? */
+bool grpc_fd_is_shutdown(grpc_fd *fd);
+
+/* Cause any current and future callbacks to fail. */
 void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd);
 
 /* Register read interest, causing read_cb to be called once when fd becomes
diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c
index fcf39ffd54..c44aafcddf 100644
--- a/src/core/lib/iomgr/exec_ctx.c
+++ b/src/core/lib/iomgr/exec_ctx.c
@@ -82,9 +82,9 @@ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {
   grpc_exec_ctx_flush(exec_ctx);
 }
 
-void grpc_exec_ctx_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                        grpc_error *error,
-                        grpc_workqueue *offload_target_or_null) {
+void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                         grpc_error *error,
+                         grpc_workqueue *offload_target_or_null) {
   GPR_ASSERT(offload_target_or_null == NULL);
   grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
 }
diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h
index 617f48f1b8..38f27d9b13 100644
--- a/src/core/lib/iomgr/exec_ctx.h
+++ b/src/core/lib/iomgr/exec_ctx.h
@@ -94,9 +94,9 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx);
  *  the instance is destroyed, or work may be lost. */
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx);
 /** Add a closure to be executed at the next flush/finish point */
-void grpc_exec_ctx_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                        grpc_error *error,
-                        grpc_workqueue *offload_target_or_null);
+void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                         grpc_error *error,
+                         grpc_workqueue *offload_target_or_null);
 /** Returns true if we'd like to leave this execution context as soon as
     possible: useful for deciding whether to do something more or not depending
     on outside context */
diff --git a/src/core/lib/iomgr/iocp_windows.c b/src/core/lib/iomgr/iocp_windows.c
index 1faa8eeedf..2532e52e48 100644
--- a/src/core/lib/iomgr/iocp_windows.c
+++ b/src/core/lib/iomgr/iocp_windows.c
@@ -39,7 +39,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/thd.h>
 
 #include "src/core/lib/iomgr/iocp_windows.h"
diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c
index 60cef8ba77..89292a153e 100644
--- a/src/core/lib/iomgr/iomgr.c
+++ b/src/core/lib/iomgr/iomgr.c
@@ -96,7 +96,8 @@ void grpc_iomgr_shutdown(void) {
             gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), last_warning_time),
             gpr_time_from_seconds(1, GPR_TIMESPAN)) >= 0) {
       if (g_root_object.next != &g_root_object) {
-        gpr_log(GPR_DEBUG, "Waiting for %d iomgr objects to be destroyed",
+        gpr_log(GPR_DEBUG,
+                "Waiting for %" PRIuPTR " iomgr objects to be destroyed",
                 count_objects());
       }
       last_warning_time = gpr_now(GPR_CLOCK_REALTIME);
@@ -114,9 +115,9 @@ void grpc_iomgr_shutdown(void) {
       if (gpr_cv_wait(&g_rcv, &g_mu, short_deadline)) {
         if (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), shutdown_deadline) > 0) {
           if (g_root_object.next != &g_root_object) {
-            gpr_log(GPR_DEBUG,
-                    "Failed to free %d iomgr objects before shutdown deadline: "
-                    "memory leaks are likely",
+            gpr_log(GPR_DEBUG, "Failed to free %" PRIuPTR
+                               " iomgr objects before shutdown deadline: "
+                               "memory leaks are likely",
                     count_objects());
             dump_objects("LEAKED");
             if (grpc_iomgr_abort_on_leaks()) {
diff --git a/src/core/lib/iomgr/iomgr_windows.c b/src/core/lib/iomgr/iomgr_windows.c
index 398517fc75..7653f6e635 100644
--- a/src/core/lib/iomgr/iomgr_windows.c
+++ b/src/core/lib/iomgr/iomgr_windows.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 
 #include <grpc/support/log.h>
 
diff --git a/src/core/lib/iomgr/polling_entity.c b/src/core/lib/iomgr/polling_entity.c
new file mode 100644
index 0000000000..d1686aa12f
--- /dev/null
+++ b/src/core/lib/iomgr/polling_entity.c
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/iomgr/polling_entity.h"
+
+grpc_polling_entity grpc_polling_entity_create_from_pollset_set(
+    grpc_pollset_set *pollset_set) {
+  grpc_polling_entity pollent;
+  pollent.pollent.pollset_set = pollset_set;
+  pollent.tag = POPS_POLLSET_SET;
+  return pollent;
+}
+
+grpc_polling_entity grpc_polling_entity_create_from_pollset(
+    grpc_pollset *pollset) {
+  grpc_polling_entity pollent;
+  pollent.pollent.pollset = pollset;
+  pollent.tag = POPS_POLLSET;
+  return pollent;
+}
+
+grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) {
+  if (pollent->tag == POPS_POLLSET) {
+    return pollent->pollent.pollset;
+  }
+  return NULL;
+}
+
+grpc_pollset_set *grpc_polling_entity_pollset_set(
+    grpc_polling_entity *pollent) {
+  if (pollent->tag == POPS_POLLSET_SET) {
+    return pollent->pollent.pollset_set;
+  }
+  return NULL;
+}
+
+bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) {
+  return pollent->tag == POPS_NONE;
+}
+
+void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
+                                            grpc_polling_entity *pollent,
+                                            grpc_pollset_set *pss_dst) {
+  if (pollent->tag == POPS_POLLSET) {
+    GPR_ASSERT(pollent->pollent.pollset != NULL);
+    grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset);
+  } else if (pollent->tag == POPS_POLLSET_SET) {
+    GPR_ASSERT(pollent->pollent.pollset_set != NULL);
+    grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst,
+                                     pollent->pollent.pollset_set);
+  } else {
+    gpr_log(GPR_ERROR, "Invalid grpc_polling_entity tag '%d'", pollent->tag);
+    abort();
+  }
+}
+
+void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx,
+                                              grpc_polling_entity *pollent,
+                                              grpc_pollset_set *pss_dst) {
+  if (pollent->tag == POPS_POLLSET) {
+    GPR_ASSERT(pollent->pollent.pollset != NULL);
+    grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset);
+  } else if (pollent->tag == POPS_POLLSET_SET) {
+    GPR_ASSERT(pollent->pollent.pollset_set != NULL);
+    grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst,
+                                     pollent->pollent.pollset_set);
+  } else {
+    gpr_log(GPR_ERROR, "Invalid grpc_polling_entity tag '%d'", pollent->tag);
+    abort();
+  }
+}
diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h
new file mode 100644
index 0000000000..e81531053c
--- /dev/null
+++ b/src/core/lib/iomgr/polling_entity.h
@@ -0,0 +1,81 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H
+#define GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H
+
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+
+/* A grpc_polling_entity is a pollset-or-pollset_set container. It allows
+ * functions that
+ * accept a pollset XOR a pollset_set to do so through an abstract interface.
+ * No ownership is taken. */
+
+typedef struct grpc_polling_entity {
+  union {
+    grpc_pollset *pollset;
+    grpc_pollset_set *pollset_set;
+  } pollent;
+  enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag;
+} grpc_polling_entity;
+
+grpc_polling_entity grpc_polling_entity_create_from_pollset_set(
+    grpc_pollset_set *pollset_set);
+grpc_polling_entity grpc_polling_entity_create_from_pollset(
+    grpc_pollset *pollset);
+
+/** If \a pollent contains a pollset, return it. Otherwise, return NULL */
+grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent);
+
+/** If \a pollent contains a pollset_set, return it. Otherwise, return NULL */
+grpc_pollset_set *grpc_polling_entity_pollset_set(grpc_polling_entity *pollent);
+
+bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent);
+
+/** Add the pollset or pollset_set in \a pollent to the destination pollset_set
+ * \a
+ * pss_dst */
+void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
+                                            grpc_polling_entity *pollent,
+                                            grpc_pollset_set *pss_dst);
+
+/** Delete the pollset or pollset_set in \a pollent from the destination
+ * pollset_set \a
+ * pss_dst */
+void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx,
+                                              grpc_polling_entity *pollent,
+                                              grpc_pollset_set *pss_dst);
+/* pollset_set specific */
+
+#endif /* GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H */
diff --git a/src/core/lib/iomgr/pollset_set_windows.c b/src/core/lib/iomgr/pollset_set_windows.c
index 89f60b92fb..a35a9766fc 100644
--- a/src/core/lib/iomgr/pollset_set_windows.c
+++ b/src/core/lib/iomgr/pollset_set_windows.c
@@ -32,12 +32,15 @@
  */
 
 #include <grpc/support/port_platform.h>
+#include <stdint.h>
 
 #ifdef GPR_WINSOCK_SOCKET
 
 #include "src/core/lib/iomgr/pollset_set_windows.h"
 
-grpc_pollset_set* grpc_pollset_set_create(void) { return NULL; }
+grpc_pollset_set* grpc_pollset_set_create(void) {
+  return (grpc_pollset_set*)((intptr_t)0xdeafbeef);
+}
 
 void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {}
 
diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c
index 5882d8d71d..626dd784b3 100644
--- a/src/core/lib/iomgr/pollset_windows.c
+++ b/src/core/lib/iomgr/pollset_windows.c
@@ -109,7 +109,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   pollset->shutting_down = 1;
   grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
   if (!pollset->is_iocp_worker) {
-    grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
   } else {
     pollset->on_shutdown = closure;
   }
@@ -167,8 +167,8 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
 
       if (pollset->shutting_down && pollset->on_shutdown != NULL) {
-        grpc_exec_ctx_push(exec_ctx, pollset->on_shutdown, GRPC_ERROR_NONE,
-                           NULL);
+        grpc_exec_ctx_sched(exec_ctx, pollset->on_shutdown, GRPC_ERROR_NONE,
+                            NULL);
         pollset->on_shutdown = NULL;
       }
       goto done;
diff --git a/src/core/lib/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c
index 9a320dc952..4e9f978584 100644
--- a/src/core/lib/iomgr/resolve_address_posix.c
+++ b/src/core/lib/iomgr/resolve_address_posix.c
@@ -163,7 +163,7 @@ typedef struct {
 static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp,
                               grpc_error *error) {
   request *r = rp;
-  grpc_exec_ctx_push(
+  grpc_exec_ctx_sched(
       exec_ctx, r->on_done,
       grpc_blocking_resolve_address(r->name, r->default_port, r->addrs_out),
       NULL);
diff --git a/src/core/lib/iomgr/resolve_address_windows.c b/src/core/lib/iomgr/resolve_address_windows.c
index 22a9feb72d..2af8af82dc 100644
--- a/src/core/lib/iomgr/resolve_address_windows.c
+++ b/src/core/lib/iomgr/resolve_address_windows.c
@@ -43,7 +43,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
@@ -154,7 +154,7 @@ static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp,
   } else {
     GRPC_ERROR_REF(error);
   }
-  grpc_exec_ctx_push(exec_ctx, r->on_done, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, r->on_done, error, NULL);
   gpr_free(r->name);
   gpr_free(r->default_port);
   gpr_free(r);
diff --git a/src/core/lib/iomgr/sockaddr.h b/src/core/lib/iomgr/sockaddr.h
index 891a2f094f..5563d0b8a6 100644
--- a/src/core/lib/iomgr/sockaddr.h
+++ b/src/core/lib/iomgr/sockaddr.h
@@ -36,8 +36,8 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#ifdef GPR_WINDOWS
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 #endif
 
 #ifdef GPR_POSIX_SOCKETADDR
diff --git a/src/core/lib/iomgr/sockaddr_win32.h b/src/core/lib/iomgr/sockaddr_windows.h
index 02aeae7619..971db5b32b 100644
--- a/src/core/lib/iomgr/sockaddr_win32.h
+++ b/src/core/lib/iomgr/sockaddr_windows.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H
-#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H
 
 #include <winsock2.h>
 #include <ws2tcpip.h>
@@ -40,4 +40,4 @@
 // must be included after the above
 #include <mswsock.h>
 
-#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H */
diff --git a/src/core/lib/iomgr/socket_utils_common_posix.c b/src/core/lib/iomgr/socket_utils_common_posix.c
index bd8ef00c85..d2f6261e2a 100644
--- a/src/core/lib/iomgr/socket_utils_common_posix.c
+++ b/src/core/lib/iomgr/socket_utils_common_posix.c
@@ -49,6 +49,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
@@ -116,6 +117,20 @@ grpc_error *grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd) {
   return GRPC_ERROR_NONE;
 }
 
+grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes) {
+  return 0 == setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffer_size_bytes,
+                         sizeof(buffer_size_bytes))
+             ? GRPC_ERROR_NONE
+             : GRPC_OS_ERROR(errno, "setsockopt(SO_SNDBUF)");
+}
+
+grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes) {
+  return 0 == setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &buffer_size_bytes,
+                         sizeof(buffer_size_bytes))
+             ? GRPC_ERROR_NONE
+             : GRPC_OS_ERROR(errno, "setsockopt(SO_RCVBUF)");
+}
+
 /* set a socket to close on exec */
 grpc_error *grpc_set_socket_cloexec(int fd, int close_on_exec) {
   int oldflags = fcntl(fd, F_GETFD, 0);
@@ -238,6 +253,16 @@ static int set_socket_dualstack(int fd) {
   }
 }
 
+static grpc_error *error_for_fd(int fd, const struct sockaddr *addr) {
+  if (fd >= 0) return GRPC_ERROR_NONE;
+  char *addr_str;
+  grpc_sockaddr_to_string(&addr_str, addr, 0);
+  grpc_error *err = grpc_error_set_str(GRPC_OS_ERROR(errno, "socket"),
+                                       GRPC_ERROR_STR_TARGET_ADDRESS, addr_str);
+  gpr_free(addr_str);
+  return err;
+}
+
 grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
                                          int protocol,
                                          grpc_dualstack_mode *dsmode,
@@ -258,7 +283,7 @@ grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
     /* If this isn't an IPv4 address, then return whatever we've got. */
     if (!grpc_sockaddr_is_v4mapped(addr, NULL)) {
       *dsmode = GRPC_DSMODE_IPV6;
-      return GRPC_ERROR_NONE;
+      return error_for_fd(*newfd, addr);
     }
     /* Fall back to AF_INET. */
     if (*newfd >= 0) {
@@ -268,10 +293,7 @@ grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
   }
   *dsmode = family == AF_INET ? GRPC_DSMODE_IPV4 : GRPC_DSMODE_NONE;
   *newfd = socket(family, type, protocol);
-  if (*newfd == -1) {
-    return GRPC_OS_ERROR(errno, "socket");
-  }
-  return GRPC_ERROR_NONE;
+  return error_for_fd(*newfd, addr);
 }
 
 #endif
diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h
index 607fbc7a79..7bcc2219ae 100644
--- a/src/core/lib/iomgr/socket_utils_posix.h
+++ b/src/core/lib/iomgr/socket_utils_posix.h
@@ -69,20 +69,23 @@ grpc_error *grpc_set_socket_reuse_port(int fd, int reuse);
 int grpc_ipv6_loopback_available(void);
 
 /* Tries to set SO_NOSIGPIPE if available on this platform.
-   Returns 1 on success, 0 on failure.
    If SO_NO_SIGPIPE is not available, returns 1. */
 grpc_error *grpc_set_socket_no_sigpipe_if_possible(int fd);
 
 /* Tries to set IP_PKTINFO if available on this platform.
-   Returns 1 on success, 0 on failure.
    If IP_PKTINFO is not available, returns 1. */
 grpc_error *grpc_set_socket_ip_pktinfo_if_possible(int fd);
 
 /* Tries to set IPV6_RECVPKTINFO if available on this platform.
-   Returns 1 on success, 0 on failure.
    If IPV6_RECVPKTINFO is not available, returns 1. */
 grpc_error *grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd);
 
+/* Tries to set the socket's send buffer to given size. */
+grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes);
+
+/* Tries to set the socket's receive buffer to given size. */
+grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes);
+
 /* An enum to keep track of IPv4/IPv6 socket modes.
 
    Currently, this information is only used when a socket is first created, but
diff --git a/src/core/lib/iomgr/socket_windows.c b/src/core/lib/iomgr/socket_windows.c
index 8919452751..d7d5f6f157 100644
--- a/src/core/lib/iomgr/socket_windows.c
+++ b/src/core/lib/iomgr/socket_windows.c
@@ -42,7 +42,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/iomgr/iocp_windows.h"
@@ -123,7 +123,7 @@ static void socket_notify_on_iocp(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&socket->state_mu);
   if (info->has_pending_iocp) {
     info->has_pending_iocp = 0;
-    grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
   } else {
     info->closure = closure;
   }
@@ -146,7 +146,7 @@ void grpc_socket_become_ready(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket,
   GPR_ASSERT(!info->has_pending_iocp);
   gpr_mu_lock(&socket->state_mu);
   if (info->closure) {
-    grpc_exec_ctx_push(exec_ctx, info->closure, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, info->closure, GRPC_ERROR_NONE, NULL);
     info->closure = NULL;
   } else {
     info->has_pending_iocp = 1;
diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index 9c4ca1617f..80c7a3f128 100644
--- a/src/core/lib/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -221,7 +221,7 @@ finish:
     gpr_free(ac->addr_str);
     gpr_free(ac);
   }
-  grpc_exec_ctx_push(exec_ctx, closure, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
 }
 
 static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
@@ -250,7 +250,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
 
   error = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd);
   if (error != GRPC_ERROR_NONE) {
-    grpc_exec_ctx_push(exec_ctx, closure, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
     return;
   }
   if (dsmode == GRPC_DSMODE_IPV4) {
@@ -260,7 +260,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
     addr_len = sizeof(addr4_copy);
   }
   if ((error = prepare_socket(addr, fd)) != GRPC_ERROR_NONE) {
-    grpc_exec_ctx_push(exec_ctx, closure, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
     return;
   }
 
@@ -276,14 +276,14 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
 
   if (err >= 0) {
     *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str);
-    grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
     goto done;
   }
 
   if (errno != EWOULDBLOCK && errno != EINPROGRESS) {
     grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error");
-    grpc_exec_ctx_push(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"),
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"),
+                        NULL);
     goto done;
   }
 
diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c
index dbf116422f..562cb9c6bf 100644
--- a/src/core/lib/iomgr/tcp_client_windows.c
+++ b/src/core/lib/iomgr/tcp_client_windows.c
@@ -35,11 +35,11 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/useful.h>
 
@@ -121,7 +121,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   async_connect_unlock_and_cleanup(ac, socket);
   /* If the connection was aborted, the callback was already called when
      the deadline was met. */
-  grpc_exec_ctx_push(exec_ctx, on_done, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, on_done, error, NULL);
 }
 
 /* Tries to issue one async connection, then schedules both an IOCP
@@ -225,7 +225,7 @@ failure:
   } else if (sock != INVALID_SOCKET) {
     closesocket(sock);
   }
-  grpc_exec_ctx_push(exec_ctx, on_done, final_error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, on_done, final_error, NULL);
 }
 
 #endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index 48a19bed84..017f52c367 100644
--- a/src/core/lib/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -160,7 +160,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
                          grpc_error *error) {
   grpc_closure *cb = tcp->read_cb;
 
-  if (grpc_tcp_trace) {
+  if (false && grpc_tcp_trace) {
     size_t i;
     const char *str = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "read: error=%s", str);
@@ -175,7 +175,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
 
   tcp->read_cb = NULL;
   tcp->incoming_buffer = NULL;
-  grpc_exec_ctx_push(exec_ctx, cb, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
 }
 
 #define MAX_READ_IOVEC 4
@@ -277,7 +277,7 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     tcp->finished_edge = false;
     grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure);
   } else {
-    grpc_exec_ctx_push(exec_ctx, &tcp->read_closure, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &tcp->read_closure, GRPC_ERROR_NONE, NULL);
   }
 }
 
@@ -394,7 +394,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   grpc_tcp *tcp = (grpc_tcp *)ep;
   grpc_error *error = GRPC_ERROR_NONE;
 
-  if (grpc_tcp_trace) {
+  if (false && grpc_tcp_trace) {
     size_t i;
 
     for (i = 0; i < buf->count; i++) {
@@ -410,7 +410,10 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
   if (buf->length == 0) {
     GPR_TIMER_END("tcp_write", 0);
-    grpc_exec_ctx_push(exec_ctx, cb, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb, grpc_fd_is_shutdown(tcp->em_fd)
+                                          ? GRPC_ERROR_CREATE("EOF")
+                                          : GRPC_ERROR_NONE,
+                        NULL);
     return;
   }
   tcp->outgoing_buffer = buf;
@@ -422,7 +425,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     tcp->write_cb = cb;
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
-    grpc_exec_ctx_push(exec_ctx, cb, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
   }
 
   GPR_TIMER_END("tcp_write", 0);
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index aac3beb8bd..a1a463550a 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -132,6 +132,9 @@ struct grpc_tcp_server {
   grpc_pollset **pollsets;
   /* number of pollsets in the pollsets array */
   size_t pollset_count;
+
+  /* next pollset to assign a channel to */
+  size_t next_pollset_to_assign;
 };
 
 static gpr_once check_init = GPR_ONCE_INIT;
@@ -178,13 +181,14 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
   s->head = NULL;
   s->tail = NULL;
   s->nports = 0;
+  s->next_pollset_to_assign = 0;
   *server = s;
   return GRPC_ERROR_NONE;
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (s->shutdown_complete != NULL) {
-    grpc_exec_ctx_push(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }
 
   gpr_mu_destroy(&s->mu);
@@ -364,7 +368,9 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
     goto error;
   }
 
-  read_notifier_pollset = grpc_fd_get_read_notifier_pollset(exec_ctx, sp->emfd);
+  read_notifier_pollset =
+      sp->server->pollsets[(sp->server->next_pollset_to_assign++) %
+                           sp->server->pollset_count];
 
   /* loop until accept4 returns EAGAIN, and then re-arm notification */
   for (;;) {
@@ -588,9 +594,9 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
       if (port == 0 && sp != NULL) {
         grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port);
       }
-      addr = (struct sockaddr *)&wild4;
-      addr_len = sizeof(wild4);
     }
+    addr = (struct sockaddr *)&wild4;
+    addr_len = sizeof(wild4);
   }
 
   errs[1] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd);
diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 481111c9f9..2a51671ec7 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -41,7 +41,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
@@ -121,7 +121,7 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (s->shutdown_complete != NULL) {
-    grpc_exec_ctx_push(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }
 
   /* Now that the accepts have been aborted, we can destroy the sockets.
diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c
index 88c9354f6e..b2af8030aa 100644
--- a/src/core/lib/iomgr/tcp_windows.c
+++ b/src/core/lib/iomgr/tcp_windows.c
@@ -37,11 +37,11 @@
 
 #include <limits.h>
 
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
@@ -183,7 +183,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) {
 
   tcp->read_cb = NULL;
   TCP_UNREF(tcp, "read");
-  grpc_exec_ctx_push(exec_ctx, cb, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
 }
 
 static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
@@ -197,8 +197,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   WSABUF buffer;
 
   if (tcp->shutting_down) {
-    grpc_exec_ctx_push(exec_ctx, cb,
-                       GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb,
+                        GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL);
     return;
   }
 
@@ -222,7 +222,7 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   /* Did we get data immediately ? Yay. */
   if (info->wsa_error != WSAEWOULDBLOCK) {
     info->bytes_transfered = bytes_read;
-    grpc_exec_ctx_push(exec_ctx, &tcp->on_read, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &tcp->on_read, GRPC_ERROR_NONE, NULL);
     return;
   }
 
@@ -235,8 +235,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     int wsa_error = WSAGetLastError();
     if (wsa_error != WSA_IO_PENDING) {
       info->wsa_error = wsa_error;
-      grpc_exec_ctx_push(exec_ctx, &tcp->on_read,
-                         GRPC_WSA_ERROR(info->wsa_error, "WSARecv"), NULL);
+      grpc_exec_ctx_sched(exec_ctx, &tcp->on_read,
+                          GRPC_WSA_ERROR(info->wsa_error, "WSARecv"), NULL);
       return;
     }
   }
@@ -267,7 +267,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) {
   }
 
   TCP_UNREF(tcp, "write");
-  grpc_exec_ctx_push(exec_ctx, cb, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
 }
 
 /* Initiates a write. */
@@ -285,8 +285,8 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   size_t len;
 
   if (tcp->shutting_down) {
-    grpc_exec_ctx_push(exec_ctx, cb,
-                       GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb,
+                        GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL);
     return;
   }
 
@@ -317,7 +317,7 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     grpc_error *error = status == 0
                             ? GRPC_ERROR_NONE
                             : GRPC_WSA_ERROR(info->wsa_error, "WSASend");
-    grpc_exec_ctx_push(exec_ctx, cb, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
     return;
   }
 
@@ -334,8 +334,8 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     int wsa_error = WSAGetLastError();
     if (wsa_error != WSA_IO_PENDING) {
       TCP_UNREF(tcp, "write");
-      grpc_exec_ctx_push(exec_ctx, cb, GRPC_WSA_ERROR(wsa_error, "WSASend"),
-                         NULL);
+      grpc_exec_ctx_sched(exec_ctx, cb, GRPC_WSA_ERROR(wsa_error, "WSASend"),
+                          NULL);
       return;
     }
   }
diff --git a/src/core/lib/iomgr/timer.c b/src/core/lib/iomgr/timer.c
index b46c6df287..9975fa1671 100644
--- a/src/core/lib/iomgr/timer.c
+++ b/src/core/lib/iomgr/timer.c
@@ -186,7 +186,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 
   if (!g_initialized) {
     timer->triggered = 1;
-    grpc_exec_ctx_push(
+    grpc_exec_ctx_sched(
         exec_ctx, &timer->closure,
         GRPC_ERROR_CREATE("Attempt to create timer before initialization"),
         NULL);
@@ -195,7 +195,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 
   if (gpr_time_cmp(deadline, now) <= 0) {
     timer->triggered = 1;
-    grpc_exec_ctx_push(exec_ctx, &timer->closure, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_NONE, NULL);
     return;
   }
 
@@ -247,7 +247,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
   shard_type *shard = &g_shards[shard_idx(timer)];
   gpr_mu_lock(&shard->mu);
   if (!timer->triggered) {
-    grpc_exec_ctx_push(exec_ctx, &timer->closure, GRPC_ERROR_CANCELLED, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_CANCELLED, NULL);
     timer->triggered = 1;
     if (timer->heap_index == INVALID_HEAP_INDEX) {
       list_remove(timer);
@@ -287,8 +287,8 @@ static int refill_queue(shard_type *shard, gpr_timespec now) {
   return !grpc_timer_heap_is_empty(&shard->heap);
 }
 
-/* This pops the next non-cancelled timer with deadline <= now from the queue,
-   or returns NULL if there isn't one.
+/* This pops the next non-cancelled timer with deadline <= now from the
+   queue, or returns NULL if there isn't one.
    REQUIRES: shard->mu locked */
 static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) {
   grpc_timer *timer;
@@ -313,7 +313,7 @@ static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard,
   grpc_timer *timer;
   gpr_mu_lock(&shard->mu);
   while ((timer = pop_one(shard, now))) {
-    grpc_exec_ctx_push(exec_ctx, &timer->closure, GRPC_ERROR_REF(error), NULL);
+    grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_REF(error), NULL);
     n++;
   }
   *new_min_deadline = compute_min_deadline(shard);
diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c
index 98ffccd59b..16150687d3 100644
--- a/src/core/lib/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@@ -210,6 +210,8 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
                           size_t addr_len) {
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
+  /* Set send/receive socket buffers to 1 MB */
+  int buffer_size_bytes = 1024 * 1024;
 
   if (fd < 0) {
     goto error;
@@ -239,6 +241,18 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
     goto error;
   }
 
+  if (!grpc_set_socket_sndbuf(fd, buffer_size_bytes)) {
+    gpr_log(GPR_ERROR, "Failed to set send buffer size to %d bytes",
+            buf_size_bytes);
+    goto error;
+  }
+
+  if (!grpc_set_socket_rcvbuf(fd, buffer_size_bytes)) {
+    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %d bytes",
+            buf_size_bytes);
+    goto error;
+  }
+
   return grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
 
 error:
diff --git a/src/core/lib/iomgr/workqueue.h b/src/core/lib/iomgr/workqueue.h
index 6e8b1218be..5cc40eea50 100644
--- a/src/core/lib/iomgr/workqueue.h
+++ b/src/core/lib/iomgr/workqueue.h
@@ -43,7 +43,7 @@
 #include "src/core/lib/iomgr/workqueue_posix.h"
 #endif
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 #include "src/core/lib/iomgr/workqueue_windows.h"
 #endif
 
@@ -78,7 +78,7 @@ void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx,
                                    grpc_pollset *pollset);
 
 /** Add a work item to a workqueue */
-void grpc_workqueue_push(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
-                         grpc_closure *closure, grpc_error *error);
+void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
+                            grpc_closure *closure, grpc_error *error);
 
 #endif /* GRPC_CORE_LIB_IOMGR_WORKQUEUE_H */
diff --git a/src/core/lib/iomgr/workqueue_posix.c b/src/core/lib/iomgr/workqueue_posix.c
index 297e8d3102..45e0f6063b 100644
--- a/src/core/lib/iomgr/workqueue_posix.c
+++ b/src/core/lib/iomgr/workqueue_posix.c
@@ -137,8 +137,8 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   }
 }
 
-void grpc_workqueue_push(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
-                         grpc_closure *closure, grpc_error *error) {
+void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
+                            grpc_closure *closure, grpc_error *error) {
   grpc_error *push_error = GRPC_ERROR_NONE;
   gpr_mu_lock(&workqueue->mu);
   if (grpc_closure_list_empty(workqueue->closure_list)) {
diff --git a/src/core/lib/iomgr/workqueue_windows.c b/src/core/lib/iomgr/workqueue_windows.c
index c3c0446a57..275f040b1c 100644
--- a/src/core/lib/iomgr/workqueue_windows.c
+++ b/src/core/lib/iomgr/workqueue_windows.c
@@ -33,8 +33,8 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include "src/core/lib/iomgr/workqueue.h"
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
index 18189a8fb8..07db8bfd75 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.c
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,6 +35,7 @@
 
 #include <string.h>
 
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/surface/api_trace.h"
 
 #include <grpc/support/alloc.h>
@@ -49,7 +50,7 @@ typedef struct {
   grpc_credentials_md_store *md_elems;
   grpc_auth_metadata_context auth_md_context;
   void *user_data;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   grpc_credentials_metadata_cb cb;
 } grpc_composite_call_credentials_metadata_context;
 
@@ -93,7 +94,7 @@ static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
     grpc_call_credentials *inner_creds =
         ctx->composite_creds->inner.creds_array[ctx->creds_index++];
     grpc_call_credentials_get_request_metadata(
-        exec_ctx, inner_creds, ctx->pollset, ctx->auth_md_context,
+        exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
         composite_call_metadata_cb, ctx);
     return;
   }
@@ -106,7 +107,7 @@ static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
 
 static void composite_call_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context auth_md_context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context auth_md_context,
     grpc_credentials_metadata_cb cb, void *user_data) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
   grpc_composite_call_credentials_metadata_context *ctx;
@@ -117,10 +118,10 @@ static void composite_call_get_request_metadata(
   ctx->user_data = user_data;
   ctx->cb = cb;
   ctx->composite_creds = c;
-  ctx->pollset = pollset;
+  ctx->pollent = pollent;
   ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
   grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], pollset,
+      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pollent,
       auth_md_context, composite_call_metadata_cb, ctx);
 }
 
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h
index 43974a9623..0d8966f464 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.h
+++ b/src/core/lib/security/credentials/composite/composite_credentials.h
@@ -68,4 +68,5 @@ typedef struct {
   grpc_call_credentials_array inner;
 } grpc_composite_call_credentials;
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
index ce31919f74..0eadaec191 100644
--- a/src/core/lib/security/credentials/credentials.c
+++ b/src/core/lib/security/credentials/credentials.c
@@ -113,7 +113,7 @@ void grpc_call_credentials_release(grpc_call_credentials *creds) {
 
 void grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data) {
   if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
     if (cb != NULL) {
@@ -121,7 +121,7 @@ void grpc_call_credentials_get_request_metadata(
     }
     return;
   }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pollset, context, cb,
+  creds->vtable->get_request_metadata(exec_ctx, creds, pollent, context, cb,
                                       user_data);
 }
 
diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h
index 675e02b58d..ce235e3a1d 100644
--- a/src/core/lib/security/credentials/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -41,6 +41,7 @@
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/security/transport/security_connector.h"
 
 struct grpc_http_response;
@@ -164,7 +165,8 @@ typedef void (*grpc_credentials_metadata_cb)(grpc_exec_ctx *exec_ctx,
 typedef struct {
   void (*destruct)(grpc_call_credentials *c);
   void (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
-                               grpc_call_credentials *c, grpc_pollset *pollset,
+                               grpc_call_credentials *c,
+                               grpc_polling_entity *pollent,
                                grpc_auth_metadata_context context,
                                grpc_credentials_metadata_cb cb,
                                void *user_data);
@@ -180,7 +182,7 @@ grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds);
 void grpc_call_credentials_unref(grpc_call_credentials *creds);
 void grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data);
 
 /* Metadata-only credentials with the specified key and value where
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
index 1ff7bd14a5..ee6d964de1 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.c
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -106,7 +106,7 @@ static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
 
 static void md_only_test_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
 
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
index adeba94bdb..9cf38084a3 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.h
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -53,4 +53,4 @@ typedef struct {
   int is_async;
 } grpc_md_only_test_credentials;
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/google_default/credentials_win32.c b/src/core/lib/security/credentials/google_default/credentials_windows.c
index cd8b48080a..208b8fd9ad 100644
--- a/src/core/lib/security/credentials/google_default/credentials_win32.c
+++ b/src/core/lib/security/credentials/google_default/credentials_windows.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
@@ -58,4 +58,4 @@ char *grpc_get_well_known_google_credentials_file_path_impl(void) {
   return result;
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c
index 29f818b8c8..312a3d4f90 100644
--- a/src/core/lib/security/credentials/google_default/google_default_credentials.c
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c
@@ -42,6 +42,7 @@
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/load_file.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
 #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/support/env.h"
@@ -63,7 +64,7 @@ static gpr_once g_once = GPR_ONCE_INIT;
 static void init_default_credentials(void) { gpr_mu_init(&g_state_mu); }
 
 typedef struct {
-  grpc_pollset *pollset;
+  grpc_polling_entity pollent;
   int is_done;
   int success;
   grpc_http_response response;
@@ -89,7 +90,9 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx,
   }
   gpr_mu_lock(g_polling_mu);
   detector->is_done = 1;
-  GRPC_LOG_IF_ERROR("Pollset kick", grpc_pollset_kick(detector->pollset, NULL));
+  GRPC_LOG_IF_ERROR(
+      "Pollset kick",
+      grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL));
   gpr_mu_unlock(g_polling_mu);
 }
 
@@ -108,8 +111,9 @@ static int is_stack_running_on_compute_engine(void) {
      on compute engine. */
   gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN);
 
-  detector.pollset = gpr_malloc(grpc_pollset_size());
-  grpc_pollset_init(detector.pollset, &g_polling_mu);
+  grpc_pollset *pollset = gpr_malloc(grpc_pollset_size());
+  grpc_pollset_init(pollset, &g_polling_mu);
+  detector.pollent = grpc_polling_entity_create_from_pollset(pollset);
   detector.is_done = 0;
   detector.success = 0;
 
@@ -121,7 +125,7 @@ static int is_stack_running_on_compute_engine(void) {
   grpc_httpcli_context_init(&context);
 
   grpc_httpcli_get(
-      &exec_ctx, &context, detector.pollset, &request,
+      &exec_ctx, &context, &detector.pollent, &request,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay),
       grpc_closure_create(on_compute_engine_detection_http_response, &detector),
       &detector.response);
@@ -135,8 +139,9 @@ static int is_stack_running_on_compute_engine(void) {
     grpc_pollset_worker *worker = NULL;
     if (!GRPC_LOG_IF_ERROR(
             "pollset_work",
-            grpc_pollset_work(&exec_ctx, detector.pollset, &worker,
-                              gpr_now(GPR_CLOCK_MONOTONIC),
+            grpc_pollset_work(&exec_ctx,
+                              grpc_polling_entity_pollset(&detector.pollent),
+                              &worker, gpr_now(GPR_CLOCK_MONOTONIC),
                               gpr_inf_future(GPR_CLOCK_MONOTONIC)))) {
       detector.is_done = 1;
       detector.success = 0;
@@ -145,12 +150,15 @@ static int is_stack_running_on_compute_engine(void) {
   gpr_mu_unlock(g_polling_mu);
 
   grpc_httpcli_context_destroy(&context);
-  grpc_closure_init(&destroy_closure, destroy_pollset, detector.pollset);
-  grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure);
+  grpc_closure_init(&destroy_closure, destroy_pollset,
+                    grpc_polling_entity_pollset(&detector.pollent));
+  grpc_pollset_shutdown(&exec_ctx,
+                        grpc_polling_entity_pollset(&detector.pollent),
+                        &destroy_closure);
   grpc_exec_ctx_finish(&exec_ctx);
   g_polling_mu = NULL;
 
-  gpr_free(detector.pollset);
+  gpr_free(grpc_polling_entity_pollset(&detector.pollent));
   grpc_http_response_destroy(&detector.response);
 
   return detector.success;
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.h b/src/core/lib/security/credentials/google_default/google_default_credentials.h
index 0d14cbfdad..fac4377e2c 100644
--- a/src/core/lib/security/credentials/google_default/google_default_credentials.h
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.h
@@ -42,4 +42,5 @@
 
 void grpc_flush_cached_google_default_credentials(void);
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
index 89defa7c60..64d5871844 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.c
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -49,7 +49,7 @@ static void iam_destruct(grpc_call_credentials *creds) {
 
 static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
-                                     grpc_pollset *pollset,
+                                     grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
                                      grpc_credentials_metadata_cb cb,
                                      void *user_data) {
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h
index e989bfc28e..af54faa586 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.h
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -41,4 +41,4 @@ typedef struct {
   grpc_credentials_md_store *iam_md;
 } grpc_google_iam_credentials;
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
index 8755a96af4..973fb75eaa 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.c
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -64,7 +64,7 @@ static void jwt_destruct(grpc_call_credentials *creds) {
 
 static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
-                                     grpc_pollset *pollset,
+                                     grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
                                      grpc_credentials_metadata_cb cb,
                                      void *user_data) {
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
index dfa4a1ebd9..d572606179 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.h
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -59,4 +59,4 @@ grpc_call_credentials *
 grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
     grpc_auth_json_key key, gpr_timespec token_lifetime);
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
index bdee6c85f7..73eb2e3258 100644
--- a/src/core/lib/security/credentials/jwt/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -37,6 +37,7 @@
 #include <string.h>
 
 #include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/security/util/b64.h"
 #include "src/core/lib/tsi/ssl_types.h"
 
@@ -320,9 +321,15 @@ grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
 
 /* --- verifier_cb_ctx object. --- */
 
+typedef enum {
+  HTTP_RESPONSE_OPENID = 0,
+  HTTP_RESPONSE_KEYS,
+  HTTP_RESPONSE_COUNT /* must be last */
+} http_response_index;
+
 typedef struct {
   grpc_jwt_verifier *verifier;
-  grpc_pollset *pollset;
+  grpc_polling_entity pollent;
   jose_header *header;
   grpc_jwt_claims *claims;
   char *audience;
@@ -330,7 +337,7 @@ typedef struct {
   gpr_slice signed_data;
   void *user_data;
   grpc_jwt_verification_done_cb user_cb;
-  grpc_http_response responses[2];
+  grpc_http_response responses[HTTP_RESPONSE_COUNT];
 } verifier_cb_ctx;
 
 /* Takes ownership of the header, claims and signature. */
@@ -339,10 +346,11 @@ static verifier_cb_ctx *verifier_cb_ctx_create(
     grpc_jwt_claims *claims, const char *audience, gpr_slice signature,
     const char *signed_jwt, size_t signed_jwt_len, void *user_data,
     grpc_jwt_verification_done_cb cb) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   verifier_cb_ctx *ctx = gpr_malloc(sizeof(verifier_cb_ctx));
   memset(ctx, 0, sizeof(verifier_cb_ctx));
   ctx->verifier = verifier;
-  ctx->pollset = pollset;
+  ctx->pollent = grpc_polling_entity_create_from_pollset(pollset);
   ctx->header = header;
   ctx->audience = gpr_strdup(audience);
   ctx->claims = claims;
@@ -350,6 +358,7 @@ static verifier_cb_ctx *verifier_cb_ctx_create(
   ctx->signed_data = gpr_slice_from_copied_buffer(signed_jwt, signed_jwt_len);
   ctx->user_data = user_data;
   ctx->user_cb = cb;
+  grpc_exec_ctx_finish(&exec_ctx);
   return ctx;
 }
 
@@ -359,7 +368,7 @@ void verifier_cb_ctx_destroy(verifier_cb_ctx *ctx) {
   gpr_slice_unref(ctx->signature);
   gpr_slice_unref(ctx->signed_data);
   jose_header_destroy(ctx->header);
-  for (size_t i = 0; i < GPR_ARRAY_SIZE(ctx->responses); i++) {
+  for (size_t i = 0; i < HTTP_RESPONSE_COUNT; i++) {
     grpc_http_response_destroy(&ctx->responses[i]);
   }
   /* TODO: see what to do with claims... */
@@ -578,7 +587,7 @@ end:
 static void on_keys_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
                               grpc_error *error) {
   verifier_cb_ctx *ctx = (verifier_cb_ctx *)user_data;
-  grpc_json *json = json_from_http(&ctx->responses[1]);
+  grpc_json *json = json_from_http(&ctx->responses[HTTP_RESPONSE_KEYS]);
   EVP_PKEY *verification_key = NULL;
   grpc_jwt_verifier_status status = GRPC_JWT_VERIFIER_GENERIC_ERROR;
   grpc_jwt_claims *claims = NULL;
@@ -620,7 +629,7 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
                                        grpc_error *error) {
   const grpc_json *cur;
   verifier_cb_ctx *ctx = (verifier_cb_ctx *)user_data;
-  const grpc_http_response *response = &ctx->responses[0];
+  const grpc_http_response *response = &ctx->responses[HTTP_RESPONSE_OPENID];
   grpc_json *json = json_from_http(response);
   grpc_httpcli_request req;
   const char *jwks_uri;
@@ -649,9 +658,10 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
   }
 
   grpc_httpcli_get(
-      exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req,
+      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
-      grpc_closure_create(on_keys_retrieved, ctx), &ctx->responses[1]);
+      grpc_closure_create(on_keys_retrieved, ctx),
+      &ctx->responses[HTTP_RESPONSE_KEYS]);
   grpc_json_destroy(json);
   gpr_free(req.host);
   return;
@@ -699,7 +709,7 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
   grpc_httpcli_request req;
   memset(&req, 0, sizeof(grpc_httpcli_request));
   req.handshaker = &grpc_httpcli_ssl;
-  int rsp_idx;
+  http_response_index rsp_idx;
 
   GPR_ASSERT(ctx != NULL && ctx->header != NULL && ctx->claims != NULL);
   iss = ctx->claims->iss;
@@ -739,7 +749,7 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
       gpr_asprintf(&req.http.path, "/%s/%s", path_prefix, iss);
     }
     http_cb = grpc_closure_create(on_keys_retrieved, ctx);
-    rsp_idx = 1;
+    rsp_idx = HTTP_RESPONSE_KEYS;
   } else {
     req.host = gpr_strdup(strstr(iss, "https://") == iss ? iss + 8 : iss);
     path_prefix = strchr(req.host, '/');
@@ -751,11 +761,11 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
                    GRPC_OPENID_CONFIG_URL_SUFFIX);
     }
     http_cb = grpc_closure_create(on_openid_config_retrieved, ctx);
-    rsp_idx = 0;
+    rsp_idx = HTTP_RESPONSE_OPENID;
   }
 
   grpc_httpcli_get(
-      exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req,
+      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
       http_cb, &ctx->responses[rsp_idx]);
   gpr_free(req.host);
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
index 671dea31e6..1102553dd3 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -226,6 +226,8 @@ static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx,
   gpr_timespec token_lifetime;
   grpc_credentials_status status;
 
+  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
+
   gpr_mu_lock(&c->mu);
   status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
       &r->response, &c->access_token_md, &token_lifetime);
@@ -244,7 +246,7 @@ static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx,
 
 static void oauth2_token_fetcher_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
@@ -270,7 +272,7 @@ static void oauth2_token_fetcher_get_request_metadata(
     c->fetch_func(
         exec_ctx,
         grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pollset, on_oauth2_token_fetcher_http_response,
+        &c->httpcli_context, pollent, on_oauth2_token_fetcher_http_response,
         gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
   }
 }
@@ -295,7 +297,7 @@ static grpc_call_credentials_vtable compute_engine_vtable = {
 
 static void compute_engine_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
+    grpc_httpcli_context *httpcli_context, grpc_polling_entity *pollent,
     grpc_iomgr_cb_func response_cb, gpr_timespec deadline) {
   grpc_http_header header = {"Metadata-Flavor", "Google"};
   grpc_httpcli_request request;
@@ -304,7 +306,7 @@ static void compute_engine_fetch_oauth2(
   request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
   request.http.hdr_count = 1;
   request.http.hdrs = &header;
-  grpc_httpcli_get(exec_ctx, httpcli_context, pollset, &request, deadline,
+  grpc_httpcli_get(exec_ctx, httpcli_context, pollent, &request, deadline,
                    grpc_closure_create(response_cb, metadata_req),
                    &metadata_req->response);
 }
@@ -337,7 +339,7 @@ static grpc_call_credentials_vtable refresh_token_vtable = {
 
 static void refresh_token_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
+    grpc_httpcli_context *httpcli_context, grpc_polling_entity *pollent,
     grpc_iomgr_cb_func response_cb, gpr_timespec deadline) {
   grpc_google_refresh_token_credentials *c =
       (grpc_google_refresh_token_credentials *)metadata_req->creds;
@@ -354,7 +356,7 @@ static void refresh_token_fetch_oauth2(
   request.http.hdr_count = 1;
   request.http.hdrs = &header;
   request.handshaker = &grpc_httpcli_ssl;
-  grpc_httpcli_post(exec_ctx, httpcli_context, pollset, &request, body,
+  grpc_httpcli_post(exec_ctx, httpcli_context, pollent, &request, body,
                     strlen(body), deadline,
                     grpc_closure_create(response_cb, metadata_req),
                     &metadata_req->response);
@@ -399,7 +401,7 @@ static void access_token_destruct(grpc_call_credentials *creds) {
 
 static void access_token_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
   cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
index be7bc81626..7f6f205c22 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -70,7 +70,7 @@ void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
 typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
                                        grpc_credentials_metadata_request *req,
                                        grpc_httpcli_context *http_context,
-                                       grpc_pollset *pollset,
+                                       grpc_polling_entity *pollent,
                                        grpc_iomgr_cb_func cb,
                                        gpr_timespec deadline);
 typedef struct {
@@ -106,4 +106,4 @@ grpc_oauth2_token_fetcher_credentials_parse_server_response(
     const struct grpc_http_response *response,
     grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
index bae357321e..9fb55e8466 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.c
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -94,7 +94,7 @@ static void plugin_md_request_metadata_ready(void *request,
 
 static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                         grpc_call_credentials *creds,
-                                        grpc_pollset *pollset,
+                                        grpc_polling_entity *pollent,
                                         grpc_auth_metadata_context context,
                                         grpc_credentials_metadata_cb cb,
                                         void *user_data) {
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
index b5dd3ad5e2..89073cb3d1 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.h
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -42,4 +42,4 @@ typedef struct {
   grpc_credentials_md_store *plugin_md;
 } grpc_plugin_credentials;
 
-#endif  // GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index e3cbcb4433..76be2acd72 100644
--- a/src/core/lib/security/transport/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -54,11 +54,11 @@ typedef struct {
   grpc_call_credentials *creds;
   grpc_mdstr *host;
   grpc_mdstr *method;
-  /* pollset bound to this call; if we need to make external
-     network requests, they should be done under this pollset
-     so that work can progress when this call wants work to
-     progress */
-  grpc_pollset *pollset;
+  /* pollset{_set} bound to this call; if we need to make external
+     network requests, they should be done under a pollset added to this
+     pollset_set so that work can progress when this call wants work to progress
+  */
+  grpc_polling_entity *pollent;
   grpc_transport_stream_op op;
   uint8_t security_context_set;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
@@ -184,9 +184,9 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
   build_auth_metadata_context(&chand->security_connector->base,
                               chand->auth_context, calld);
   calld->op = *op; /* Copy op (originates from the caller's stack). */
-  GPR_ASSERT(calld->pollset);
+  GPR_ASSERT(calld->pollent != NULL);
   grpc_call_credentials_get_request_metadata(
-      exec_ctx, calld->creds, calld->pollset, calld->auth_md_context,
+      exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
       on_credentials_metadata, elem);
 }
 
@@ -270,15 +270,16 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   memset(calld, 0, sizeof(*calld));
 }
 
-static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                        grpc_pollset *pollset) {
+static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                       grpc_call_element *elem,
+                                       grpc_polling_entity *pollent) {
   call_data *calld = elem->call_data;
-  calld->pollset = pollset;
+  calld->pollent = pollent;
 }
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {
+                              const grpc_call_stats *stats, void *ignored) {
   call_data *calld = elem->call_data;
   grpc_call_credentials_unref(calld->creds);
   if (calld->host != NULL) {
@@ -329,8 +330,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   GRPC_AUTH_CONTEXT_UNREF(chand->auth_context, "client_auth_filter");
 }
 
-const grpc_channel_filter grpc_client_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem,          set_pollset,          destroy_call_elem,
-    sizeof(channel_data),    init_channel_elem,    destroy_channel_elem,
-    grpc_call_next_get_peer, "client-auth"};
+const grpc_channel_filter grpc_client_auth_filter = {auth_start_transport_op,
+                                                     grpc_channel_next_op,
+                                                     sizeof(call_data),
+                                                     init_call_elem,
+                                                     set_pollset_or_pollset_set,
+                                                     destroy_call_elem,
+                                                     sizeof(channel_data),
+                                                     init_channel_elem,
+                                                     destroy_channel_elem,
+                                                     grpc_call_next_get_peer,
+                                                     "client-auth"};
diff --git a/src/core/lib/security/transport/handshake.c b/src/core/lib/security/transport/handshake.c
index c87262f124..b374ca7633 100644
--- a/src/core/lib/security/transport/handshake.c
+++ b/src/core/lib/security/transport/handshake.c
@@ -39,6 +39,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
+#include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/tsi_error.h"
@@ -61,6 +62,8 @@ typedef struct {
   grpc_closure on_handshake_data_sent_to_peer;
   grpc_closure on_handshake_data_received_from_peer;
   grpc_auth_context *auth_context;
+  grpc_timer timer;
+  gpr_refcount refs;
 } grpc_security_handshake;
 
 static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
@@ -97,9 +100,23 @@ static void security_connector_remove_handshake(grpc_security_handshake *h) {
   gpr_mu_unlock(&sc->mu);
 }
 
+static void unref_handshake(grpc_security_handshake *h) {
+  if (gpr_unref(&h->refs)) {
+    if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker);
+    if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer);
+    gpr_slice_buffer_destroy(&h->left_overs);
+    gpr_slice_buffer_destroy(&h->outgoing);
+    gpr_slice_buffer_destroy(&h->incoming);
+    GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
+    GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
+    gpr_free(h);
+  }
+}
+
 static void security_handshake_done(grpc_exec_ctx *exec_ctx,
                                     grpc_security_handshake *h,
                                     grpc_error *error) {
+  grpc_timer_cancel(exec_ctx, &h->timer);
   if (!h->is_client_side) {
     security_connector_remove_handshake(h);
   }
@@ -119,14 +136,7 @@ static void security_handshake_done(grpc_exec_ctx *exec_ctx,
     }
     h->cb(exec_ctx, h->user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   }
-  if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker);
-  if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer);
-  gpr_slice_buffer_destroy(&h->left_overs);
-  gpr_slice_buffer_destroy(&h->outgoing);
-  gpr_slice_buffer_destroy(&h->incoming);
-  GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
-  GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
-  gpr_free(h);
+  unref_handshake(h);
   GRPC_ERROR_UNREF(error);
 }
 
@@ -149,7 +159,7 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
   if (result != TSI_OK) {
     security_handshake_done(
         exec_ctx, h,
-        grpc_set_tsi_error_bits(
+        grpc_set_tsi_error_result(
             GRPC_ERROR_CREATE("Frame protector creation failed"), result));
     return;
   }
@@ -168,7 +178,7 @@ static void check_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) {
 
   if (result != TSI_OK) {
     security_handshake_done(
-        exec_ctx, h, grpc_set_tsi_error_bits(
+        exec_ctx, h, grpc_set_tsi_error_result(
                          GRPC_ERROR_CREATE("Peer extraction failed"), result));
     return;
   }
@@ -195,9 +205,9 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
   } while (result == TSI_INCOMPLETE_DATA);
 
   if (result != TSI_OK) {
-    security_handshake_done(
-        exec_ctx, h,
-        grpc_set_tsi_error_bits(GRPC_ERROR_CREATE("Handshake failed"), result));
+    security_handshake_done(exec_ctx, h,
+                            grpc_set_tsi_error_result(
+                                GRPC_ERROR_CREATE("Handshake failed"), result));
     return;
   }
 
@@ -249,9 +259,9 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
   }
 
   if (result != TSI_OK) {
-    security_handshake_done(
-        exec_ctx, h,
-        grpc_set_tsi_error_bits(GRPC_ERROR_CREATE("Handshake failed"), result));
+    security_handshake_done(exec_ctx, h,
+                            grpc_set_tsi_error_result(
+                                GRPC_ERROR_CREATE("Handshake failed"), result));
     return;
   }
 
@@ -304,13 +314,19 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
-                                tsi_handshaker *handshaker,
-                                grpc_security_connector *connector,
-                                bool is_client_side,
-                                grpc_endpoint *nonsecure_endpoint,
-                                grpc_security_handshake_done_cb cb,
-                                void *user_data) {
+static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  grpc_security_handshake *h = arg;
+  if (error == GRPC_ERROR_NONE) {
+    grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
+  }
+  unref_handshake(h);
+}
+
+void grpc_do_security_handshake(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector, bool is_client_side,
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data) {
   grpc_security_connector_handshake_list *handshake_node;
   grpc_security_handshake *h = gpr_malloc(sizeof(grpc_security_handshake));
   memset(h, 0, sizeof(grpc_security_handshake));
@@ -322,6 +338,7 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
   h->wrapped_endpoint = nonsecure_endpoint;
   h->user_data = user_data;
   h->cb = cb;
+  gpr_ref_init(&h->refs, 2); /* timer and handshake proper each get a ref */
   grpc_closure_init(&h->on_handshake_data_sent_to_peer,
                     on_handshake_data_sent_to_peer, h);
   grpc_closure_init(&h->on_handshake_data_received_from_peer,
@@ -340,6 +357,8 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
     gpr_mu_unlock(&server_connector->mu);
   }
   send_handshake_bytes_to_peer(exec_ctx, h);
+  grpc_timer_init(exec_ctx, &h->timer, deadline, on_timeout, h,
+                  gpr_now(deadline.clock_type));
 }
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/lib/security/transport/handshake.h b/src/core/lib/security/transport/handshake.h
index 6ed850b315..c0906dd6af 100644
--- a/src/core/lib/security/transport/handshake.h
+++ b/src/core/lib/security/transport/handshake.h
@@ -38,13 +38,11 @@
 #include "src/core/lib/security/transport/security_connector.h"
 
 /* Calls the callback upon completion. Takes owership of handshaker. */
-void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
-                                tsi_handshaker *handshaker,
-                                grpc_security_connector *connector,
-                                bool is_client_side,
-                                grpc_endpoint *nonsecure_endpoint,
-                                grpc_security_handshake_done_cb cb,
-                                void *user_data);
+void grpc_do_security_handshake(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector, bool is_client_side,
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data);
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
 
diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
index 23b51f33f6..7650d68e89 100644
--- a/src/core/lib/security/transport/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -128,7 +128,7 @@ static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur,
 
 static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep,
                          grpc_error *error) {
-  if (grpc_trace_secure_endpoint) {
+  if (false && grpc_trace_secure_endpoint) {
     size_t i;
     for (i = 0; i < ep->read_buffer->count; i++) {
       char *data = gpr_dump_slice(ep->read_buffer->slices[i],
@@ -138,7 +138,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep,
     }
   }
   ep->read_buffer = NULL;
-  grpc_exec_ctx_push(exec_ctx, ep->read_cb, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, ep->read_cb, error, NULL);
   SECURE_ENDPOINT_UNREF(exec_ctx, ep, "read");
 }
 
@@ -211,7 +211,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
 
   if (result != TSI_OK) {
     gpr_slice_buffer_reset_and_unref(ep->read_buffer);
-    call_read_cb(exec_ctx, ep, grpc_set_tsi_error_bits(
+    call_read_cb(exec_ctx, ep, grpc_set_tsi_error_result(
                                    GRPC_ERROR_CREATE("Unwrap failed"), result));
     return;
   }
@@ -256,7 +256,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
 
   gpr_slice_buffer_reset_and_unref(&ep->output_buffer);
 
-  if (grpc_trace_secure_endpoint) {
+  if (false && grpc_trace_secure_endpoint) {
     for (i = 0; i < slices->count; i++) {
       char *data =
           gpr_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
@@ -319,9 +319,9 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
   if (result != TSI_OK) {
     /* TODO(yangg) do different things according to the error type? */
     gpr_slice_buffer_reset_and_unref(&ep->output_buffer);
-    grpc_exec_ctx_push(
+    grpc_exec_ctx_sched(
         exec_ctx, cb,
-        grpc_set_tsi_error_bits(GRPC_ERROR_CREATE("Wrap failed"), result),
+        grpc_set_tsi_error_result(GRPC_ERROR_CREATE("Wrap failed"), result),
         NULL);
     return;
   }
diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 03b64c5121..f0ee6770e5 100644
--- a/src/core/lib/security/transport/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -127,23 +127,25 @@ void grpc_server_security_connector_shutdown(
 
 void grpc_channel_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
-    grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb,
-    void *user_data) {
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data) {
   if (sc == NULL || nonsecure_endpoint == NULL) {
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
-    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data);
+    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
 void grpc_server_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
     grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_security_handshake_done_cb cb, void *user_data) {
+    gpr_timespec deadline, grpc_security_handshake_done_cb cb,
+    void *user_data) {
   if (sc == NULL || nonsecure_endpoint == NULL) {
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
-    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, cb, user_data);
+    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, deadline, cb,
+                     user_data);
   }
 }
 
@@ -310,20 +312,23 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
 static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx,
                                       grpc_channel_security_connector *sc,
                                       grpc_endpoint *nonsecure_endpoint,
+                                      gpr_timespec deadline,
                                       grpc_security_handshake_done_cb cb,
                                       void *user_data) {
   grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base,
-                             true, nonsecure_endpoint, cb, user_data);
+                             true, nonsecure_endpoint, deadline, cb, user_data);
 }
 
 static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx,
                                      grpc_server_security_connector *sc,
                                      grpc_tcp_server_acceptor *acceptor,
                                      grpc_endpoint *nonsecure_endpoint,
+                                     gpr_timespec deadline,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
   grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base,
-                             false, nonsecure_endpoint, cb, user_data);
+                             false, nonsecure_endpoint, deadline, cb,
+                             user_data);
 }
 
 static grpc_security_connector_vtable fake_channel_vtable = {
@@ -413,6 +418,7 @@ static grpc_security_status ssl_create_handshaker(
 static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_security_connector *sc,
                                      grpc_endpoint *nonsecure_endpoint,
+                                     gpr_timespec deadline,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
   grpc_ssl_channel_security_connector *c =
@@ -427,7 +433,7 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
     cb(exec_ctx, user_data, status, NULL, NULL);
   } else {
     grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, cb, user_data);
+                               nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
@@ -435,6 +441,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx,
                                     grpc_server_security_connector *sc,
                                     grpc_tcp_server_acceptor *acceptor,
                                     grpc_endpoint *nonsecure_endpoint,
+                                    gpr_timespec deadline,
                                     grpc_security_handshake_done_cb cb,
                                     void *user_data) {
   grpc_ssl_server_security_connector *c =
@@ -446,7 +453,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx,
     cb(exec_ctx, user_data, status, NULL, NULL);
   } else {
     grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false,
-                               nonsecure_endpoint, cb, user_data);
+                               nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 84e586deaa..c2ddf5ee1e 100644
--- a/src/core/lib/security/transport/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -143,7 +143,7 @@ struct grpc_channel_security_connector {
                           grpc_security_call_host_check_cb cb, void *user_data);
   void (*do_handshake)(grpc_exec_ctx *exec_ctx,
                        grpc_channel_security_connector *sc,
-                       grpc_endpoint *nonsecure_endpoint,
+                       grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
                        grpc_security_handshake_done_cb cb, void *user_data);
 };
 
@@ -156,8 +156,8 @@ void grpc_channel_security_connector_check_call_host(
 /* Handshake. */
 void grpc_channel_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
-    grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb,
-    void *user_data);
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data);
 
 /* --- server_security_connector object. ---
 
@@ -174,14 +174,14 @@ struct grpc_server_security_connector {
   void (*do_handshake)(grpc_exec_ctx *exec_ctx,
                        grpc_server_security_connector *sc,
                        grpc_tcp_server_acceptor *acceptor,
-                       grpc_endpoint *nonsecure_endpoint,
+                       grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
                        grpc_security_handshake_done_cb cb, void *user_data);
 };
 
 void grpc_server_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
     grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_security_handshake_done_cb cb, void *user_data);
+    gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data);
 
 void grpc_server_security_connector_shutdown(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector);
diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
index d45ec4020c..12e789bde9 100644
--- a/src/core/lib/security/transport/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -128,7 +128,7 @@ static void on_md_processing_done(
     grpc_metadata_batch_filter(calld->recv_initial_metadata, remove_consumed_md,
                                elem);
     grpc_metadata_array_destroy(&calld->md);
-    grpc_exec_ctx_push(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL);
   } else {
     gpr_slice message;
     grpc_transport_stream_op close_op;
@@ -146,10 +146,10 @@ static void on_md_processing_done(
     calld->transport_op.send_trailing_metadata = NULL;
     grpc_transport_stream_op_add_close(&close_op, status, &message);
     grpc_call_next_op(&exec_ctx, elem, &close_op);
-    grpc_exec_ctx_push(&exec_ctx, calld->on_done_recv,
-                       grpc_error_set_int(GRPC_ERROR_CREATE(error_details),
-                                          GRPC_ERROR_INT_GRPC_STATUS, status),
-                       NULL);
+    grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv,
+                        grpc_error_set_int(GRPC_ERROR_CREATE(error_details),
+                                           GRPC_ERROR_INT_GRPC_STATUS, status),
+                        NULL);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
@@ -169,8 +169,8 @@ static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
       return;
     }
   }
-  grpc_exec_ctx_push(exec_ctx, calld->on_done_recv, GRPC_ERROR_REF(error),
-                     NULL);
+  grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv, GRPC_ERROR_REF(error),
+                      NULL);
 }
 
 static void set_recv_ops_md_callbacks(grpc_call_element *elem,
@@ -224,12 +224,9 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
       grpc_server_security_context_destroy;
 }
 
-static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                        grpc_pollset *pollset) {}
-
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {}
+                              const grpc_call_stats *stats, void *ignored) {}
 
 /* Constructor for channel_data */
 static void init_channel_elem(grpc_exec_ctx *exec_ctx,
@@ -262,7 +259,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 }
 
 const grpc_channel_filter grpc_server_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem,          set_pollset,          destroy_call_elem,
-    sizeof(channel_data),    init_channel_elem,    destroy_channel_elem,
-    grpc_call_next_get_peer, "server-auth"};
+    auth_start_transport_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "server-auth"};
diff --git a/src/core/lib/security/transport/tsi_error.c b/src/core/lib/security/transport/tsi_error.c
index b9fb814905..afc1733567 100644
--- a/src/core/lib/security/transport/tsi_error.c
+++ b/src/core/lib/security/transport/tsi_error.c
@@ -33,7 +33,7 @@
 
 #include "src/core/lib/security/transport/tsi_error.h"
 
-grpc_error *grpc_set_tsi_error_bits(grpc_error *error, tsi_result result) {
+grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result) {
   return grpc_error_set_int(grpc_error_set_str(error, GRPC_ERROR_STR_TSI_ERROR,
                                                tsi_result_to_string(result)),
                             GRPC_ERROR_INT_TSI_CODE, result);
diff --git a/src/core/lib/security/transport/tsi_error.h b/src/core/lib/security/transport/tsi_error.h
index 0b691a427a..636fbb89cf 100644
--- a/src/core/lib/security/transport/tsi_error.h
+++ b/src/core/lib/security/transport/tsi_error.h
@@ -37,6 +37,6 @@
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
 
-grpc_error *grpc_set_tsi_error_bits(grpc_error *error, tsi_result result);
+grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result);
 
 #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_TSI_ERROR_H */
diff --git a/src/core/lib/security/util/json_util.h b/src/core/lib/security/util/json_util.h
index 5959626a5f..137900593f 100644
--- a/src/core/lib/security/util/json_util.h
+++ b/src/core/lib/security/util/json_util.h
@@ -52,4 +52,4 @@ const char *grpc_json_get_string_property(const grpc_json *json,
 bool grpc_copy_json_string_property(const grpc_json *json,
                                     const char *prop_name, char **copied_value);
 
-#endif  // GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H */
diff --git a/src/core/lib/support/cpu_windows.c b/src/core/lib/support/cpu_windows.c
index ce32eb0a9d..34d006bfc8 100644
--- a/src/core/lib/support/cpu_windows.c
+++ b/src/core/lib/support/cpu_windows.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 #include <grpc/support/log.h>
 
 unsigned gpr_cpu_num_cores(void) {
@@ -44,4 +44,4 @@ unsigned gpr_cpu_num_cores(void) {
 
 unsigned gpr_cpu_current_cpu(void) { return GetCurrentProcessorNumber(); }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/env_win32.c b/src/core/lib/support/env_windows.c
index e670e1e8d0..9116959442 100644
--- a/src/core/lib/support/env_win32.c
+++ b/src/core/lib/support/env_windows.c
@@ -33,13 +33,13 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_ENV
+#ifdef GPR_WINDOWS_ENV
 
 #include <windows.h>
 
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -76,4 +76,4 @@ void gpr_setenv(const char *name, const char *value) {
   GPR_ASSERT(res);
 }
 
-#endif /* GPR_WIN32_ENV */
+#endif /* GPR_WINDOWS_ENV */
diff --git a/src/core/lib/support/log.c b/src/core/lib/support/log.c
index 882abf673c..bae0957df7 100644
--- a/src/core/lib/support/log.c
+++ b/src/core/lib/support/log.c
@@ -96,4 +96,6 @@ void gpr_log_verbosity_init() {
   }
 }
 
-void gpr_set_log_function(gpr_log_func f) { g_log_func = f; }
+void gpr_set_log_function(gpr_log_func f) {
+  g_log_func = f ? f : gpr_default_log;
+}
diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c
index ca04c022e3..508fae4eec 100644
--- a/src/core/lib/support/log_linux.c
+++ b/src/core/lib/support/log_linux.c
@@ -95,9 +95,9 @@ void gpr_default_log(gpr_log_func_args *args) {
     strcpy(time_buffer, "error:strftime");
   }
 
-  gpr_asprintf(&prefix, "%s%s.%09d %7tu %s:%d]",
+  gpr_asprintf(&prefix, "%s%s.%09" PRId32 " %7ld %s:%d]",
                gpr_log_severity_string(args->severity), time_buffer,
-               (int)(now.tv_nsec), gettid(), display_file, args->line);
+               now.tv_nsec, gettid(), display_file, args->line);
 
   fprintf(stderr, "%-60s %s\n", prefix, args->message);
   gpr_free(prefix);
diff --git a/src/core/lib/support/log_win32.c b/src/core/lib/support/log_windows.c
index 29735bd18c..ea898c359d 100644
--- a/src/core/lib/support/log_win32.c
+++ b/src/core/lib/support/log_windows.c
@@ -33,19 +33,19 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_LOG
+#ifdef GPR_WINDOWS_LOG
 
 #include <stdarg.h>
 #include <stdio.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
 #include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 
 void gpr_log(const char *file, int line, gpr_log_severity severity,
              const char *format, ...) {
@@ -109,4 +109,4 @@ void gpr_default_log(gpr_log_func_args *args) {
   fflush(stderr);
 }
 
-#endif /* GPR_WIN32_LOG */
+#endif /* GPR_WINDOWS_LOG */
diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c
index a2ab6c5f1f..30c1e67647 100644
--- a/src/core/lib/support/string.c
+++ b/src/core/lib/support/string.c
@@ -194,6 +194,16 @@ int int64_ttoa(int64_t value, char *string) {
   return i;
 }
 
+char *gpr_leftpad(const char *str, char flag, size_t length) {
+  const size_t str_length = strlen(str);
+  const size_t out_length = str_length > length ? str_length : length;
+  char *out = gpr_malloc(out_length + 1);
+  memset(out, flag, out_length - str_length);
+  memcpy(out + out_length - str_length, str, str_length);
+  out[out_length] = 0;
+  return out;
+}
+
 char *gpr_strjoin(const char **strs, size_t nstrs, size_t *final_length) {
   return gpr_strjoin_sep(strs, nstrs, "", final_length);
 }
diff --git a/src/core/lib/support/string.h b/src/core/lib/support/string.h
index ea58610914..2b6bb3eec6 100644
--- a/src/core/lib/support/string.h
+++ b/src/core/lib/support/string.h
@@ -83,6 +83,10 @@ int int64_ttoa(int64_t value, char *output);
 /* Reverse a run of bytes */
 void gpr_reverse_bytes(char *str, int len);
 
+/* Pad a string with flag characters. The given length specifies the minimum
+   field width. The input string is never truncated. */
+char *gpr_leftpad(const char *str, char flag, size_t length);
+
 /* Join a set of strings, returning the resulting string.
    Total combined length (excluding null terminator) is returned in total_length
    if it is non-null. */
diff --git a/src/core/lib/support/string_util_win32.c b/src/core/lib/support/string_util_windows.c
index 0d7bcdb5aa..049c9a8c04 100644
--- a/src/core/lib/support/string_util_win32.c
+++ b/src/core/lib/support/string_util_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 /* Some platforms (namely msys) need wchar to be included BEFORE
    anything else, especially strsafe.h. */
@@ -91,4 +91,4 @@ char *gpr_format_message(int messageid) {
   return message;
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/string_win32.c b/src/core/lib/support/string_windows.c
index 6b92f79253..ecc2a3a4e5 100644
--- a/src/core/lib/support/string_win32.c
+++ b/src/core/lib/support/string_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_STRING
+#ifdef GPR_WINDOWS_STRING
 
 #include <stdarg.h>
 #include <stdio.h>
@@ -80,4 +80,4 @@ int gpr_asprintf(char **strp, const char *format, ...) {
   return -1;
 }
 
-#endif /* GPR_WIN32_STRING */
+#endif /* GPR_WINDOWS_STRING */
diff --git a/src/core/lib/support/string_win32.h b/src/core/lib/support/string_windows.h
index ff4a694ca9..899563b72d 100644
--- a/src/core/lib/support/string_win32.h
+++ b/src/core/lib/support/string_windows.h
@@ -31,17 +31,17 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H
-#define GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H
+#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H
+#define GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 /* These allocate new strings using gpr_malloc to convert from and to utf-8. */
 LPTSTR gpr_char_to_tchar(LPCSTR input);
 LPSTR gpr_tchar_to_char(LPCTSTR input);
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
 
-#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H */
diff --git a/src/core/lib/support/subprocess_windows.c b/src/core/lib/support/subprocess_windows.c
index 264306f1bd..dee8c44ac1 100644
--- a/src/core/lib/support/subprocess_windows.c
+++ b/src/core/lib/support/subprocess_windows.c
@@ -43,7 +43,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/subprocess.h>
 #include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 
 struct gpr_subprocess {
   PROCESS_INFORMATION pi;
diff --git a/src/core/lib/support/sync_win32.c b/src/core/lib/support/sync_windows.c
index 470a9f9704..8f0e8ff69f 100644
--- a/src/core/lib/support/sync_win32.c
+++ b/src/core/lib/support/sync_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -130,4 +130,4 @@ void gpr_once_init(gpr_once *once, void (*init_function)(void)) {
   InitOnceExecuteOnce(once, run_once_func, &arg, &dummy);
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/thd_win32.c b/src/core/lib/support/thd_windows.c
index 6deb3140eb..74d2250df4 100644
--- a/src/core/lib/support/thd_win32.c
+++ b/src/core/lib/support/thd_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -114,4 +114,4 @@ void gpr_thd_join(gpr_thd_id t) {
   destroy_thread(info);
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/time_win32.c b/src/core/lib/support/time_windows.c
index 9e924ab3f4..6459732879 100644
--- a/src/core/lib/support/time_win32.c
+++ b/src/core/lib/support/time_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_TIME
+#ifdef GPR_WINDOWS_TIME
 
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
@@ -107,4 +107,4 @@ void gpr_sleep_until(gpr_timespec until) {
   }
 }
 
-#endif /* GPR_WIN32_TIME */
+#endif /* GPR_WINDOWS_TIME */
diff --git a/src/core/lib/support/tmpfile_msys.c b/src/core/lib/support/tmpfile_msys.c
index 2fdc89a64f..4f566c4c28 100644
--- a/src/core/lib/support/tmpfile_msys.c
+++ b/src/core/lib/support/tmpfile_msys.c
@@ -44,7 +44,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 #include "src/core/lib/support/tmpfile.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
diff --git a/src/core/lib/support/tmpfile_win32.c b/src/core/lib/support/tmpfile_windows.c
index 9ac73128c3..542f53e589 100644
--- a/src/core/lib/support/tmpfile_win32.c
+++ b/src/core/lib/support/tmpfile_windows.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_TMPFILE
+#ifdef GPR_WINDOWS_TMPFILE
 
 #include <io.h>
 #include <stdio.h>
@@ -44,7 +44,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 #include "src/core/lib/support/tmpfile.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
@@ -81,4 +81,4 @@ end:
   return result;
 }
 
-#endif /* GPR_WIN32_TMPFILE */
+#endif /* GPR_WINDOWS_TMPFILE */
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index 3915ba57a2..04291b0ee0 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -40,6 +40,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/slice.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
@@ -52,7 +53,9 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/transport/metadata.h"
 #include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport.h"
 
 /** The maximum number of concurrent batches possible.
     Based upon the maximum number of individually queueable ops in the batch
@@ -65,12 +68,6 @@
       - status/close recv (depending on client/server) */
 #define MAX_CONCURRENT_BATCHES 6
 
-typedef struct {
-  grpc_ioreq_completion_func on_complete;
-  void *user_data;
-  int success;
-} completed_request;
-
 #define MAX_SEND_EXTRA_METADATA_COUNT 3
 
 /* Status data for a request can come from several sources; this
@@ -97,25 +94,6 @@ typedef struct {
   grpc_mdstr *details;
 } received_status;
 
-/* How far through the GRPC stream have we read? */
-typedef enum {
-  /* We are still waiting for initial metadata to complete */
-  READ_STATE_INITIAL = 0,
-  /* We have gotten initial metadata, and are reading either
-     messages or trailing metadata */
-  READ_STATE_GOT_INITIAL_METADATA,
-  /* The stream is closed for reading */
-  READ_STATE_READ_CLOSED,
-  /* The stream is closed for reading & writing */
-  READ_STATE_STREAM_CLOSED
-} read_state;
-
-typedef enum {
-  WRITE_STATE_INITIAL = 0,
-  WRITE_STATE_STARTED,
-  WRITE_STATE_WRITE_CLOSED
-} write_state;
-
 typedef struct batch_control {
   grpc_call *call;
   grpc_cq_completion cq_completion;
@@ -135,6 +113,7 @@ typedef struct batch_control {
 
 struct grpc_call {
   grpc_completion_queue *cq;
+  grpc_polling_entity pollent;
   grpc_channel *channel;
   grpc_call *parent;
   grpc_call *first_child;
@@ -176,10 +155,10 @@ struct grpc_call {
   received_status status[STATUS_SOURCE_COUNT];
 
   /* Call stats: only valid after trailing metadata received */
-  grpc_transport_stream_stats stats;
+  grpc_call_stats stats;
 
-  /* Compression algorithm for the call */
-  grpc_compression_algorithm compression_algorithm;
+  /* Compression algorithm for *incoming* data */
+  grpc_compression_algorithm incoming_compression_algorithm;
   /* Supported encodings (compression algorithms), a bitset */
   uint32_t encodings_accepted_by_peer;
 
@@ -238,18 +217,19 @@ static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
 static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
                                           grpc_status_code status,
                                           const char *description);
+static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
+                                         grpc_status_code status,
+                                         const char *description);
 static void destroy_call(grpc_exec_ctx *exec_ctx, void *call_stack,
                          grpc_error *error);
 static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
                                   grpc_error *error);
 
-grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
-                            uint32_t propagation_mask,
-                            grpc_completion_queue *cq,
-                            const void *server_transport_data,
-                            grpc_mdelem **add_initial_metadata,
-                            size_t add_initial_metadata_count,
-                            gpr_timespec send_deadline) {
+grpc_call *grpc_call_create(
+    grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+    grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative,
+    const void *server_transport_data, grpc_mdelem **add_initial_metadata,
+    size_t add_initial_metadata_count, gpr_timespec send_deadline) {
   size_t i, j;
   grpc_channel_stack *channel_stack = grpc_channel_get_channel_stack(channel);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -286,9 +266,20 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
                        call->context, server_transport_data,
                        CALL_STACK_FROM_CALL(call));
   if (cq != NULL) {
+    GPR_ASSERT(
+        pollset_set_alternative == NULL &&
+        "Only one of 'cq' and 'pollset_set_alternative' should be non-NULL.");
     GRPC_CQ_INTERNAL_REF(cq, "bind");
-    grpc_call_stack_set_pollset(&exec_ctx, CALL_STACK_FROM_CALL(call),
-                                grpc_cq_pollset(cq));
+    call->pollent =
+        grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
+  }
+  if (pollset_set_alternative != NULL) {
+    call->pollent =
+        grpc_polling_entity_create_from_pollset_set(pollset_set_alternative);
+  }
+  if (!grpc_polling_entity_is_empty(&call->pollent)) {
+    grpc_call_stack_set_pollset_or_pollset_set(
+        &exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent);
   }
   if (parent_call != NULL) {
     GRPC_CALL_INTERNAL_REF(parent_call, "child");
@@ -343,10 +334,16 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
 void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call,
                                     grpc_completion_queue *cq) {
   GPR_ASSERT(cq);
+
+  if (grpc_polling_entity_pollset_set(&call->pollent) != NULL) {
+    gpr_log(GPR_ERROR, "A pollset_set is already registered for this call.");
+    abort();
+  }
   call->cq = cq;
   GRPC_CQ_INTERNAL_REF(cq, "bind");
-  grpc_call_stack_set_pollset(exec_ctx, CALL_STACK_FROM_CALL(call),
-                              grpc_cq_pollset(cq));
+  call->pollent = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
+  grpc_call_stack_set_pollset_or_pollset_set(
+      exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent);
 }
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
@@ -394,7 +391,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call,
     GRPC_CQ_INTERNAL_UNREF(c->cq, "bind");
   }
   grpc_channel *channel = c->channel;
-  grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), c);
+  grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->stats, c);
   GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call");
   GPR_TIMER_END("destroy_call", 0);
 }
@@ -409,21 +406,27 @@ static void set_status_code(grpc_call *call, status_source source,
   /* TODO(ctiller): what to do about the flush that was previously here */
 }
 
-static void set_compression_algorithm(grpc_call *call,
-                                      grpc_compression_algorithm algo) {
+static void set_incoming_compression_algorithm(
+    grpc_call *call, grpc_compression_algorithm algo) {
   GPR_ASSERT(algo < GRPC_COMPRESS_ALGORITHMS_COUNT);
-  call->compression_algorithm = algo;
+  call->incoming_compression_algorithm = algo;
 }
 
 grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
     grpc_call *call) {
   grpc_compression_algorithm algorithm;
   gpr_mu_lock(&call->mu);
-  algorithm = call->compression_algorithm;
+  algorithm = call->incoming_compression_algorithm;
   gpr_mu_unlock(&call->mu);
   return algorithm;
 }
 
+static grpc_compression_algorithm compression_algorithm_for_level_locked(
+    grpc_call *call, grpc_compression_level level) {
+  return grpc_compression_algorithm_for_level(level,
+                                              call->encodings_accepted_by_peer);
+}
+
 uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) {
   uint32_t flags;
   gpr_mu_lock(&call->mu);
@@ -549,15 +552,28 @@ static grpc_linked_mdelem *linked_from_md(grpc_metadata *md) {
   return (grpc_linked_mdelem *)&md->internal_data;
 }
 
+static grpc_metadata *get_md_elem(grpc_metadata *metadata,
+                                  grpc_metadata *additional_metadata, int i,
+                                  int count) {
+  grpc_metadata *res =
+      i < count ? &metadata[i] : &additional_metadata[i - count];
+  GPR_ASSERT(res);
+  return res;
+}
+
 static int prepare_application_metadata(grpc_call *call, int count,
                                         grpc_metadata *metadata,
                                         int is_trailing,
-                                        int prepend_extra_metadata) {
+                                        int prepend_extra_metadata,
+                                        grpc_metadata *additional_metadata,
+                                        int additional_metadata_count) {
+  int total_count = count + additional_metadata_count;
   int i;
   grpc_metadata_batch *batch =
       &call->metadata_batch[0 /* is_receiving */][is_trailing];
-  for (i = 0; i < count; i++) {
-    grpc_metadata *md = &metadata[i];
+  for (i = 0; i < total_count; i++) {
+    const grpc_metadata *md =
+        get_md_elem(metadata, additional_metadata, i, count);
     grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
     GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data));
     l->md = grpc_mdelem_from_string_and_buffer(
@@ -576,9 +592,10 @@ static int prepare_application_metadata(grpc_call *call, int count,
       break;
     }
   }
-  if (i != count) {
+  if (i != total_count) {
     for (int j = 0; j <= i; j++) {
-      grpc_metadata *md = &metadata[j];
+      const grpc_metadata *md =
+          get_md_elem(metadata, additional_metadata, j, count);
       grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
       GRPC_MDELEM_UNREF(l->md);
     }
@@ -599,24 +616,36 @@ static int prepare_application_metadata(grpc_call *call, int count,
       }
     }
   }
-  for (i = 1; i < count; i++) {
-    linked_from_md(&metadata[i])->prev = linked_from_md(&metadata[i - 1]);
+  for (i = 1; i < total_count; i++) {
+    grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
+    grpc_metadata *prev_md =
+        get_md_elem(metadata, additional_metadata, i - 1, count);
+    linked_from_md(md)->prev = linked_from_md(prev_md);
   }
-  for (i = 0; i < count - 1; i++) {
-    linked_from_md(&metadata[i])->next = linked_from_md(&metadata[i + 1]);
+  for (i = 0; i < total_count - 1; i++) {
+    grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
+    grpc_metadata *next_md =
+        get_md_elem(metadata, additional_metadata, i + 1, count);
+    linked_from_md(md)->next = linked_from_md(next_md);
   }
-  switch (prepend_extra_metadata * 2 + (count != 0)) {
+
+  switch (prepend_extra_metadata * 2 + (total_count != 0)) {
     case 0:
       /* no prepend, no metadata => nothing to do */
       batch->list.head = batch->list.tail = NULL;
       break;
-    case 1:
+    case 1: {
       /* metadata, but no prepend */
-      batch->list.head = linked_from_md(&metadata[0]);
-      batch->list.tail = linked_from_md(&metadata[count - 1]);
+      grpc_metadata *first_md =
+          get_md_elem(metadata, additional_metadata, 0, count);
+      grpc_metadata *last_md =
+          get_md_elem(metadata, additional_metadata, total_count - 1, count);
+      batch->list.head = linked_from_md(first_md);
+      batch->list.tail = linked_from_md(last_md);
       batch->list.head->prev = NULL;
       batch->list.tail->next = NULL;
       break;
+    }
     case 2:
       /* prepend, but no md */
       batch->list.head = &call->send_extra_metadata[0];
@@ -625,17 +654,22 @@ static int prepare_application_metadata(grpc_call *call, int count,
       batch->list.head->prev = NULL;
       batch->list.tail->next = NULL;
       break;
-    case 3:
+    case 3: {
       /* prepend AND md */
+      grpc_metadata *first_md =
+          get_md_elem(metadata, additional_metadata, 0, count);
+      grpc_metadata *last_md =
+          get_md_elem(metadata, additional_metadata, total_count - 1, count);
       batch->list.head = &call->send_extra_metadata[0];
       call->send_extra_metadata[call->send_extra_metadata_count - 1].next =
-          linked_from_md(&metadata[0]);
-      linked_from_md(&metadata[0])->prev =
+          linked_from_md(first_md);
+      linked_from_md(first_md)->prev =
           &call->send_extra_metadata[call->send_extra_metadata_count - 1];
-      batch->list.tail = linked_from_md(&metadata[count - 1]);
+      batch->list.tail = linked_from_md(last_md);
       batch->list.head->prev = NULL;
       batch->list.tail->next = NULL;
       break;
+    }
     default:
       GPR_UNREACHABLE_CODE(return 0);
   }
@@ -704,48 +738,103 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
   return r;
 }
 
-typedef struct cancel_closure {
+typedef struct termination_closure {
   grpc_closure closure;
   grpc_call *call;
   grpc_status_code status;
-} cancel_closure;
+  gpr_slice optional_message;
+  grpc_closure *op_closure;
+  enum { TC_CANCEL, TC_CLOSE } type;
+} termination_closure;
+
+static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp,
+                             grpc_error *error) {
+  termination_closure *tc = tcp;
+  switch (tc->type) {
+    case TC_CANCEL:
+      GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "cancel");
+      break;
+    case TC_CLOSE:
+      GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "close");
+      break;
+  }
+  gpr_slice_unref(tc->optional_message);
+  grpc_exec_ctx_sched(exec_ctx, tc->op_closure, GRPC_ERROR_NONE, NULL);
+  gpr_free(tc);
+}
 
-static void done_cancel(grpc_exec_ctx *exec_ctx, void *ccp, grpc_error *error) {
-  cancel_closure *cc = ccp;
-  GRPC_CALL_INTERNAL_UNREF(exec_ctx, cc->call, "cancel");
-  gpr_free(cc);
+static void send_cancel(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
+  grpc_transport_stream_op op;
+  termination_closure *tc = tcp;
+  memset(&op, 0, sizeof(op));
+  op.cancel_with_status = tc->status;
+  /* reuse closure to catch completion */
+  grpc_closure_init(&tc->closure, done_termination, tc);
+  op.on_complete = &tc->closure;
+  execute_op(exec_ctx, tc->call, &op);
 }
 
-static void send_cancel(grpc_exec_ctx *exec_ctx, void *ccp, grpc_error *error) {
+static void send_close(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
   grpc_transport_stream_op op;
-  cancel_closure *cc = ccp;
+  termination_closure *tc = tcp;
   memset(&op, 0, sizeof(op));
-  op.cancel_with_status = cc->status;
+  tc->optional_message = gpr_slice_ref(tc->optional_message);
+  grpc_transport_stream_op_add_close(&op, tc->status, &tc->optional_message);
   /* reuse closure to catch completion */
-  grpc_closure_init(&cc->closure, done_cancel, cc);
-  op.on_complete = &cc->closure;
-  execute_op(exec_ctx, cc->call, &op);
+  grpc_closure_init(&tc->closure, done_termination, tc);
+  tc->op_closure = op.on_complete;
+  op.on_complete = &tc->closure;
+  execute_op(exec_ctx, tc->call, &op);
+}
+
+static grpc_call_error terminate_with_status(grpc_exec_ctx *exec_ctx,
+                                             termination_closure *tc) {
+  grpc_mdstr *details = NULL;
+  if (GPR_SLICE_LENGTH(tc->optional_message) > 0) {
+    tc->optional_message = gpr_slice_ref(tc->optional_message);
+    details = grpc_mdstr_from_slice(tc->optional_message);
+  }
+
+  set_status_code(tc->call, STATUS_FROM_API_OVERRIDE, (uint32_t)tc->status);
+  set_status_details(tc->call, STATUS_FROM_API_OVERRIDE, details);
+
+  if (tc->type == TC_CANCEL) {
+    grpc_closure_init(&tc->closure, send_cancel, tc);
+    GRPC_CALL_INTERNAL_REF(tc->call, "cancel");
+  } else if (tc->type == TC_CLOSE) {
+    grpc_closure_init(&tc->closure, send_close, tc);
+    GRPC_CALL_INTERNAL_REF(tc->call, "close");
+  }
+  grpc_exec_ctx_sched(exec_ctx, &tc->closure, GRPC_ERROR_NONE, NULL);
+  return GRPC_CALL_OK;
 }
 
 static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
                                           grpc_status_code status,
                                           const char *description) {
-  grpc_mdstr *details =
-      description ? grpc_mdstr_from_string(description) : NULL;
-  cancel_closure *cc = gpr_malloc(sizeof(*cc));
-
+  termination_closure *tc = gpr_malloc(sizeof(*tc));
+  memset(tc, 0, sizeof(termination_closure));
+  tc->type = TC_CANCEL;
+  tc->call = c;
+  tc->optional_message = gpr_slice_from_copied_string(description);
   GPR_ASSERT(status != GRPC_STATUS_OK);
+  tc->status = status;
 
-  set_status_code(c, STATUS_FROM_API_OVERRIDE, (uint32_t)status);
-  set_status_details(c, STATUS_FROM_API_OVERRIDE, details);
+  return terminate_with_status(exec_ctx, tc);
+}
 
-  grpc_closure_init(&cc->closure, send_cancel, cc);
-  cc->call = c;
-  cc->status = status;
-  GRPC_CALL_INTERNAL_REF(c, "cancel");
-  grpc_exec_ctx_push(exec_ctx, &cc->closure, GRPC_ERROR_NONE, NULL);
+static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
+                                         grpc_status_code status,
+                                         const char *description) {
+  termination_closure *tc = gpr_malloc(sizeof(*tc));
+  memset(tc, 0, sizeof(termination_closure));
+  tc->type = TC_CLOSE;
+  tc->call = c;
+  tc->optional_message = gpr_slice_from_copied_string(description);
+  GPR_ASSERT(status != GRPC_STATUS_OK);
+  tc->status = status;
 
-  return GRPC_CALL_OK;
+  return terminate_with_status(exec_ctx, tc);
 }
 
 static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
@@ -886,9 +975,9 @@ static grpc_mdelem *recv_initial_filter(void *callp, grpc_mdelem *elem) {
   if (elem == NULL) {
     return NULL;
   } else if (elem->key == GRPC_MDSTR_GRPC_ENCODING) {
-    GPR_TIMER_BEGIN("compression_algorithm", 0);
-    set_compression_algorithm(call, decode_compression(elem));
-    GPR_TIMER_END("compression_algorithm", 0);
+    GPR_TIMER_BEGIN("incoming_compression_algorithm", 0);
+    set_incoming_compression_algorithm(call, decode_compression(elem));
+    GPR_TIMER_END("incoming_compression_algorithm", 0);
     return NULL;
   } else if (elem->key == GRPC_MDSTR_GRPC_ACCEPT_ENCODING) {
     GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0);
@@ -970,7 +1059,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,
   grpc_call *call = bctl->call;
   if (bctl->is_notify_tag_closure) {
     /* unrefs bctl->error */
-    grpc_exec_ctx_push(exec_ctx, bctl->notify_tag, bctl->error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, bctl->notify_tag, bctl->error, NULL);
     gpr_mu_lock(&call->mu);
     bctl->call->used_batches =
         (uint8_t)(bctl->call->used_batches &
@@ -1020,6 +1109,9 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
                          call->receiving_slice);
     continue_receiving_slices(exec_ctx, bctl);
   } else {
+    if (grpc_trace_operation_failures) {
+      GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
+    }
     grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
     call->receiving_stream = NULL;
     grpc_byte_buffer_destroy(*call->receiving_buffer);
@@ -1053,9 +1145,9 @@ static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl,
   } else {
     call->test_only_last_message_flags = call->receiving_stream->flags;
     if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) &&
-        (call->compression_algorithm > GRPC_COMPRESS_NONE)) {
+        (call->incoming_compression_algorithm > GRPC_COMPRESS_NONE)) {
       *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create(
-          NULL, 0, call->compression_algorithm);
+          NULL, 0, call->incoming_compression_algorithm);
     } else {
       *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0);
     }
@@ -1083,6 +1175,56 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
   }
 }
 
+static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
+                                       batch_control *bctl) {
+  grpc_call *call = bctl->call;
+  /* validate call->incoming_compression_algorithm */
+  if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) {
+    const grpc_compression_algorithm algo =
+        call->incoming_compression_algorithm;
+    char *error_msg = NULL;
+    const grpc_compression_options compression_options =
+        grpc_channel_compression_options(call->channel);
+    /* check if algorithm is known */
+    if (algo >= GRPC_COMPRESS_ALGORITHMS_COUNT) {
+      gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.",
+                   algo);
+      gpr_log(GPR_ERROR, "%s", error_msg);
+      close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg);
+    } else if (grpc_compression_options_is_algorithm_enabled(
+                   &compression_options, algo) == 0) {
+      /* check if algorithm is supported by current channel config */
+      char *algo_name;
+      grpc_compression_algorithm_name(algo, &algo_name);
+      gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.",
+                   algo_name);
+      gpr_log(GPR_ERROR, "%s", error_msg);
+      close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg);
+    } else {
+      call->incoming_compression_algorithm = algo;
+    }
+    gpr_free(error_msg);
+  }
+
+  /* make sure the received grpc-encoding is amongst the ones listed in
+   * grpc-accept-encoding */
+  GPR_ASSERT(call->encodings_accepted_by_peer != 0);
+  if (!GPR_BITGET(call->encodings_accepted_by_peer,
+                  call->incoming_compression_algorithm)) {
+    extern int grpc_compression_trace;
+    if (grpc_compression_trace) {
+      char *algo_name;
+      grpc_compression_algorithm_name(call->incoming_compression_algorithm,
+                                      &algo_name);
+      gpr_log(GPR_ERROR,
+              "Compression algorithm (grpc-encoding = '%s') not present in "
+              "the bitset of accepted encodings (grpc-accept-encodings: "
+              "'0x%x')",
+              algo_name, call->encodings_accepted_by_peer);
+    }
+  }
+}
+
 static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
                                              void *bctlp, grpc_error *error) {
   batch_control *bctl = bctlp;
@@ -1097,24 +1239,10 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
         &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
     grpc_metadata_batch_filter(md, recv_initial_filter, call);
 
-    /* make sure the received grpc-encoding is amongst the ones listed in
-     * grpc-accept-encoding */
-
-    GPR_ASSERT(call->encodings_accepted_by_peer != 0);
-    if (!GPR_BITGET(call->encodings_accepted_by_peer,
-                    call->compression_algorithm)) {
-      extern int grpc_compression_trace;
-      if (grpc_compression_trace) {
-        char *algo_name;
-        grpc_compression_algorithm_name(call->compression_algorithm,
-                                        &algo_name);
-        gpr_log(GPR_ERROR,
-                "Compression algorithm (grpc-encoding = '%s') not present in "
-                "the bitset of accepted encodings (grpc-accept-encodings: "
-                "'0x%x')",
-                algo_name, call->encodings_accepted_by_peer);
-      }
-    }
+    GPR_TIMER_BEGIN("validate_filtered_metadata", 0);
+    validate_filtered_metadata(exec_ctx, bctl);
+    GPR_TIMER_END("validate_filtered_metadata", 0);
+
     if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) !=
             0 &&
         !call->is_client) {
@@ -1129,7 +1257,7 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
     grpc_closure *saved_rsr_closure = grpc_closure_create(
         receiving_stream_ready, call->saved_receiving_stream_ready_bctlp);
     call->saved_receiving_stream_ready_bctlp = NULL;
-    grpc_exec_ctx_push(exec_ctx, saved_rsr_closure, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, saved_rsr_closure, error, NULL);
   }
 
   gpr_mu_unlock(&call->mu);
@@ -1264,7 +1392,40 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
           goto done_with_error;
         }
-        if (op->data.send_initial_metadata.count > INT_MAX) {
+        /* process compression level */
+        grpc_metadata compression_md;
+        memset(&compression_md, 0, sizeof(grpc_metadata));
+        size_t additional_metadata_count = 0;
+        grpc_compression_level effective_compression_level;
+        bool level_set = false;
+        if (op->data.send_initial_metadata.maybe_compression_level.is_set) {
+          effective_compression_level =
+              op->data.send_initial_metadata.maybe_compression_level.level;
+          level_set = true;
+        } else {
+          const grpc_compression_options copts =
+              grpc_channel_compression_options(call->channel);
+          level_set = copts.default_level.is_set;
+          if (level_set) {
+            effective_compression_level = copts.default_level.level;
+          }
+        }
+        if (level_set && !call->is_client) {
+          const grpc_compression_algorithm calgo =
+              compression_algorithm_for_level_locked(
+                  call, effective_compression_level);
+          char *calgo_name;
+          grpc_compression_algorithm_name(calgo, &calgo_name);
+          // the following will be picked up by the compress filter and used as
+          // the call's compression algorithm.
+          compression_md.key = GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY;
+          compression_md.value = calgo_name;
+          compression_md.value_length = strlen(calgo_name);
+          additional_metadata_count++;
+        }
+
+        if (op->data.send_initial_metadata.count + additional_metadata_count >
+            INT_MAX) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1272,7 +1433,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         call->sent_initial_metadata = 1;
         if (!prepare_application_metadata(
                 call, (int)op->data.send_initial_metadata.count,
-                op->data.send_initial_metadata.metadata, 0, call->is_client)) {
+                op->data.send_initial_metadata.metadata, 0, call->is_client,
+                &compression_md, (int)additional_metadata_count)) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1360,7 +1522,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         if (!prepare_application_metadata(
                 call,
                 (int)op->data.send_status_from_server.trailing_metadata_count,
-                op->data.send_status_from_server.trailing_metadata, 1, 1)) {
+                op->data.send_status_from_server.trailing_metadata, 1, 1, NULL,
+                0)) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1432,7 +1595,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         bctl->recv_final_op = 1;
         stream_op.recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
-        stream_op.collect_stats = &call->stats;
+        stream_op.collect_stats = &call->stats.transport_stream_stats;
         break;
       case GRPC_OP_RECV_CLOSE_ON_SERVER:
         /* Flag validation: currently allow no flags */
@@ -1454,7 +1617,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         bctl->recv_final_op = 1;
         stream_op.recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
-        stream_op.collect_stats = &call->stats;
+        stream_op.collect_stats = &call->stats.transport_stream_stats;
         break;
     }
   }
@@ -1549,9 +1712,10 @@ uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; }
 grpc_compression_algorithm grpc_call_compression_for_level(
     grpc_call *call, grpc_compression_level level) {
   gpr_mu_lock(&call->mu);
-  const uint32_t accepted_encodings = call->encodings_accepted_by_peer;
+  grpc_compression_algorithm algo =
+      compression_algorithm_for_level_locked(call, level);
   gpr_mu_unlock(&call->mu);
-  return grpc_compression_algorithm_for_level(level, accepted_encodings);
+  return algo;
 }
 
 const char *grpc_call_error_to_string(grpc_call_error error) {
diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h
index 2725e060b8..b640345c21 100644
--- a/src/core/lib/surface/call.h
+++ b/src/core/lib/surface/call.h
@@ -53,6 +53,8 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx,
 grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
                             uint32_t propagation_mask,
                             grpc_completion_queue *cq,
+                            /* if not NULL, it'll be used in lieu of \a cq */
+                            grpc_pollset_set *pollset_set_alternative,
                             const void *server_transport_data,
                             grpc_mdelem **add_initial_metadata,
                             size_t add_initial_metadata_count,
diff --git a/src/core/lib/surface/call_log_batch.c b/src/core/lib/surface/call_log_batch.c
index a6d1d5149f..31c074f15d 100644
--- a/src/core/lib/surface/call_log_batch.c
+++ b/src/core/lib/surface/call_log_batch.c
@@ -112,7 +112,7 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity,
   size_t i;
   for (i = 0; i < nops; i++) {
     tmp = grpc_op_string(&ops[i]);
-    gpr_log(file, line, severity, "ops[%d]: %s", i, tmp);
+    gpr_log(file, line, severity, "ops[%" PRIuPTR "]: %s", i, tmp);
     gpr_free(tmp);
   }
 }
diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c
index 3c68dc0c4b..8936e7164f 100644
--- a/src/core/lib/surface/channel.c
+++ b/src/core/lib/surface/channel.c
@@ -36,16 +36,17 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <grpc/compression.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel_init.h"
-#include "src/core/lib/surface/init.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 /** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS.
@@ -64,10 +65,12 @@ typedef struct registered_call {
 struct grpc_channel {
   int is_client;
   uint32_t max_message_length;
+  grpc_compression_options compression_options;
   grpc_mdelem *default_authority;
 
   gpr_mu registered_call_mu;
   registered_call *registered_calls;
+
   char *target;
 };
 
@@ -112,6 +115,7 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
   channel->registered_calls = NULL;
 
   channel->max_message_length = DEFAULT_MAX_MESSAGE_LENGTH;
+  grpc_compression_options_init(&channel->compression_options);
   if (args) {
     for (size_t i = 0; i < args->num_args; i++) {
       if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_MESSAGE_LENGTH)) {
@@ -152,6 +156,27 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
                 ":authority", args->args[i].value.string);
           }
         }
+      } else if (0 == strcmp(args->args[i].key,
+                             GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) {
+        channel->compression_options.default_level.is_set = true;
+        GPR_ASSERT(args->args[i].value.integer >= 0 &&
+                   args->args[i].value.integer < GRPC_COMPRESS_LEVEL_COUNT);
+        channel->compression_options.default_level.level =
+            (grpc_compression_level)args->args[i].value.integer;
+      } else if (0 == strcmp(args->args[i].key,
+                             GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) {
+        channel->compression_options.default_algorithm.is_set = true;
+        GPR_ASSERT(args->args[i].value.integer >= 0 &&
+                   args->args[i].value.integer <
+                       GRPC_COMPRESS_ALGORITHMS_COUNT);
+        channel->compression_options.default_algorithm.algorithm =
+            (grpc_compression_algorithm)args->args[i].value.integer;
+      } else if (0 ==
+                 strcmp(args->args[i].key,
+                        GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) {
+        channel->compression_options.enabled_algorithms_bitset =
+            (uint32_t)args->args[i].value.integer |
+            0x1; /* always support no compression */
       }
     }
     grpc_channel_args_destroy(args);
@@ -167,12 +192,14 @@ char *grpc_channel_get_target(grpc_channel *channel) {
 
 static grpc_call *grpc_channel_create_call_internal(
     grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
-    grpc_completion_queue *cq, grpc_mdelem *path_mdelem,
-    grpc_mdelem *authority_mdelem, gpr_timespec deadline) {
+    grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative,
+    grpc_mdelem *path_mdelem, grpc_mdelem *authority_mdelem,
+    gpr_timespec deadline) {
   grpc_mdelem *send_metadata[2];
   size_t num_metadata = 0;
 
   GPR_ASSERT(channel->is_client);
+  GPR_ASSERT(!(cq != NULL && pollset_set_alternative != NULL));
 
   send_metadata[num_metadata++] = path_mdelem;
   if (authority_mdelem != NULL) {
@@ -181,8 +208,9 @@ static grpc_call *grpc_channel_create_call_internal(
     send_metadata[num_metadata++] = GRPC_MDELEM_REF(channel->default_authority);
   }
 
-  return grpc_call_create(channel, parent_call, propagation_mask, cq, NULL,
-                          send_metadata, num_metadata, deadline);
+  return grpc_call_create(channel, parent_call, propagation_mask, cq,
+                          pollset_set_alternative, NULL, send_metadata,
+                          num_metadata, deadline);
 }
 
 grpc_call *grpc_channel_create_call(grpc_channel *channel,
@@ -202,7 +230,22 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel,
            (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
-      channel, parent_call, propagation_mask, cq,
+      channel, parent_call, propagation_mask, cq, NULL,
+      grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH,
+                                        grpc_mdstr_from_string(method)),
+      host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY,
+                                               grpc_mdstr_from_string(host))
+           : NULL,
+      deadline);
+}
+
+grpc_call *grpc_channel_create_pollset_set_call(
+    grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+    grpc_pollset_set *pollset_set, const char *method, const char *host,
+    gpr_timespec deadline, void *reserved) {
+  GPR_ASSERT(!reserved);
+  return grpc_channel_create_call_internal(
+      channel, parent_call, propagation_mask, NULL, pollset_set,
       grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH,
                                         grpc_mdstr_from_string(method)),
       host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY,
@@ -246,7 +289,7 @@ grpc_call *grpc_channel_create_registered_call(
           (int)deadline.tv_nsec, (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
-      channel, parent_call, propagation_mask, completion_queue,
+      channel, parent_call, propagation_mask, completion_queue, NULL,
       GRPC_MDELEM_REF(rc->path),
       rc->authority ? GRPC_MDELEM_REF(rc->authority) : NULL, deadline);
 }
@@ -307,6 +350,11 @@ grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel) {
   return CHANNEL_STACK_FROM_CHANNEL(channel);
 }
 
+grpc_compression_options grpc_channel_compression_options(
+    const grpc_channel *channel) {
+  return channel->compression_options;
+}
+
 grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) {
   char tmp[GPR_LTOA_MIN_BUFSIZE];
   switch (i) {
diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h
index 22dae930e4..7eff7b8883 100644
--- a/src/core/lib/surface/channel.h
+++ b/src/core/lib/surface/channel.h
@@ -42,6 +42,11 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
                                   grpc_channel_stack_type channel_stack_type,
                                   grpc_transport *optional_transport);
 
+grpc_call *grpc_channel_create_pollset_set_call(
+    grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+    grpc_pollset_set *pollset_set, const char *method, const char *host,
+    gpr_timespec deadline, void *reserved);
+
 /** Get a (borrowed) pointer to this channels underlying channel stack */
 grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel);
 
@@ -71,4 +76,8 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
   grpc_channel_internal_unref(exec_ctx, channel)
 #endif
 
+/** Return the channel's compression options. */
+grpc_compression_options grpc_channel_compression_options(
+    const grpc_channel *channel);
+
 #endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_H */
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index ecc127876b..2cc6aa74e0 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -50,6 +50,8 @@
 #include "src/core/lib/surface/event_string.h"
 #include "src/core/lib/surface/surface_trace.h"
 
+int grpc_trace_operation_failures;
+
 typedef struct {
   grpc_pollset_worker **worker;
   void *tag;
@@ -231,12 +233,16 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
 #endif
 
   GPR_TIMER_BEGIN("grpc_cq_end_op", 0);
-  if (grpc_api_trace) {
+  if (grpc_api_trace ||
+      (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) {
     const char *errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
         "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, "
         "done_arg=%p, storage=%p)",
         7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage));
+    if (grpc_trace_operation_failures) {
+      gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
+    }
     grpc_error_free_string(errmsg);
   }
 
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index a866858cb5..b9dd1092b6 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -39,6 +39,8 @@
 #include <grpc/grpc.h>
 #include "src/core/lib/iomgr/pollset.h"
 
+extern int grpc_trace_operation_failures;
+
 typedef struct grpc_cq_completion {
   /** user supplied tag */
   void *tag;
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
index 1c8b709015..f07039cb94 100644
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@@ -165,6 +165,7 @@ void grpc_init(void) {
                          &grpc_trace_channel_stack_builder);
     grpc_register_tracer("http1", &grpc_http1_trace);
     grpc_register_tracer("compression", &grpc_compression_trace);
+    grpc_register_tracer("op_failure", &grpc_trace_operation_failures);
     grpc_security_pre_init();
     grpc_iomgr_init();
     grpc_executor_init();
diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.c
index 5c15b676f8..5ea4cba5d1 100644
--- a/src/core/lib/surface/lame_client.c
+++ b/src/core/lib/surface/lame_client.c
@@ -92,17 +92,17 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
                                     grpc_channel_element *elem,
                                     grpc_transport_op *op) {
   if (op->on_connectivity_state_change) {
-    GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE);
-    *op->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE;
-    grpc_exec_ctx_push(exec_ctx, op->on_connectivity_state_change,
-                       GRPC_ERROR_NONE, NULL);
+    GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_SHUTDOWN);
+    *op->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+    grpc_exec_ctx_sched(exec_ctx, op->on_connectivity_state_change,
+                        GRPC_ERROR_NONE, NULL);
   }
   if (op->on_consumed != NULL) {
-    grpc_exec_ctx_push(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
   }
   if (op->send_ping != NULL) {
-    grpc_exec_ctx_push(exec_ctx, op->send_ping,
-                       GRPC_ERROR_CREATE("lame client channel"), NULL);
+    grpc_exec_ctx_sched(exec_ctx, op->send_ping,
+                        GRPC_ERROR_CREATE("lame client channel"), NULL);
   }
   GRPC_ERROR_UNREF(op->disconnect_with_error);
 }
@@ -111,6 +111,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                            grpc_call_element_args *args) {}
 
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats,
                               void *and_free_memory) {
   gpr_free(and_free_memory);
 }
@@ -130,7 +131,7 @@ const grpc_channel_filter grpc_lame_filter = {
     lame_start_transport_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index f261321750..def6e5068b 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -344,8 +344,8 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx,
     grpc_closure_init(
         &calld->kill_zombie_closure, kill_zombie,
         grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
-    grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
+                        NULL);
   }
 }
 
@@ -526,7 +526,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_closure_init(
         &calld->kill_zombie_closure, kill_zombie,
         grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
-    grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, error, NULL);
     return;
   }
 
@@ -571,8 +571,8 @@ static void finish_start_new_rpc(
     calld->state = ZOMBIED;
     gpr_mu_unlock(&calld->mu_state);
     grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
-    grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
-                       NULL);
+    grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
+                        NULL);
     return;
   }
 
@@ -757,8 +757,8 @@ static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
         GRPC_ERROR_CREATE_REFERENCING("Missing :authority or :path", &error, 1);
   }
 
-  grpc_exec_ctx_push(exec_ctx, calld->on_done_recv_initial_metadata, error,
-                     NULL);
+  grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv_initial_metadata, error,
+                      NULL);
 }
 
 static void server_mutate_op(grpc_call_element *elem,
@@ -794,8 +794,8 @@ static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
       calld->state = ZOMBIED;
       gpr_mu_unlock(&calld->mu_state);
       grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
-      grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
-                         NULL);
+      grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure,
+                          GRPC_ERROR_NONE, NULL);
     } else if (calld->state == PENDING) {
       calld->state = ZOMBIED;
       gpr_mu_unlock(&calld->mu_state);
@@ -812,9 +812,9 @@ static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd,
                           const void *transport_server_data) {
   channel_data *chand = cd;
   /* create a call */
-  grpc_call *call =
-      grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data,
-                       NULL, 0, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+  grpc_call *call = grpc_call_create(chand->channel, NULL, 0, NULL, NULL,
+                                     transport_server_data, NULL, 0,
+                                     gpr_inf_future(GPR_CLOCK_MONOTONIC));
   grpc_call_element *elem =
       grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
   call_data *calld = elem->call_data;
@@ -831,7 +831,7 @@ static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd,
                                          grpc_error *error) {
   channel_data *chand = cd;
   grpc_server *server = chand->server;
-  if (chand->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) {
+  if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
     grpc_transport_op op;
     memset(&op, 0, sizeof(op));
     op.on_connectivity_state_change = &chand->channel_connectivity_changed,
@@ -864,7 +864,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {
+                              const grpc_call_stats *stats, void *ignored) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
 
@@ -929,7 +929,7 @@ const grpc_channel_filter grpc_server_top_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
@@ -1339,8 +1339,8 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
         grpc_closure_init(
             &calld->kill_zombie_closure, kill_zombie,
             grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
-        grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure,
-                           GRPC_ERROR_NONE, NULL);
+        grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure,
+                            GRPC_ERROR_NONE, NULL);
       } else {
         GPR_ASSERT(calld->state == PENDING);
         calld->state = ACTIVATED;
diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c
index 3286af9fb2..054f112127 100644
--- a/src/core/lib/transport/connectivity_state.c
+++ b/src/core/lib/transport/connectivity_state.c
@@ -51,8 +51,8 @@ const char *grpc_connectivity_state_name(grpc_connectivity_state state) {
       return "READY";
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
       return "TRANSIENT_FAILURE";
-    case GRPC_CHANNEL_FATAL_FAILURE:
-      return "FATAL_FAILURE";
+    case GRPC_CHANNEL_SHUTDOWN:
+      return "SHUTDOWN";
   }
   GPR_UNREACHABLE_CODE(return "UNKNOWN");
 }
@@ -73,13 +73,13 @@ void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx,
   while ((w = tracker->watchers)) {
     tracker->watchers = w->next;
 
-    if (GRPC_CHANNEL_FATAL_FAILURE != *w->current) {
-      *w->current = GRPC_CHANNEL_FATAL_FAILURE;
+    if (GRPC_CHANNEL_SHUTDOWN != *w->current) {
+      *w->current = GRPC_CHANNEL_SHUTDOWN;
       error = GRPC_ERROR_NONE;
     } else {
       error = GRPC_ERROR_CREATE("Shutdown connectivity owner");
     }
-    grpc_exec_ctx_push(exec_ctx, w->notify, error, NULL);
+    grpc_exec_ctx_sched(exec_ctx, w->notify, error, NULL);
     gpr_free(w);
   }
   GRPC_ERROR_UNREF(tracker->current_error);
@@ -114,7 +114,7 @@ int grpc_connectivity_state_notify_on_state_change(
   if (current == NULL) {
     grpc_connectivity_state_watcher *w = tracker->watchers;
     if (w != NULL && w->notify == notify) {
-      grpc_exec_ctx_push(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
+      grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
       tracker->watchers = w->next;
       gpr_free(w);
       return 0;
@@ -122,7 +122,7 @@ int grpc_connectivity_state_notify_on_state_change(
     while (w != NULL) {
       grpc_connectivity_state_watcher *rm_candidate = w->next;
       if (rm_candidate != NULL && rm_candidate->notify == notify) {
-        grpc_exec_ctx_push(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
+        grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
         w->next = w->next->next;
         gpr_free(rm_candidate);
         return 0;
@@ -133,8 +133,8 @@ int grpc_connectivity_state_notify_on_state_change(
   } else {
     if (tracker->current_state != *current) {
       *current = tracker->current_state;
-      grpc_exec_ctx_push(exec_ctx, notify,
-                         GRPC_ERROR_REF(tracker->current_error), NULL);
+      grpc_exec_ctx_sched(exec_ctx, notify,
+                          GRPC_ERROR_REF(tracker->current_error), NULL);
     } else {
       grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w));
       w->current = current;
@@ -164,7 +164,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
     case GRPC_CHANNEL_READY:
       GPR_ASSERT(error == GRPC_ERROR_NONE);
       break;
-    case GRPC_CHANNEL_FATAL_FAILURE:
+    case GRPC_CHANNEL_SHUTDOWN:
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
       GPR_ASSERT(error != GRPC_ERROR_NONE);
       break;
@@ -174,13 +174,13 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
   if (tracker->current_state == state) {
     return;
   }
-  GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_FATAL_FAILURE);
+  GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_SHUTDOWN);
   tracker->current_state = state;
   while ((w = tracker->watchers) != NULL) {
     *w->current = tracker->current_state;
     tracker->watchers = w->next;
-    grpc_exec_ctx_push(exec_ctx, w->notify,
-                       GRPC_ERROR_REF(tracker->current_error), NULL);
+    grpc_exec_ctx_sched(exec_ctx, w->notify,
+                        GRPC_ERROR_REF(tracker->current_error), NULL);
     gpr_free(w);
   }
 }
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 82c8e239f6..0677f29766 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -129,7 +129,10 @@ typedef struct mdtab_shard {
   internal_metadata **elems;
   size_t count;
   size_t capacity;
-  size_t free;
+  /** Estimate of the number of unreferenced mdelems in the hash table.
+      This will eventually converge to the exact number, but it's instantaneous
+      accuracy is not guaranteed */
+  gpr_atm free_estimate;
 } mdtab_shard;
 
 #define LOG2_STRTAB_SHARD_COUNT 5
@@ -217,7 +220,7 @@ void grpc_mdctx_global_init(void) {
     mdtab_shard *shard = &g_mdtab_shard[i];
     gpr_mu_init(&shard->mu);
     shard->count = 0;
-    shard->free = 0;
+    gpr_atm_no_barrier_store(&shard->free_estimate, 0);
     shard->capacity = INITIAL_MDTAB_CAPACITY;
     shard->elems = gpr_malloc(sizeof(*shard->elems) * shard->capacity);
     memset(shard->elems, 0, sizeof(*shard->elems) * shard->capacity);
@@ -232,7 +235,7 @@ void grpc_mdctx_global_shutdown(void) {
     gc_mdtab(shard);
     /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
     if (shard->count != 0) {
-      gpr_log(GPR_DEBUG, "WARNING: %d metadata elements were leaked",
+      gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata elements were leaked",
               shard->count);
       if (grpc_iomgr_abort_on_leaks()) {
         abort();
@@ -245,7 +248,7 @@ void grpc_mdctx_global_shutdown(void) {
     gpr_mu_destroy(&shard->mu);
     /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
     if (shard->count != 0) {
-      gpr_log(GPR_DEBUG, "WARNING: %d metadata strings were leaked",
+      gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked",
               shard->count);
       for (size_t j = 0; j < shard->capacity; j++) {
         for (internal_string *s = shard->strs[j]; s; s = s->bucket_next) {
@@ -281,10 +284,8 @@ static void ref_md_locked(mdtab_shard *shard,
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->value));
 #endif
-  if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 2)) {
-    shard->free--;
-  } else {
-    GPR_ASSERT(1 != gpr_atm_no_barrier_fetch_add(&md->refcnt, -1));
+  if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 1)) {
+    gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -1);
   }
 }
 
@@ -447,6 +448,7 @@ static void gc_mdtab(mdtab_shard *shard) {
   size_t i;
   internal_metadata **prev_next;
   internal_metadata *md, *next;
+  gpr_atm num_freed = 0;
 
   GPR_TIMER_BEGIN("gc_mdtab", 0);
   for (i = 0; i < shard->capacity; i++) {
@@ -463,13 +465,14 @@ static void gc_mdtab(mdtab_shard *shard) {
         }
         gpr_free(md);
         *prev_next = next;
-        shard->free--;
+        num_freed++;
         shard->count--;
       } else {
         prev_next = &md->bucket_next;
       }
     }
   }
+  gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -num_freed);
   GPR_TIMER_END("gc_mdtab", 0);
 }
 
@@ -504,7 +507,8 @@ static void grow_mdtab(mdtab_shard *shard) {
 }
 
 static void rehash_mdtab(mdtab_shard *shard) {
-  if (shard->free > shard->capacity / 4) {
+  if (gpr_atm_no_barrier_load(&shard->free_estimate) >
+      (gpr_atm)(shard->capacity / 4)) {
     gc_mdtab(shard);
   } else {
     grow_mdtab(shard);
@@ -553,7 +557,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdstr *mkey,
 
   /* not found: create a new pair */
   md = gpr_malloc(sizeof(internal_metadata));
-  gpr_atm_rel_store(&md->refcnt, 2);
+  gpr_atm_rel_store(&md->refcnt, 1);
   md->key = key;
   md->value = value;
   md->user_data = 0;
@@ -645,7 +649,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) {
      this function - meaning that no adjustment to mdtab_free is necessary,
      simplifying the logic here to be just an atomic increment */
   /* use C assert to have this removed in opt builds */
-  assert(gpr_atm_no_barrier_load(&md->refcnt) >= 2);
+  assert(gpr_atm_no_barrier_load(&md->refcnt) >= 1);
   gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
   return gmd;
 }
@@ -662,18 +666,13 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) {
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->value));
 #endif
-  if (2 == gpr_atm_full_fetch_add(&md->refcnt, -1)) {
-    uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash);
+  uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash);
+  if (1 == gpr_atm_full_fetch_add(&md->refcnt, -1)) {
+    /* once the refcount hits zero, some other thread can come along and
+       free md at any time: it's unsafe from this point on to access it */
     mdtab_shard *shard =
         &g_mdtab_shard[SHARD_IDX(hash, LOG2_MDTAB_SHARD_COUNT)];
-    GPR_TIMER_BEGIN("grpc_mdelem_unref.to_zero", 0);
-    gpr_mu_lock(&shard->mu);
-    if (1 == gpr_atm_no_barrier_load(&md->refcnt)) {
-      shard->free++;
-      gpr_atm_no_barrier_store(&md->refcnt, 0);
-    }
-    gpr_mu_unlock(&shard->mu);
-    GPR_TIMER_END("grpc_mdelem_unref.to_zero", 0);
+    gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1);
   }
 }
 
diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c
index 73b0041fd4..c5f16e530d 100644
--- a/src/core/lib/transport/static_metadata.c
+++ b/src/core/lib/transport/static_metadata.c
@@ -48,7 +48,7 @@ uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 4, 8, 6, 2, 4, 8, 6, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
     {11, 35, 10, 35, 12, 35, 12, 49, 13, 35, 14, 35, 15, 35, 16, 35, 17, 35,
@@ -56,10 +56,10 @@ const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
      30, 18, 30, 35, 31, 35, 32, 35, 36, 35, 37, 35, 38, 35, 39, 35, 42, 33,
      42, 34, 42, 48, 42, 53, 42, 54, 42, 55, 42, 56, 43, 33, 43, 48, 43, 53,
      46, 0,  46, 1,  46, 2,  50, 35, 57, 35, 58, 35, 59, 35, 60, 35, 61, 35,
-     62, 35, 63, 35, 64, 35, 65, 35, 66, 40, 66, 68, 66, 71, 67, 79, 67, 80,
-     69, 35, 70, 35, 72, 35, 73, 35, 74, 35, 75, 35, 76, 41, 76, 51, 76, 52,
-     77, 35, 78, 35, 81, 3,  81, 4,  81, 5,  81, 6,  81, 7,  81, 8,  81, 9,
-     82, 35, 83, 84, 85, 35, 86, 35, 87, 35, 88, 35, 89, 35};
+     62, 35, 63, 35, 64, 35, 65, 35, 66, 35, 67, 40, 67, 69, 67, 72, 68, 80,
+     68, 81, 70, 35, 71, 35, 73, 35, 74, 35, 75, 35, 76, 35, 77, 41, 77, 51,
+     77, 52, 78, 35, 79, 35, 82, 3,  82, 4,  82, 5,  82, 6,  82, 7,  82, 8,
+     82, 9,  83, 35, 84, 85, 86, 35, 87, 35, 88, 35, 89, 35, 90, 35};
 
 const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "0",
@@ -126,6 +126,7 @@ const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "if-unmodified-since",
     "last-modified",
     "link",
+    "load-reporting",
     "location",
     "max-forwards",
     ":method",
diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h
index f9d8bcdc8f..5ff0d2f3bc 100644
--- a/src/core/lib/transport/static_metadata.h
+++ b/src/core/lib/transport/static_metadata.h
@@ -44,7 +44,7 @@
 
 #include "src/core/lib/transport/metadata.h"
 
-#define GRPC_STATIC_MDSTR_COUNT 90
+#define GRPC_STATIC_MDSTR_COUNT 91
 extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 /* "0" */
 #define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0])
@@ -175,60 +175,62 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[62])
 /* "link" */
 #define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[63])
+/* "load-reporting" */
+#define GRPC_MDSTR_LOAD_REPORTING (&grpc_static_mdstr_table[64])
 /* "location" */
-#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[64])
+#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[65])
 /* "max-forwards" */
-#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[65])
+#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[66])
 /* ":method" */
-#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[66])
+#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[67])
 /* ":path" */
-#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[67])
+#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[68])
 /* "POST" */
-#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[68])
+#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[69])
 /* "proxy-authenticate" */
-#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[69])
+#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[70])
 /* "proxy-authorization" */
-#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[70])
+#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[71])
 /* "PUT" */
-#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[71])
+#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[72])
 /* "range" */
-#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[72])
+#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[73])
 /* "referer" */
-#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[73])
+#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[74])
 /* "refresh" */
-#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[74])
+#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[75])
 /* "retry-after" */
-#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[75])
+#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[76])
 /* ":scheme" */
-#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[76])
+#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[77])
 /* "server" */
-#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[77])
+#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[78])
 /* "set-cookie" */
-#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[78])
+#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[79])
 /* "/" */
-#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[79])
+#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[80])
 /* "/index.html" */
-#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[80])
+#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[81])
 /* ":status" */
-#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[81])
+#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[82])
 /* "strict-transport-security" */
-#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[82])
+#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[83])
 /* "te" */
-#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[83])
+#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[84])
 /* "trailers" */
-#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[84])
+#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[85])
 /* "transfer-encoding" */
-#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[85])
+#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[86])
 /* "user-agent" */
-#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[86])
+#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[87])
 /* "vary" */
-#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[87])
+#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[88])
 /* "via" */
-#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[88])
+#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[89])
 /* "www-authenticate" */
-#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[89])
+#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[90])
 
-#define GRPC_STATIC_MDELEM_COUNT 79
+#define GRPC_STATIC_MDELEM_COUNT 80
 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 /* "accept-charset": "" */
@@ -333,71 +335,73 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 #define GRPC_MDELEM_LAST_MODIFIED_EMPTY (&grpc_static_mdelem_table[45])
 /* "link": "" */
 #define GRPC_MDELEM_LINK_EMPTY (&grpc_static_mdelem_table[46])
+/* "load-reporting": "" */
+#define GRPC_MDELEM_LOAD_REPORTING_EMPTY (&grpc_static_mdelem_table[47])
 /* "location": "" */
-#define GRPC_MDELEM_LOCATION_EMPTY (&grpc_static_mdelem_table[47])
+#define GRPC_MDELEM_LOCATION_EMPTY (&grpc_static_mdelem_table[48])
 /* "max-forwards": "" */
-#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (&grpc_static_mdelem_table[48])
+#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (&grpc_static_mdelem_table[49])
 /* ":method": "GET" */
-#define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[49])
+#define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[50])
 /* ":method": "POST" */
-#define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[50])
+#define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[51])
 /* ":method": "PUT" */
-#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[51])
+#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[52])
 /* ":path": "/" */
-#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[52])
+#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[53])
 /* ":path": "/index.html" */
-#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[53])
+#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[54])
 /* "proxy-authenticate": "" */
-#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[54])
+#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[55])
 /* "proxy-authorization": "" */
-#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[55])
+#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[56])
 /* "range": "" */
-#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[56])
+#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[57])
 /* "referer": "" */
-#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[57])
+#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[58])
 /* "refresh": "" */
-#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[58])
+#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[59])
 /* "retry-after": "" */
-#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[59])
+#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[60])
 /* ":scheme": "grpc" */
-#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[60])
+#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[61])
 /* ":scheme": "http" */
-#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[61])
+#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[62])
 /* ":scheme": "https" */
-#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[62])
+#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[63])
 /* "server": "" */
-#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[63])
+#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[64])
 /* "set-cookie": "" */
-#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[64])
+#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[65])
 /* ":status": "200" */
-#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[65])
+#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[66])
 /* ":status": "204" */
-#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[66])
+#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[67])
 /* ":status": "206" */
-#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[67])
+#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[68])
 /* ":status": "304" */
-#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[68])
+#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[69])
 /* ":status": "400" */
-#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[69])
+#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[70])
 /* ":status": "404" */
-#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[70])
+#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[71])
 /* ":status": "500" */
-#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[71])
+#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[72])
 /* "strict-transport-security": "" */
 #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \
-  (&grpc_static_mdelem_table[72])
+  (&grpc_static_mdelem_table[73])
 /* "te": "trailers" */
-#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[73])
+#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[74])
 /* "transfer-encoding": "" */
-#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[74])
+#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[75])
 /* "user-agent": "" */
-#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[75])
+#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[76])
 /* "vary": "" */
-#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[76])
+#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[77])
 /* "via": "" */
-#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[77])
+#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[78])
 /* "www-authenticate": "" */
-#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[78])
+#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[79])
 
 extern const uint8_t
     grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2];
diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c
index fdf0f4b2aa..1105494a85 100644
--- a/src/core/lib/transport/transport.c
+++ b/src/core/lib/transport/transport.c
@@ -60,7 +60,7 @@ void grpc_stream_unref(grpc_exec_ctx *exec_ctx,
                        grpc_stream_refcount *refcount) {
 #endif
   if (gpr_unref(&refcount->refs)) {
-    grpc_exec_ctx_push(exec_ctx, &refcount->destroy, GRPC_ERROR_NONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &refcount->destroy, GRPC_ERROR_NONE, NULL);
   }
 }
 
@@ -125,10 +125,19 @@ void grpc_transport_perform_op(grpc_exec_ctx *exec_ctx,
   transport->vtable->perform_op(exec_ctx, transport, op);
 }
 
-void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx,
-                                grpc_transport *transport, grpc_stream *stream,
-                                grpc_pollset *pollset) {
-  transport->vtable->set_pollset(exec_ctx, transport, stream, pollset);
+void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport,
+                             grpc_stream *stream,
+                             grpc_polling_entity *pollent) {
+  grpc_pollset *pollset;
+  grpc_pollset_set *pollset_set;
+  if ((pollset = grpc_polling_entity_pollset(pollent)) != NULL) {
+    transport->vtable->set_pollset(exec_ctx, transport, stream, pollset);
+  } else if ((pollset_set = grpc_polling_entity_pollset_set(pollent)) != NULL) {
+    transport->vtable->set_pollset_set(exec_ctx, transport, stream,
+                                       pollset_set);
+  } else {
+    abort();
+  }
 }
 
 void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
@@ -146,11 +155,11 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
 void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx,
                                                   grpc_transport_stream_op *op,
                                                   grpc_error *error) {
-  grpc_exec_ctx_push(exec_ctx, op->recv_message_ready, GRPC_ERROR_REF(error),
-                     NULL);
-  grpc_exec_ctx_push(exec_ctx, op->recv_initial_metadata_ready,
-                     GRPC_ERROR_REF(error), NULL);
-  grpc_exec_ctx_push(exec_ctx, op->on_complete, error, NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->recv_message_ready, GRPC_ERROR_REF(error),
+                      NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->recv_initial_metadata_ready,
+                      GRPC_ERROR_REF(error), NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->on_complete, error, NULL);
 }
 
 void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op,
diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h
index f2d750e870..a46ccb643c 100644
--- a/src/core/lib/transport/transport.h
+++ b/src/core/lib/transport/transport.h
@@ -37,6 +37,7 @@
 #include <stddef.h>
 
 #include "src/core/lib/channel/context.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 #include "src/core/lib/transport/byte_stream.h"
@@ -197,9 +198,8 @@ int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx,
                                grpc_stream_refcount *refcount,
                                const void *server_data);
 
-void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx,
-                                grpc_transport *transport, grpc_stream *stream,
-                                grpc_pollset *pollset);
+void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport,
+                             grpc_stream *stream, grpc_polling_entity *pollent);
 
 /* Destroy transport data for a stream.
 
diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h
index 956155eec8..fc7140671b 100644
--- a/src/core/lib/transport/transport_impl.h
+++ b/src/core/lib/transport/transport_impl.h
@@ -53,6 +53,10 @@ typedef struct grpc_transport_vtable {
   void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                       grpc_stream *stream, grpc_pollset *pollset);
 
+  /* implementation of grpc_transport_set_pollset */
+  void (*set_pollset_set)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
+                          grpc_stream *stream, grpc_pollset_set *pollset_set);
+
   /* implementation of grpc_transport_perform_stream_op */
   void (*perform_stream_op)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                             grpc_stream *stream, grpc_transport_stream_op *op);
diff --git a/src/core/plugin_registry/grpc_cronet_plugin_registry.c b/src/core/plugin_registry/grpc_cronet_plugin_registry.c
new file mode 100644
index 0000000000..d0b5f5c702
--- /dev/null
+++ b/src/core/plugin_registry/grpc_cronet_plugin_registry.c
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc.h>
+
+extern void grpc_chttp2_plugin_init(void);
+extern void grpc_chttp2_plugin_shutdown(void);
+extern void grpc_client_config_init(void);
+extern void grpc_client_config_shutdown(void);
+
+void grpc_register_built_in_plugins(void) {
+  grpc_register_plugin(grpc_chttp2_plugin_init,
+                       grpc_chttp2_plugin_shutdown);
+  grpc_register_plugin(grpc_client_config_init,
+                       grpc_client_config_shutdown);
+}
diff --git a/src/core/plugin_registry/grpc_plugin_registry.c b/src/core/plugin_registry/grpc_plugin_registry.c
index 822aa6d8b7..905cd59e23 100644
--- a/src/core/plugin_registry/grpc_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_plugin_registry.c
@@ -45,6 +45,8 @@ extern void grpc_resolver_dns_native_init(void);
 extern void grpc_resolver_dns_native_shutdown(void);
 extern void grpc_resolver_sockaddr_init(void);
 extern void grpc_resolver_sockaddr_shutdown(void);
+extern void grpc_load_reporting_plugin_init(void);
+extern void grpc_load_reporting_plugin_shutdown(void);
 extern void census_grpc_plugin_init(void);
 extern void census_grpc_plugin_shutdown(void);
 
@@ -61,6 +63,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_resolver_dns_native_shutdown);
   grpc_register_plugin(grpc_resolver_sockaddr_init,
                        grpc_resolver_sockaddr_shutdown);
+  grpc_register_plugin(grpc_load_reporting_plugin_init,
+                       grpc_load_reporting_plugin_shutdown);
   grpc_register_plugin(census_grpc_plugin_init,
                        census_grpc_plugin_shutdown);
 }
diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
index a6108ae7a9..7995078725 100644
--- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
@@ -41,6 +41,8 @@ extern void grpc_resolver_dns_native_init(void);
 extern void grpc_resolver_dns_native_shutdown(void);
 extern void grpc_resolver_sockaddr_init(void);
 extern void grpc_resolver_sockaddr_shutdown(void);
+extern void grpc_load_reporting_plugin_init(void);
+extern void grpc_load_reporting_plugin_shutdown(void);
 extern void grpc_lb_policy_pick_first_init(void);
 extern void grpc_lb_policy_pick_first_shutdown(void);
 extern void grpc_lb_policy_round_robin_init(void);
@@ -57,6 +59,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_resolver_dns_native_shutdown);
   grpc_register_plugin(grpc_resolver_sockaddr_init,
                        grpc_resolver_sockaddr_shutdown);
+  grpc_register_plugin(grpc_load_reporting_plugin_init,
+                       grpc_load_reporting_plugin_shutdown);
   grpc_register_plugin(grpc_lb_policy_pick_first_init,
                        grpc_lb_policy_pick_first_shutdown);
   grpc_register_plugin(grpc_lb_policy_round_robin_init,