559 files changed, 7842 insertions, 5619 deletions

diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc
index 97295bed45..206a6e1fe5 100644
--- a/src/compiler/cpp_generator.cc
+++ b/src/compiler/cpp_generator.cc
@@ -83,6 +83,28 @@ grpc::string FilenameIdentifier(const grpc::string &filename) {
 }
 }  // namespace
 
+template<class T, size_t N>
+T *array_end(T (&array)[N]) { return array + N; }
+
+void PrintIncludes(grpc::protobuf::io::Printer *printer, const std::vector<grpc::string>& headers, const Parameters &params) {
+  std::map<grpc::string, grpc::string> vars;
+
+  vars["l"] = params.use_system_headers ? '<' : '"';
+  vars["r"] = params.use_system_headers ? '>' : '"';
+
+  if (!params.grpc_search_path.empty()) {
+    vars["l"] += params.grpc_search_path;
+    if (params.grpc_search_path.back() != '/') {
+      vars["l"] += '/';
+    }
+  }
+
+  for (auto i = headers.begin(); i != headers.end(); i++) {
+    vars["h"] = *i;
+    printer->Print(vars, "#include $l$$h$$r$\n");
+  }
+}
+
 grpc::string GetHeaderPrologue(const grpc::protobuf::FileDescriptor *file,
                                const Parameters &params) {
   grpc::string output;
@@ -111,36 +133,46 @@ grpc::string GetHeaderPrologue(const grpc::protobuf::FileDescriptor *file,
 
 grpc::string GetHeaderIncludes(const grpc::protobuf::FileDescriptor *file,
                                const Parameters &params) {
-  grpc::string temp =
-      "#include <grpc++/impl/codegen/async_stream.h>\n"
-      "#include <grpc++/impl/codegen/async_unary_call.h>\n"
-      "#include <grpc++/impl/codegen/proto_utils.h>\n"
-      "#include <grpc++/impl/codegen/rpc_method.h>\n"
-      "#include <grpc++/impl/codegen/service_type.h>\n"
-      "#include <grpc++/impl/codegen/status.h>\n"
-      "#include <grpc++/impl/codegen/stub_options.h>\n"
-      "#include <grpc++/impl/codegen/sync_stream.h>\n"
-      "\n"
-      "namespace grpc {\n"
-      "class CompletionQueue;\n"
-      "class RpcService;\n"
-      "class ServerCompletionQueue;\n"
-      "class ServerContext;\n"
-      "}  // namespace grpc\n\n";
+  grpc::string output;
+  {
+    // Scope the output stream so it closes and finalizes output to the string.
+    grpc::protobuf::io::StringOutputStream output_stream(&output);
+    grpc::protobuf::io::Printer printer(&output_stream, '$');
+    std::map<grpc::string, grpc::string> vars;
 
-  if (!file->package().empty()) {
-    std::vector<grpc::string> parts =
-        grpc_generator::tokenize(file->package(), ".");
+    static const char *headers_strs[] = {
+      "grpc++/impl/codegen/async_stream.h",
+      "grpc++/impl/codegen/async_unary_call.h",
+      "grpc++/impl/codegen/proto_utils.h",
+      "grpc++/impl/codegen/rpc_method.h",
+      "grpc++/impl/codegen/service_type.h",
+      "grpc++/impl/codegen/status.h",
+      "grpc++/impl/codegen/stub_options.h",
+      "grpc++/impl/codegen/sync_stream.h"
+    };
+    std::vector<grpc::string> headers(headers_strs, array_end(headers_strs));
+    PrintIncludes(&printer, headers, params);
+    printer.Print(vars, "\n");
+    printer.Print(vars, "namespace grpc {\n");
+    printer.Print(vars, "class CompletionQueue;\n");
+    printer.Print(vars, "class Channel;\n");
+    printer.Print(vars, "class RpcService;\n");
+    printer.Print(vars, "class ServerCompletionQueue;\n");
+    printer.Print(vars, "class ServerContext;\n");
+    printer.Print(vars, "}  // namespace grpc\n\n");
 
-    for (auto part = parts.begin(); part != parts.end(); part++) {
-      temp.append("namespace ");
-      temp.append(*part);
-      temp.append(" {\n");
+    if (!file->package().empty()) {
+      std::vector<grpc::string> parts =
+          grpc_generator::tokenize(file->package(), ".");
+
+      for (auto part = parts.begin(); part != parts.end(); part++) {
+        vars["part"] = *part;
+        printer.Print(vars, "namespace $part$ {\n");
+      }
+      printer.Print(vars, "\n");
     }
-    temp.append("\n");
   }
-
-  return temp;
+  return output;
 }
 
 void PrintHeaderClientMethodInterfaces(
@@ -852,7 +884,7 @@ grpc::string GetSourcePrologue(const grpc::protobuf::FileDescriptor *file,
 }
 
 grpc::string GetSourceIncludes(const grpc::protobuf::FileDescriptor *file,
-                               const Parameters &param) {
+                               const Parameters &params) {
   grpc::string output;
   {
     // Scope the output stream so it closes and finalizes output to the string.
@@ -860,16 +892,18 @@ grpc::string GetSourceIncludes(const grpc::protobuf::FileDescriptor *file,
     grpc::protobuf::io::Printer printer(&output_stream, '$');
     std::map<grpc::string, grpc::string> vars;
 
-    printer.Print(vars, "#include <grpc++/impl/codegen/async_stream.h>\n");
-    printer.Print(vars, "#include <grpc++/impl/codegen/async_unary_call.h>\n");
-    printer.Print(vars, "#include <grpc++/impl/codegen/channel_interface.h>\n");
-    printer.Print(vars, "#include <grpc++/impl/codegen/client_unary_call.h>\n");
-    printer.Print(vars,
-                  "#include <grpc++/impl/codegen/method_handler_impl.h>\n");
-    printer.Print(vars,
-                  "#include <grpc++/impl/codegen/rpc_service_method.h>\n");
-    printer.Print(vars, "#include <grpc++/impl/codegen/service_type.h>\n");
-    printer.Print(vars, "#include <grpc++/impl/codegen/sync_stream.h>\n");
+    static const char *headers_strs[] = {
+      "grpc++/impl/codegen/async_stream.h",
+      "grpc++/impl/codegen/async_unary_call.h",
+      "grpc++/impl/codegen/channel_interface.h",
+      "grpc++/impl/codegen/client_unary_call.h",
+      "grpc++/impl/codegen/method_handler_impl.h",
+      "grpc++/impl/codegen/rpc_service_method.h",
+      "grpc++/impl/codegen/service_type.h",
+      "grpc++/impl/codegen/sync_stream.h"
+    };
+    std::vector<grpc::string> headers(headers_strs, array_end(headers_strs));
+    PrintIncludes(&printer, headers, params);
 
     if (!file->package().empty()) {
       std::vector<grpc::string> parts =
diff --git a/src/compiler/cpp_generator.h b/src/compiler/cpp_generator.h
index 70c2e985f6..4f9de9d11a 100644
--- a/src/compiler/cpp_generator.h
+++ b/src/compiler/cpp_generator.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -42,6 +42,10 @@ namespace grpc_cpp_generator {
 struct Parameters {
   // Puts the service into a namespace
   grpc::string services_namespace;
+  // Use system includes (<>) or local includes ("")
+  bool use_system_headers;
+  // Prefix to any grpc include
+  grpc::string grpc_search_path;
 };
 
 // Return the prologue of the generated header file.
diff --git a/src/compiler/cpp_plugin.cc b/src/compiler/cpp_plugin.cc
index 88c704948e..d8ada4835c 100644
--- a/src/compiler/cpp_plugin.cc
+++ b/src/compiler/cpp_plugin.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -59,6 +59,7 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
     }
 
     grpc_cpp_generator::Parameters generator_parameters;
+    generator_parameters.use_system_headers = true;
 
     if (!parameter.empty()) {
       std::vector<grpc::string> parameters_list =
@@ -70,6 +71,17 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
           grpc_generator::tokenize(*parameter_string, "=");
         if (param[0] == "services_namespace") {
           generator_parameters.services_namespace = param[1];
+        } else if (param[0] == "use_system_headers") {
+          if (param[1] == "true") {
+            generator_parameters.use_system_headers = true;
+          } else if (param[1] == "false") {
+            generator_parameters.use_system_headers = false;
+          } else {
+            *error = grpc::string("Invalid parameter: ") + *parameter_string;
+            return false;
+          }
+        } else if (param[0] == "grpc_search_path") {
+          generator_parameters.grpc_search_path = param[1];
         } else {
           *error = grpc::string("Unknown parameter: ") + *parameter_string;
           return false;
diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc
index 4fd5dfbecb..5a2ecce1d4 100644
--- a/src/compiler/python_generator.cc
+++ b/src/compiler/python_generator.cc
@@ -190,11 +190,10 @@ bool PrintBetaServicer(const ServiceDescriptor* service,
         "Documentation", doc,
       });
   out->Print("\n");
-  out->Print(dict, "class Beta$Service$Servicer(object):\n");
+  out->Print(dict, "class Beta$Service$Servicer(six.with_metaclass(abc.ABCMeta, object)):\n");
   {
     IndentScope raii_class_indent(out);
     out->Print(dict, "\"\"\"$Documentation$\"\"\"\n");
-    out->Print("__metaclass__ = abc.ABCMeta\n");
     for (int i = 0; i < service->method_count(); ++i) {
       auto meth = service->method(i);
       grpc::string arg_name = meth->client_streaming() ?
@@ -219,11 +218,10 @@ bool PrintBetaStub(const ServiceDescriptor* service,
         "Documentation", doc,
       });
   out->Print("\n");
-  out->Print(dict, "class Beta$Service$Stub(object):\n");
+  out->Print(dict, "class Beta$Service$Stub(six.with_metaclass(abc.ABCMeta, object)):\n");
   {
     IndentScope raii_class_indent(out);
     out->Print(dict, "\"\"\"$Documentation$\"\"\"\n");
-    out->Print("__metaclass__ = abc.ABCMeta\n");
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* meth = service->method(i);
       grpc::string arg_name = meth->client_streaming() ?
@@ -449,6 +447,7 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
 bool PrintPreamble(const FileDescriptor* file,
                    const GeneratorConfiguration& config, Printer* out) {
   out->Print("import abc\n");
+  out->Print("import six\n");
   out->Print("from $Package$ import implementations as beta_implementations\n",
              "Package", config.beta_package_root);
   out->Print("from grpc.framework.common import cardinality\n");
diff --git a/src/core/channel/client_uchannel.c b/src/core/channel/client_uchannel.c
deleted file mode 100644
index 83fcc3a87f..0000000000
--- a/src/core/channel/client_uchannel.c
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- *
- * Copyright 2015-2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/channel/client_uchannel.h"
-
-#include <string.h>
-
-#include "src/core/census/grpc_filter.h"
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/client_channel.h"
-#include "src/core/channel/compress_filter.h"
-#include "src/core/channel/subchannel_call_holder.h"
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/channel.h"
-#include "src/core/transport/connectivity_state.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/useful.h>
-
-/** Microchannel (uchannel) implementation: a lightweight channel without any
- * load-balancing mechanisms meant for communication from within the core. */
-
-typedef struct client_uchannel_channel_data {
-  /** master channel - the grpc_channel instance that ultimately owns
-      this channel_data via its channel stack.
-      We occasionally use this to bump the refcount on the master channel
-      to keep ourselves alive through an asynchronous operation. */
-  grpc_channel_stack *owning_stack;
-
-  /** connectivity state being tracked */
-  grpc_connectivity_state_tracker state_tracker;
-
-  /** the subchannel wrapped by the microchannel */
-  grpc_connected_subchannel *connected_subchannel;
-
-  /** the callback used to stay subscribed to subchannel connectivity
-   * notifications */
-  grpc_closure connectivity_cb;
-
-  /** the current connectivity state of the wrapped subchannel */
-  grpc_connectivity_state subchannel_connectivity;
-
-  gpr_mu mu_state;
-} channel_data;
-
-typedef grpc_subchannel_call_holder call_data;
-
-static void monitor_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
-                               bool iomgr_success) {
-  channel_data *chand = arg;
-  grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
-                              chand->subchannel_connectivity,
-                              "uchannel_monitor_subchannel");
-  grpc_connected_subchannel_notify_on_state_change(
-      exec_ctx, chand->connected_subchannel, NULL,
-      &chand->subchannel_connectivity, &chand->connectivity_cb);
-}
-
-static char *cuc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data);
-}
-
-static void cuc_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
-                                          grpc_call_element *elem,
-                                          grpc_transport_stream_op *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
-  grpc_subchannel_call_holder_perform_op(exec_ctx, elem->call_data, op);
-}
-
-static void cuc_start_transport_op(grpc_exec_ctx *exec_ctx,
-                                   grpc_channel_element *elem,
-                                   grpc_transport_op *op) {
-  channel_data *chand = elem->channel_data;
-
-  grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL);
-
-  GPR_ASSERT(op->set_accept_stream == false);
-  GPR_ASSERT(op->bind_pollset == NULL);
-
-  if (op->on_connectivity_state_change != NULL) {
-    grpc_connectivity_state_notify_on_state_change(
-        exec_ctx, &chand->state_tracker, op->connectivity_state,
-        op->on_connectivity_state_change);
-    op->on_connectivity_state_change = NULL;
-    op->connectivity_state = NULL;
-  }
-
-  if (op->disconnect) {
-    grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
-                                GRPC_CHANNEL_FATAL_FAILURE, "disconnect");
-  }
-}
-
-static int cuc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
-                               grpc_metadata_batch *initial_metadata,
-                               grpc_connected_subchannel **connected_subchannel,
-                               grpc_closure *on_ready) {
-  channel_data *chand = arg;
-  GPR_ASSERT(initial_metadata != NULL);
-  *connected_subchannel = chand->connected_subchannel;
-  return 1;
-}
-
-/* Constructor for call_data */
-static void cuc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                               grpc_call_element_args *args) {
-  grpc_subchannel_call_holder_init(elem->call_data, cuc_pick_subchannel,
-                                   elem->channel_data, args->call_stack);
-}
-
-/* Destructor for call_data */
-static void cuc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
-                                  grpc_call_element *elem) {
-  grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data);
-}
-
-/* Constructor for channel_data */
-static void cuc_init_channel_elem(grpc_exec_ctx *exec_ctx,
-                                  grpc_channel_element *elem,
-                                  grpc_channel_element_args *args) {
-  channel_data *chand = elem->channel_data;
-  memset(chand, 0, sizeof(*chand));
-  grpc_closure_init(&chand->connectivity_cb, monitor_subchannel, chand);
-  GPR_ASSERT(args->is_last);
-  GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter);
-  chand->owning_stack = args->channel_stack;
-  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
-                               "client_uchannel");
-  gpr_mu_init(&chand->mu_state);
-}
-
-/* Destructor for channel_data */
-static void cuc_destroy_channel_elem(grpc_exec_ctx *exec_ctx,
-                                     grpc_channel_element *elem) {
-  channel_data *chand = elem->channel_data;
-  /* cancel subscription */
-  grpc_connected_subchannel_notify_on_state_change(
-      exec_ctx, chand->connected_subchannel, NULL, NULL,
-      &chand->connectivity_cb);
-  grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker);
-  gpr_mu_destroy(&chand->mu_state);
-  GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, chand->connected_subchannel,
-                                  "uchannel");
-}
-
-static void cuc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                            grpc_pollset *pollset) {
-  call_data *calld = elem->call_data;
-  calld->pollset = pollset;
-}
-
-const grpc_channel_filter grpc_client_uchannel_filter = {
-    cuc_start_transport_stream_op, cuc_start_transport_op, sizeof(call_data),
-    cuc_init_call_elem, cuc_set_pollset, cuc_destroy_call_elem,
-    sizeof(channel_data), cuc_init_channel_elem, cuc_destroy_channel_elem,
-    cuc_get_peer, "client-uchannel",
-};
-
-grpc_connectivity_state grpc_client_uchannel_check_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect) {
-  channel_data *chand = elem->channel_data;
-  grpc_connectivity_state out;
-  gpr_mu_lock(&chand->mu_state);
-  out = grpc_connectivity_state_check(&chand->state_tracker);
-  gpr_mu_unlock(&chand->mu_state);
-  return out;
-}
-
-void grpc_client_uchannel_watch_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
-    grpc_connectivity_state *state, grpc_closure *on_complete) {
-  channel_data *chand = elem->channel_data;
-  gpr_mu_lock(&chand->mu_state);
-  grpc_connectivity_state_notify_on_state_change(
-      exec_ctx, &chand->state_tracker, state, on_complete);
-  gpr_mu_unlock(&chand->mu_state);
-}
-
-grpc_channel *grpc_client_uchannel_create(grpc_subchannel *subchannel,
-                                          grpc_channel_args *args) {
-  grpc_channel *channel = NULL;
-#define MAX_FILTERS 3
-  const grpc_channel_filter *filters[MAX_FILTERS];
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  size_t n = 0;
-
-  if (grpc_channel_args_is_census_enabled(args)) {
-    filters[n++] = &grpc_client_census_filter;
-  }
-  filters[n++] = &grpc_compress_filter;
-  filters[n++] = &grpc_client_uchannel_filter;
-  GPR_ASSERT(n <= MAX_FILTERS);
-
-  channel =
-      grpc_channel_create_from_filters(&exec_ctx, NULL, filters, n, args, 1);
-
-  return channel;
-}
-
-void grpc_client_uchannel_set_connected_subchannel(
-    grpc_channel *uchannel, grpc_connected_subchannel *connected_subchannel) {
-  grpc_channel_element *elem =
-      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(uchannel));
-  channel_data *chand = elem->channel_data;
-  GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter);
-  gpr_mu_lock(&chand->mu_state);
-  chand->connected_subchannel = connected_subchannel;
-  GRPC_CONNECTED_SUBCHANNEL_REF(connected_subchannel, "uchannel");
-  gpr_mu_unlock(&chand->mu_state);
-}
diff --git a/src/core/ext/transport/chttp2/client/insecure/README.md b/src/core/ext/transport/chttp2/client/insecure/README.md
new file mode 100644
index 0000000000..fa11463388
--- /dev/null
+++ b/src/core/ext/transport/chttp2/client/insecure/README.md
@@ -0,0 +1 @@
+Plugin for creating insecure channels using chttp2
diff --git a/src/core/surface/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c
index fd7e20e9cc..cf987a02e0 100644
--- a/src/core/surface/channel_create.c
+++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c
@@ -40,16 +40,16 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
 
-#include "src/core/census/grpc_filter.h"
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/client_channel.h"
-#include "src/core/channel/compress_filter.h"
-#include "src/core/channel/http_client_filter.h"
-#include "src/core/client_config/resolver_registry.h"
-#include "src/core/iomgr/tcp_client.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/channel.h"
-#include "src/core/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/census/grpc_filter.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/client_channel.h"
+#include "src/core/lib/channel/compress_filter.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/client_config/resolver_registry.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/channel.h"
 
 typedef struct {
   grpc_connector base;
@@ -105,9 +105,6 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
                                         0);
     GPR_ASSERT(c->result->transport);
     c->result->channel_args = c->args.channel_args;
-    c->result->filters = gpr_malloc(sizeof(grpc_channel_filter *));
-    c->result->filters[0] = &grpc_http_client_filter;
-    c->result->num_filters = 1;
   } else {
     memset(c->result, 0, sizeof(*c->result));
   }
@@ -190,25 +187,16 @@ grpc_channel *grpc_insecure_channel_create(const char *target,
                                            const grpc_channel_args *args,
                                            void *reserved) {
   grpc_channel *channel = NULL;
-#define MAX_FILTERS 3
-  const grpc_channel_filter *filters[MAX_FILTERS];
   grpc_resolver *resolver;
   subchannel_factory *f;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  size_t n = 0;
   GRPC_API_TRACE(
       "grpc_insecure_channel_create(target=%p, args=%p, reserved=%p)", 3,
       (target, args, reserved));
   GPR_ASSERT(!reserved);
-  if (grpc_channel_args_is_census_enabled(args)) {
-    filters[n++] = &grpc_client_census_filter;
-  }
-  filters[n++] = &grpc_compress_filter;
-  filters[n++] = &grpc_client_channel_filter;
-  GPR_ASSERT(n <= MAX_FILTERS);
 
   channel =
-      grpc_channel_create_from_filters(&exec_ctx, target, filters, n, args, 1);
+      grpc_channel_create(&exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL);
 
   f = gpr_malloc(sizeof(*f));
   f->base.vtable = &subchannel_factory_vtable;
diff --git a/src/core/ext/transport/chttp2/client/secure/README.md b/src/core/ext/transport/chttp2/client/secure/README.md
new file mode 100644
index 0000000000..405a86e5db
--- /dev/null
+++ b/src/core/ext/transport/chttp2/client/secure/README.md
@@ -0,0 +1 @@
+Plugin for creating secure channels using chttp2
diff --git a/src/core/surface/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
index aadfac4c91..203475ba52 100644
--- a/src/core/surface/secure_channel_create.c
+++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
@@ -40,20 +40,17 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
 
-#include "src/core/census/grpc_filter.h"
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/client_channel.h"
-#include "src/core/channel/compress_filter.h"
-#include "src/core/channel/http_client_filter.h"
-#include "src/core/client_config/resolver_registry.h"
-#include "src/core/iomgr/tcp_client.h"
-#include "src/core/security/auth_filters.h"
-#include "src/core/security/credentials.h"
-#include "src/core/security/security_context.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/channel.h"
-#include "src/core/transport/chttp2_transport.h"
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/client_channel.h"
+#include "src/core/lib/client_config/resolver_registry.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
 
 typedef struct {
   grpc_connector base;
@@ -115,10 +112,6 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
     args_copy = grpc_channel_args_copy_and_add(c->args.channel_args,
                                                &auth_context_arg, 1);
     c->result->channel_args = args_copy;
-    c->result->filters = gpr_malloc(sizeof(grpc_channel_filter *) * 2);
-    c->result->filters[0] = &grpc_http_client_filter;
-    c->result->filters[1] = &grpc_client_auth_filter;
-    c->result->num_filters = 2;
   }
   notify = c->notify;
   c->notify = NULL;
@@ -262,10 +255,7 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds,
   grpc_channel_security_connector *security_connector;
   grpc_resolver *resolver;
   subchannel_factory *f;
-#define MAX_FILTERS 3
-  const grpc_channel_filter *filters[MAX_FILTERS];
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  size_t n = 0;
 
   GRPC_API_TRACE(
       "grpc_secure_channel_create(creds=%p, target=%s, args=%p, "
@@ -277,7 +267,7 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds,
     gpr_log(GPR_ERROR, "Cannot set security context in channel args.");
     grpc_exec_ctx_finish(&exec_ctx);
     return grpc_lame_client_channel_create(
-        target, GRPC_STATUS_INVALID_ARGUMENT,
+        target, GRPC_STATUS_INTERNAL,
         "Security connector exists in channel args.");
   }
 
@@ -286,23 +276,16 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds,
       GRPC_SECURITY_OK) {
     grpc_exec_ctx_finish(&exec_ctx);
     return grpc_lame_client_channel_create(
-        target, GRPC_STATUS_INVALID_ARGUMENT,
-        "Failed to create security connector.");
+        target, GRPC_STATUS_INTERNAL, "Failed to create security connector.");
   }
 
   connector_arg = grpc_security_connector_to_arg(&security_connector->base);
   args_copy = grpc_channel_args_copy_and_add(
       new_args_from_connector != NULL ? new_args_from_connector : args,
       &connector_arg, 1);
-  if (grpc_channel_args_is_census_enabled(args)) {
-    filters[n++] = &grpc_client_census_filter;
-  }
-  filters[n++] = &grpc_compress_filter;
-  filters[n++] = &grpc_client_channel_filter;
-  GPR_ASSERT(n <= MAX_FILTERS);
 
-  channel = grpc_channel_create_from_filters(&exec_ctx, target, filters, n,
-                                             args_copy, 1);
+  channel = grpc_channel_create(&exec_ctx, target, args_copy,
+                                GRPC_CLIENT_CHANNEL, NULL);
 
   f = gpr_malloc(sizeof(*f));
   f->base.vtable = &subchannel_factory_vtable;
diff --git a/src/core/ext/transport/chttp2/server/insecure/README.md b/src/core/ext/transport/chttp2/server/insecure/README.md
new file mode 100644
index 0000000000..fc0bc14ed7
--- /dev/null
+++ b/src/core/ext/transport/chttp2/server/insecure/README.md
@@ -0,0 +1 @@
+Plugin for creating insecure servers using chttp2
diff --git a/src/core/surface/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
index ce970dfe73..c1ccfbf639 100644
--- a/src/core/surface/server_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -33,22 +33,19 @@
 
 #include <grpc/grpc.h>
 
-#include "src/core/channel/http_server_filter.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/iomgr/tcp_server.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/server.h"
-#include "src/core/transport/chttp2_transport.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/http_server_filter.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/server.h"
 
 static void setup_transport(grpc_exec_ctx *exec_ctx, void *server,
                             grpc_transport *transport) {
-  static grpc_channel_filter const *extra_filters[] = {
-      &grpc_http_server_filter};
-  grpc_server_setup_transport(exec_ctx, server, transport, extra_filters,
-                              GPR_ARRAY_SIZE(extra_filters),
+  grpc_server_setup_transport(exec_ctx, server, transport,
                               grpc_server_get_channel_args(server));
 }
 
diff --git a/src/core/ext/transport/chttp2/server/secure/README.md b/src/core/ext/transport/chttp2/server/secure/README.md
new file mode 100644
index 0000000000..6bda696a9a
--- /dev/null
+++ b/src/core/ext/transport/chttp2/server/secure/README.md
@@ -0,0 +1 @@
+Plugin for creating secure servers using chttp2
diff --git a/src/core/security/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index 91547eb26e..80834f4e88 100644
--- a/src/core/security/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -35,22 +35,22 @@
 
 #include <string.h>
 
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/http_server_filter.h"
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/iomgr/tcp_server.h"
-#include "src/core/security/auth_filters.h"
-#include "src/core/security/credentials.h"
-#include "src/core/security/security_connector.h"
-#include "src/core/security/security_context.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/server.h"
-#include "src/core/transport/chttp2_transport.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_server_filter.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/server.h"
 
 typedef struct grpc_server_secure_state {
   grpc_server *server;
@@ -83,8 +83,6 @@ static void state_unref(grpc_server_secure_state *state) {
 static void setup_transport(grpc_exec_ctx *exec_ctx, void *statep,
                             grpc_transport *transport,
                             grpc_auth_context *auth_context) {
-  static grpc_channel_filter const *extra_filters[] = {
-      &grpc_server_auth_filter, &grpc_http_server_filter};
   grpc_server_secure_state *state = statep;
   grpc_channel_args *args_copy;
   grpc_arg args_to_add[2];
@@ -93,8 +91,7 @@ static void setup_transport(grpc_exec_ctx *exec_ctx, void *statep,
   args_copy = grpc_channel_args_copy_and_add(
       grpc_server_get_channel_args(state->server), args_to_add,
       GPR_ARRAY_SIZE(args_to_add));
-  grpc_server_setup_transport(exec_ctx, state->server, transport, extra_filters,
-                              GPR_ARRAY_SIZE(extra_filters), args_copy);
+  grpc_server_setup_transport(exec_ctx, state->server, transport, args_copy);
   grpc_channel_args_destroy(args_copy);
 }
 
diff --git a/src/core/ext/transport/chttp2/transport/README.md b/src/core/ext/transport/chttp2/transport/README.md
new file mode 100644
index 0000000000..4684e58759
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/README.md
@@ -0,0 +1,4 @@
+chttp2 transport plugin - implements grpc over http2
+
+Used by chttp2/{client,server}/{insecure,secure} plugins to implement most of
+their functionality
diff --git a/src/core/transport/chttp2/alpn.c b/src/core/ext/transport/chttp2/transport/alpn.c
index 69da4e6718..c901905d02 100644
--- a/src/core/transport/chttp2/alpn.c
+++ b/src/core/ext/transport/chttp2/transport/alpn.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/alpn.h"
+#include "src/core/ext/transport/chttp2/transport/alpn.h"
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
diff --git a/src/core/transport/chttp2/alpn.h b/src/core/ext/transport/chttp2/transport/alpn.h
index f38b4c3167..94843a1456 100644
--- a/src/core/transport/chttp2/alpn.h
+++ b/src/core/ext/transport/chttp2/transport/alpn.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_ALPN_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_ALPN_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_ALPN_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_ALPN_H
 
 #include <string.h>
 
@@ -46,4 +46,4 @@ size_t grpc_chttp2_num_alpn_versions(void);
  * grpc_chttp2_num_alpn_versions()) */
 const char *grpc_chttp2_get_alpn_version_index(size_t i);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_ALPN_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_ALPN_H */
diff --git a/src/core/transport/chttp2/bin_encoder.c b/src/core/ext/transport/chttp2/transport/bin_encoder.c
index f26bc7e29b..d39f99c271 100644
--- a/src/core/transport/chttp2/bin_encoder.c
+++ b/src/core/ext/transport/chttp2/transport/bin_encoder.c
@@ -31,12 +31,12 @@
  *
  */
 
-#include "src/core/transport/chttp2/bin_encoder.h"
+#include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
 
 #include <string.h>
 
-#include "src/core/transport/chttp2/huffsyms.h"
 #include <grpc/support/log.h>
+#include "src/core/ext/transport/chttp2/transport/huffsyms.h"
 
 static const char alphabet[] =
     "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
@@ -46,70 +46,18 @@ typedef struct {
   uint8_t length;
 } b64_huff_sym;
 
-static const b64_huff_sym huff_alphabet[64] = {{0x21, 6},
-                                               {0x5d, 7},
-                                               {0x5e, 7},
-                                               {0x5f, 7},
-                                               {0x60, 7},
-                                               {0x61, 7},
-                                               {0x62, 7},
-                                               {0x63, 7},
-                                               {0x64, 7},
-                                               {0x65, 7},
-                                               {0x66, 7},
-                                               {0x67, 7},
-                                               {0x68, 7},
-                                               {0x69, 7},
-                                               {0x6a, 7},
-                                               {0x6b, 7},
-                                               {0x6c, 7},
-                                               {0x6d, 7},
-                                               {0x6e, 7},
-                                               {0x6f, 7},
-                                               {0x70, 7},
-                                               {0x71, 7},
-                                               {0x72, 7},
-                                               {0xfc, 8},
-                                               {0x73, 7},
-                                               {0xfd, 8},
-                                               {0x3, 5},
-                                               {0x23, 6},
-                                               {0x4, 5},
-                                               {0x24, 6},
-                                               {0x5, 5},
-                                               {0x25, 6},
-                                               {0x26, 6},
-                                               {0x27, 6},
-                                               {0x6, 5},
-                                               {0x74, 7},
-                                               {0x75, 7},
-                                               {0x28, 6},
-                                               {0x29, 6},
-                                               {0x2a, 6},
-                                               {0x7, 5},
-                                               {0x2b, 6},
-                                               {0x76, 7},
-                                               {0x2c, 6},
-                                               {0x8, 5},
-                                               {0x9, 5},
-                                               {0x2d, 6},
-                                               {0x77, 7},
-                                               {0x78, 7},
-                                               {0x79, 7},
-                                               {0x7a, 7},
-                                               {0x7b, 7},
-                                               {0x0, 5},
-                                               {0x1, 5},
-                                               {0x2, 5},
-                                               {0x19, 6},
-                                               {0x1a, 6},
-                                               {0x1b, 6},
-                                               {0x1c, 6},
-                                               {0x1d, 6},
-                                               {0x1e, 6},
-                                               {0x1f, 6},
-                                               {0x7fb, 11},
-                                               {0x18, 6}};
+static const b64_huff_sym huff_alphabet[64] = {
+    {0x21, 6}, {0x5d, 7}, {0x5e, 7},   {0x5f, 7}, {0x60, 7}, {0x61, 7},
+    {0x62, 7}, {0x63, 7}, {0x64, 7},   {0x65, 7}, {0x66, 7}, {0x67, 7},
+    {0x68, 7}, {0x69, 7}, {0x6a, 7},   {0x6b, 7}, {0x6c, 7}, {0x6d, 7},
+    {0x6e, 7}, {0x6f, 7}, {0x70, 7},   {0x71, 7}, {0x72, 7}, {0xfc, 8},
+    {0x73, 7}, {0xfd, 8}, {0x3, 5},    {0x23, 6}, {0x4, 5},  {0x24, 6},
+    {0x5, 5},  {0x25, 6}, {0x26, 6},   {0x27, 6}, {0x6, 5},  {0x74, 7},
+    {0x75, 7}, {0x28, 6}, {0x29, 6},   {0x2a, 6}, {0x7, 5},  {0x2b, 6},
+    {0x76, 7}, {0x2c, 6}, {0x8, 5},    {0x9, 5},  {0x2d, 6}, {0x77, 7},
+    {0x78, 7}, {0x79, 7}, {0x7a, 7},   {0x7b, 7}, {0x0, 5},  {0x1, 5},
+    {0x2, 5},  {0x19, 6}, {0x1a, 6},   {0x1b, 6}, {0x1c, 6}, {0x1d, 6},
+    {0x1e, 6}, {0x1f, 6}, {0x7fb, 11}, {0x18, 6}};
 
 static const uint8_t tail_xtra[3] = {0, 2, 3};
 
diff --git a/src/core/transport/chttp2/bin_encoder.h b/src/core/ext/transport/chttp2/transport/bin_encoder.h
index 036fddf998..39dae973c9 100644
--- a/src/core/transport/chttp2/bin_encoder.h
+++ b/src/core/ext/transport/chttp2/transport/bin_encoder.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_BIN_ENCODER_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_BIN_ENCODER_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_ENCODER_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_ENCODER_H
 
 #include <grpc/support/slice.h>
 
@@ -51,4 +51,4 @@ gpr_slice grpc_chttp2_huffman_compress(gpr_slice input);
    return y; */
 gpr_slice grpc_chttp2_base64_encode_and_huffman_compress(gpr_slice input);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_BIN_ENCODER_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_ENCODER_H */
diff --git a/src/core/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
index 19265252ca..b835e74c56 100644
--- a/src/core/transport/chttp2_transport.c
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 
 #include <math.h>
 #include <stdio.h>
@@ -43,14 +43,14 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/transport/chttp2/http2_errors.h"
-#include "src/core/transport/chttp2/internal.h"
-#include "src/core/transport/chttp2/status_conversion.h"
-#include "src/core/transport/chttp2/timeout_encoding.h"
-#include "src/core/transport/static_metadata.h"
-#include "src/core/transport/transport_impl.h"
+#include "src/core/ext/transport/chttp2/transport/http2_errors.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
+#include "src/core/ext/transport/chttp2/transport/status_conversion.h"
+#include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport_impl.h"
 
 #define DEFAULT_WINDOW 65535
 #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024)
@@ -758,23 +758,35 @@ static void maybe_start_some_streams(
   }
 }
 
+#define CLOSURE_BARRIER_STATS_BIT (1 << 0)
+#define CLOSURE_BARRIER_FAILURE_BIT (1 << 1)
+#define CLOSURE_BARRIER_FIRST_REF_BIT (1 << 16)
+
 static grpc_closure *add_closure_barrier(grpc_closure *closure) {
-  closure->final_data += 2;
+  closure->final_data += CLOSURE_BARRIER_FIRST_REF_BIT;
   return closure;
 }
 
 void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_stream_global *stream_global,
                                        grpc_closure **pclosure, int success) {
   grpc_closure *closure = *pclosure;
   if (closure == NULL) {
     return;
   }
-  closure->final_data -= 2;
+  closure->final_data -= CLOSURE_BARRIER_FIRST_REF_BIT;
   if (!success) {
-    closure->final_data |= 1;
+    closure->final_data |= CLOSURE_BARRIER_FAILURE_BIT;
   }
-  if (closure->final_data < 2) {
-    grpc_exec_ctx_enqueue(exec_ctx, closure, closure->final_data == 0, NULL);
+  if (closure->final_data < CLOSURE_BARRIER_FIRST_REF_BIT) {
+    if (closure->final_data & CLOSURE_BARRIER_STATS_BIT) {
+      grpc_transport_move_stats(&stream_global->stats,
+                                stream_global->collecting_stats);
+      stream_global->collecting_stats = NULL;
+    }
+    grpc_exec_ctx_enqueue(
+        exec_ctx, closure,
+        (closure->final_data & CLOSURE_BARRIER_FAILURE_BIT) == 0, NULL);
   }
   *pclosure = NULL;
 }
@@ -807,7 +819,13 @@ static void perform_stream_op_locked(
   }
   /* use final_data as a barrier until enqueue time; the inital counter is
      dropped at the end of this function */
-  on_complete->final_data = 2;
+  on_complete->final_data = CLOSURE_BARRIER_FIRST_REF_BIT;
+
+  if (op->collect_stats != NULL) {
+    GPR_ASSERT(stream_global->collecting_stats == NULL);
+    stream_global->collecting_stats = op->collect_stats;
+    on_complete->final_data |= CLOSURE_BARRIER_STATS_BIT;
+  }
 
   if (op->cancel_with_status != GRPC_STATUS_OK) {
     cancel_from_api(exec_ctx, transport_global, stream_global,
@@ -840,7 +858,8 @@ static void perform_stream_op_locked(
       }
     } else {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->send_initial_metadata_finished, 0);
+          exec_ctx, stream_global,
+          &stream_global->send_initial_metadata_finished, 0);
     }
   }
 
@@ -850,10 +869,12 @@ static void perform_stream_op_locked(
     stream_global->send_message_finished = add_closure_barrier(on_complete);
     if (stream_global->write_closed) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->send_message_finished, 0);
-    } else if (stream_global->id != 0) {
+          exec_ctx, stream_global, &stream_global->send_message_finished, 0);
+    } else {
       stream_global->send_message = op->send_message;
-      grpc_chttp2_become_writable(transport_global, stream_global);
+      if (stream_global->id != 0) {
+        grpc_chttp2_become_writable(transport_global, stream_global);
+      }
     }
   }
 
@@ -868,7 +889,8 @@ static void perform_stream_op_locked(
     }
     if (stream_global->write_closed) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->send_trailing_metadata_finished,
+          exec_ctx, stream_global,
+          &stream_global->send_trailing_metadata_finished,
           grpc_metadata_batch_is_empty(op->send_trailing_metadata));
     } else if (stream_global->id != 0) {
       /* TODO(ctiller): check if there's flow control for any outstanding
@@ -907,7 +929,7 @@ static void perform_stream_op_locked(
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
 
-  grpc_chttp2_complete_closure_step(exec_ctx, &on_complete, 1);
+  grpc_chttp2_complete_closure_step(exec_ctx, stream_global, &on_complete, 1);
 
   GPR_TIMER_END("perform_stream_op_locked", 0);
 }
@@ -1078,7 +1100,8 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
             &stream_global->received_trailing_metadata,
             stream_global->recv_trailing_metadata);
         grpc_chttp2_complete_closure_step(
-            exec_ctx, &stream_global->recv_trailing_metadata_finished, 1);
+            exec_ctx, stream_global,
+            &stream_global->recv_trailing_metadata_finished, 1);
       }
     }
   }
@@ -1129,7 +1152,8 @@ static void cancel_from_api(grpc_exec_ctx *exec_ctx,
         &transport_global->qbuf,
         grpc_chttp2_rst_stream_create(
             stream_global->id,
-            (uint32_t)grpc_chttp2_grpc_status_to_http2_error(status)));
+            (uint32_t)grpc_chttp2_grpc_status_to_http2_error(status),
+            &stream_global->stats.outgoing));
   }
   grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status,
                           NULL);
@@ -1177,10 +1201,12 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
 static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
                                 grpc_chttp2_stream_global *stream_global) {
   grpc_chttp2_complete_closure_step(
-      exec_ctx, &stream_global->send_initial_metadata_finished, 0);
+      exec_ctx, stream_global, &stream_global->send_initial_metadata_finished,
+      0);
   grpc_chttp2_complete_closure_step(
-      exec_ctx, &stream_global->send_trailing_metadata_finished, 0);
-  grpc_chttp2_complete_closure_step(exec_ctx,
+      exec_ctx, stream_global, &stream_global->send_trailing_metadata_finished,
+      0);
+  grpc_chttp2_complete_closure_step(exec_ctx, stream_global,
                                     &stream_global->send_message_finished, 0);
 }
 
@@ -1317,7 +1343,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx,
 
   gpr_slice_buffer_add(
       &transport_global->qbuf,
-      grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR));
+      grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR,
+                                    &stream_global->stats.outgoing));
 
   if (optional_message) {
     gpr_slice_ref(*optional_message);
@@ -1396,8 +1423,8 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, bool success) {
     gpr_mu_unlock(&t->mu);
     GPR_TIMER_BEGIN("recv_data.parse", 0);
     for (; i < t->read_buffer.count &&
-               grpc_chttp2_perform_read(exec_ctx, transport_parsing,
-                                        t->read_buffer.slices[i]);
+           grpc_chttp2_perform_read(exec_ctx, transport_parsing,
+                                    t->read_buffer.slices[i]);
          i++)
       ;
     GPR_TIMER_END("recv_data.parse", 0);
@@ -1472,9 +1499,10 @@ static void connectivity_state_set(
     grpc_connectivity_state state, const char *reason) {
   GRPC_CHTTP2_IF_TRACING(
       gpr_log(GPR_DEBUG, "set connectivity_state=%d", state));
-  grpc_connectivity_state_set(exec_ctx, &TRANSPORT_FROM_GLOBAL(transport_global)
-                                             ->channel_callback.state_tracker,
-                              state, reason);
+  grpc_connectivity_state_set(
+      exec_ctx,
+      &TRANSPORT_FROM_GLOBAL(transport_global)->channel_callback.state_tracker,
+      state, reason);
 }
 
 /*******************************************************************************
@@ -1754,9 +1782,15 @@ static char *chttp2_get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
   return gpr_strdup(((grpc_chttp2_transport *)t)->peer_string);
 }
 
-static const grpc_transport_vtable vtable = {
-    sizeof(grpc_chttp2_stream), init_stream, set_pollset, perform_stream_op,
-    perform_transport_op, destroy_stream, destroy_transport, chttp2_get_peer};
+static const grpc_transport_vtable vtable = {sizeof(grpc_chttp2_stream),
+                                             "chttp2",
+                                             init_stream,
+                                             set_pollset,
+                                             perform_stream_op,
+                                             perform_transport_op,
+                                             destroy_stream,
+                                             destroy_transport,
+                                             chttp2_get_peer};
 
 grpc_transport *grpc_create_chttp2_transport(
     grpc_exec_ctx *exec_ctx, const grpc_channel_args *channel_args,
diff --git a/src/core/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h
index 95520501ed..8ebf9fced6 100644
--- a/src/core/transport/chttp2_transport.h
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TRANSPORT_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TRANSPORT_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/transport/transport.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/transport/transport.h"
 
 extern int grpc_http_trace;
 extern int grpc_flowctl_trace;
@@ -48,4 +48,4 @@ void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx,
                                          grpc_transport *transport,
                                          gpr_slice *slices, size_t nslices);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TRANSPORT_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H */
diff --git a/src/core/transport/chttp2/frame.h b/src/core/ext/transport/chttp2/transport/frame.h
index 879ee036fa..e1311a1805 100644
--- a/src/core/transport/chttp2/frame.h
+++ b/src/core/ext/transport/chttp2/transport/frame.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_H
 
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
@@ -66,4 +66,4 @@ typedef struct grpc_chttp2_transport_parsing grpc_chttp2_transport_parsing;
 #define GRPC_CHTTP2_DATA_FLAG_PADDED 8
 #define GRPC_CHTTP2_FLAG_HAS_PRIORITY 0x20
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_H */
diff --git a/src/core/transport/chttp2/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c
index f9a1af8873..baa66e0008 100644
--- a/src/core/transport/chttp2/frame_data.c
+++ b/src/core/ext/transport/chttp2/transport/frame_data.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,16 @@
  *
  */
 
-#include "src/core/transport/chttp2/frame_data.h"
+#include "src/core/ext/transport/chttp2/transport/frame_data.h"
 
 #include <string.h>
 
-#include "src/core/transport/chttp2/internal.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
-#include "src/core/transport/transport.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/transport.h"
 
 grpc_chttp2_parse_error grpc_chttp2_data_parser_init(
     grpc_chttp2_data_parser *parser) {
@@ -113,11 +113,13 @@ grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop(
 
 void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf,
                              uint32_t write_bytes, int is_eof,
+                             grpc_transport_one_way_stats *stats,
                              gpr_slice_buffer *outbuf) {
   gpr_slice hdr;
   uint8_t *p;
+  static const size_t header_size = 9;
 
-  hdr = gpr_slice_malloc(9);
+  hdr = gpr_slice_malloc(header_size);
   p = GPR_SLICE_START_PTR(hdr);
   GPR_ASSERT(write_bytes < (1 << 24));
   *p++ = (uint8_t)(write_bytes >> 16);
@@ -132,6 +134,9 @@ void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf,
   gpr_slice_buffer_add(outbuf, hdr);
 
   gpr_slice_buffer_move_first(inbuf, write_bytes, outbuf);
+
+  stats->framing_bytes += header_size;
+  stats->data_bytes += write_bytes;
 }
 
 grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
@@ -156,6 +161,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
   switch (p->state) {
   fh_0:
     case GRPC_CHTTP2_DATA_FH_0:
+      stream_parsing->stats.incoming.framing_bytes++;
       p->frame_type = *cur;
       switch (p->frame_type) {
         case 0:
@@ -174,6 +180,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_1:
+      stream_parsing->stats.incoming.framing_bytes++;
       p->frame_size = ((uint32_t)*cur) << 24;
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_2;
@@ -181,6 +188,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_2:
+      stream_parsing->stats.incoming.framing_bytes++;
       p->frame_size |= ((uint32_t)*cur) << 16;
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_3;
@@ -188,6 +196,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_3:
+      stream_parsing->stats.incoming.framing_bytes++;
       p->frame_size |= ((uint32_t)*cur) << 8;
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_4;
@@ -195,6 +204,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_4:
+      stream_parsing->stats.incoming.framing_bytes++;
       p->frame_size |= ((uint32_t)*cur);
       p->state = GRPC_CHTTP2_DATA_FRAME;
       ++cur;
@@ -215,7 +225,9 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       }
       grpc_chttp2_list_add_parsing_seen_stream(transport_parsing,
                                                stream_parsing);
-      if ((uint32_t)(end - cur) == p->frame_size) {
+      uint32_t remaining = (uint32_t)(end - cur);
+      if (remaining == p->frame_size) {
+        stream_parsing->stats.incoming.data_bytes += p->frame_size;
         grpc_chttp2_incoming_byte_stream_push(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg)));
@@ -224,7 +236,8 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
         p->parsing_frame = NULL;
         p->state = GRPC_CHTTP2_DATA_FH_0;
         return GRPC_CHTTP2_PARSE_OK;
-      } else if ((uint32_t)(end - cur) > p->frame_size) {
+      } else if (remaining > p->frame_size) {
+        stream_parsing->stats.incoming.data_bytes += p->frame_size;
         grpc_chttp2_incoming_byte_stream_push(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg),
@@ -235,11 +248,12 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
         cur += p->frame_size;
         goto fh_0; /* loop */
       } else {
+        GPR_ASSERT(remaining <= p->frame_size);
         grpc_chttp2_incoming_byte_stream_push(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg)));
-        GPR_ASSERT((size_t)(end - cur) <= p->frame_size);
-        p->frame_size -= (uint32_t)(end - cur);
+        p->frame_size -= remaining;
+        stream_parsing->stats.incoming.data_bytes += remaining;
         return GRPC_CHTTP2_PARSE_OK;
       }
   }
diff --git a/src/core/transport/chttp2/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h
index 936b7a2589..8a073a9c11 100644
--- a/src/core/transport/chttp2/frame_data.h
+++ b/src/core/ext/transport/chttp2/transport/frame_data.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,17 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_DATA_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_DATA_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H
 
 /* Parser for GRPC streams embedded in DATA frames */
 
-#include "src/core/iomgr/exec_ctx.h"
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
-#include "src/core/transport/byte_stream.h"
-#include "src/core/transport/chttp2/frame.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/transport/byte_stream.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef enum {
   GRPC_CHTTP2_DATA_FH_0,
@@ -96,6 +97,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
 
 void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf,
                              uint32_t write_bytes, int is_eof,
+                             grpc_transport_one_way_stats *stats,
                              gpr_slice_buffer *outbuf);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_DATA_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */
diff --git a/src/core/transport/chttp2/frame_goaway.c b/src/core/ext/transport/chttp2/transport/frame_goaway.c
index 2fa525e989..3697fdef41 100644
--- a/src/core/transport/chttp2/frame_goaway.c
+++ b/src/core/ext/transport/chttp2/transport/frame_goaway.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#include "src/core/transport/chttp2/frame_goaway.h"
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/frame_goaway.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <string.h>
 
diff --git a/src/core/transport/chttp2/frame_goaway.h b/src/core/ext/transport/chttp2/transport/frame_goaway.h
index e1a72b4013..e655134434 100644
--- a/src/core/transport/chttp2/frame_goaway.h
+++ b/src/core/ext/transport/chttp2/transport/frame_goaway.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_GOAWAY_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_GOAWAY_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_GOAWAY_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_GOAWAY_H
 
-#include "src/core/iomgr/exec_ctx.h"
-#include "src/core/transport/chttp2/frame.h"
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef enum {
   GRPC_CHTTP2_GOAWAY_LSI0,
@@ -74,4 +74,4 @@ void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code,
                                gpr_slice debug_data,
                                gpr_slice_buffer *slice_buffer);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_GOAWAY_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_GOAWAY_H */
diff --git a/src/core/transport/chttp2/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c
index c6ab522283..c0192a734d 100644
--- a/src/core/transport/chttp2/frame_ping.c
+++ b/src/core/ext/transport/chttp2/transport/frame_ping.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#include "src/core/transport/chttp2/frame_ping.h"
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/frame_ping.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <string.h>
 
diff --git a/src/core/transport/chttp2/frame_ping.h b/src/core/ext/transport/chttp2/transport/frame_ping.h
index 16d7a72618..1c1d513c99 100644
--- a/src/core/transport/chttp2/frame_ping.h
+++ b/src/core/ext/transport/chttp2/transport/frame_ping.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_PING_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_PING_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_PING_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_PING_H
 
-#include "src/core/iomgr/exec_ctx.h"
 #include <grpc/support/slice.h>
-#include "src/core/transport/chttp2/frame.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef struct {
   uint8_t byte;
@@ -53,4 +53,4 @@ grpc_chttp2_parse_error grpc_chttp2_ping_parser_parse(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_PING_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_PING_H */
diff --git a/src/core/transport/chttp2/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c
index 754529e4b9..d9a04bbf3c 100644
--- a/src/core/transport/chttp2/frame_rst_stream.c
+++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,15 +31,18 @@
  *
  */
 
-#include "src/core/transport/chttp2/frame_rst_stream.h"
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/transport/chttp2/frame.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
 
-gpr_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code) {
-  gpr_slice slice = gpr_slice_malloc(13);
+gpr_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code,
+                                        grpc_transport_one_way_stats *stats) {
+  static const size_t frame_size = 13;
+  gpr_slice slice = gpr_slice_malloc(frame_size);
+  stats->framing_bytes += frame_size;
   uint8_t *p = GPR_SLICE_START_PTR(slice);
 
   *p++ = 0;
@@ -84,6 +87,7 @@ grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_parse(
     cur++;
     p->byte++;
   }
+  stream_parsing->stats.incoming.framing_bytes += (uint64_t)(end - cur);
 
   if (p->byte == 4) {
     GPR_ASSERT(is_last);
diff --git a/src/core/transport/chttp2/frame_rst_stream.h b/src/core/ext/transport/chttp2/transport/frame_rst_stream.h
index 72ca654c32..729274e9a2 100644
--- a/src/core/transport/chttp2/frame_rst_stream.h
+++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,19 +31,21 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_RST_STREAM_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_RST_STREAM_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_RST_STREAM_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_RST_STREAM_H
 
 #include <grpc/support/slice.h>
-#include "src/core/transport/chttp2/frame.h"
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef struct {
   uint8_t byte;
   uint8_t reason_bytes[4];
 } grpc_chttp2_rst_stream_parser;
 
-gpr_slice grpc_chttp2_rst_stream_create(uint32_t stream_id, uint32_t code);
+gpr_slice grpc_chttp2_rst_stream_create(uint32_t stream_id, uint32_t code,
+                                        grpc_transport_one_way_stats *stats);
 
 grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_begin_frame(
     grpc_chttp2_rst_stream_parser *parser, uint32_t length, uint8_t flags);
@@ -52,4 +54,4 @@ grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_parse(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_RST_STREAM_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_RST_STREAM_H */
diff --git a/src/core/transport/chttp2/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c
index cc49dd4f69..799d87b87d 100644
--- a/src/core/transport/chttp2/frame_settings.c
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,18 +31,18 @@
  *
  */
 
-#include "src/core/transport/chttp2/frame_settings.h"
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/frame_settings.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <string.h>
 
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/debug/trace.h"
-#include "src/core/transport/chttp2/frame.h"
-#include "src/core/transport/chttp2/http2_errors.h"
-#include "src/core/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/ext/transport/chttp2/transport/http2_errors.h"
+#include "src/core/lib/debug/trace.h"
 
 #define MAX_MAX_HEADER_LIST_SIZE (1024 * 1024 * 1024)
 
diff --git a/src/core/transport/chttp2/frame_settings.h b/src/core/ext/transport/chttp2/transport/frame_settings.h
index 3c918e3a2a..73524addf0 100644
--- a/src/core/transport/chttp2/frame_settings.h
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_SETTINGS_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_SETTINGS_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_SETTINGS_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_SETTINGS_H
 
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
-#include "src/core/transport/chttp2/frame.h"
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef enum {
   GRPC_CHTTP2_SPS_ID0,
@@ -100,4 +100,4 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_SETTINGS_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_SETTINGS_H */
diff --git a/src/core/transport/chttp2/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c
index 62d9bac117..0baf6d2fdd 100644
--- a/src/core/transport/chttp2/frame_window_update.c
+++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,16 @@
  *
  */
 
-#include "src/core/transport/chttp2/frame_window_update.h"
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/frame_window_update.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <grpc/support/log.h>
 
-gpr_slice grpc_chttp2_window_update_create(uint32_t id,
-                                           uint32_t window_update) {
-  gpr_slice slice = gpr_slice_malloc(13);
+gpr_slice grpc_chttp2_window_update_create(
+    uint32_t id, uint32_t window_update, grpc_transport_one_way_stats *stats) {
+  static const size_t frame_size = 13;
+  gpr_slice slice = gpr_slice_malloc(frame_size);
+  stats->header_bytes += frame_size;
   uint8_t *p = GPR_SLICE_START_PTR(slice);
 
   GPR_ASSERT(window_update);
@@ -87,6 +89,10 @@ grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse(
     p->byte++;
   }
 
+  if (stream_parsing != NULL) {
+    stream_parsing->stats.incoming.framing_bytes += (uint32_t)(end - cur);
+  }
+
   if (p->byte == 4) {
     uint32_t received_update = p->amount;
     if (received_update == 0 || (received_update & 0x80000000u)) {
diff --git a/src/core/transport/chttp2/frame_window_update.h b/src/core/ext/transport/chttp2/transport/frame_window_update.h
index 89d835c079..a1093a6041 100644
--- a/src/core/transport/chttp2/frame_window_update.h
+++ b/src/core/ext/transport/chttp2/transport/frame_window_update.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_WINDOW_UPDATE_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_WINDOW_UPDATE_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_WINDOW_UPDATE_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_WINDOW_UPDATE_H
 
-#include "src/core/iomgr/exec_ctx.h"
 #include <grpc/support/slice.h>
-#include "src/core/transport/chttp2/frame.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef struct {
   uint8_t byte;
@@ -44,7 +45,8 @@ typedef struct {
   uint32_t amount;
 } grpc_chttp2_window_update_parser;
 
-gpr_slice grpc_chttp2_window_update_create(uint32_t id, uint32_t window_delta);
+gpr_slice grpc_chttp2_window_update_create(uint32_t id, uint32_t window_delta,
+                                           grpc_transport_one_way_stats *stats);
 
 grpc_chttp2_parse_error grpc_chttp2_window_update_parser_begin_frame(
     grpc_chttp2_window_update_parser *parser, uint32_t length, uint8_t flags);
@@ -53,4 +55,4 @@ grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_WINDOW_UPDATE_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_WINDOW_UPDATE_H */
diff --git a/src/core/transport/chttp2/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c
index f30f574d06..880305afdd 100644
--- a/src/core/transport/chttp2/hpack_encoder.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/hpack_encoder.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_encoder.h"
 
 #include <assert.h>
 #include <string.h>
@@ -45,11 +45,11 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/transport/chttp2/bin_encoder.h"
-#include "src/core/transport/chttp2/hpack_table.h"
-#include "src/core/transport/chttp2/timeout_encoding.h"
-#include "src/core/transport/chttp2/varint.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_table.h"
+#include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
+#include "src/core/ext/transport/chttp2/transport/varint.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 #define HASH_FRAGMENT_1(x) ((x)&255)
 #define HASH_FRAGMENT_2(x) ((x >> 8) & 255)
@@ -74,6 +74,7 @@ typedef struct {
   /* output stream id */
   uint32_t stream_id;
   gpr_slice_buffer *output;
+  grpc_transport_one_way_stats *stats;
 } framer_state;
 
 /* fills p (which is expected to be 9 bytes long) with a data frame header */
@@ -102,6 +103,7 @@ static void finish_frame(framer_state *st, int is_header_boundary,
       st->stream_id, st->output->length - st->output_length_at_start_of_frame,
       (uint8_t)((is_last_in_stream ? GRPC_CHTTP2_DATA_FLAG_END_STREAM : 0) |
                 (is_header_boundary ? GRPC_CHTTP2_DATA_FLAG_END_HEADERS : 0)));
+  st->stats->framing_bytes += 9;
   st->is_first_frame = 0;
 }
 
@@ -147,8 +149,10 @@ static void add_header_data(framer_state *st, gpr_slice slice) {
   remaining = GRPC_CHTTP2_MAX_PAYLOAD_LENGTH +
               st->output_length_at_start_of_frame - st->output->length;
   if (len <= remaining) {
+    st->stats->header_bytes += len;
     gpr_slice_buffer_add(st->output, slice);
   } else {
+    st->stats->header_bytes += remaining;
     gpr_slice_buffer_add(st->output, gpr_slice_split_head(&slice, remaining));
     finish_frame(st, 0, 0);
     begin_frame(st);
@@ -535,6 +539,7 @@ void grpc_chttp2_hpack_compressor_set_max_table_size(
 void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c,
                                uint32_t stream_id,
                                grpc_metadata_batch *metadata, int is_eof,
+                               grpc_transport_one_way_stats *stats,
                                gpr_slice_buffer *outbuf) {
   framer_state st;
   grpc_linked_mdelem *l;
@@ -546,6 +551,7 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c,
   st.stream_id = stream_id;
   st.output = outbuf;
   st.is_first_frame = 1;
+  st.stats = stats;
 
   /* Encode a metadata batch; store the returned values, representing
      a metadata element that needs to be unreffed back into the metadata
diff --git a/src/core/transport/chttp2/hpack_encoder.h b/src/core/ext/transport/chttp2/transport/hpack_encoder.h
index 19b5cb72ae..91c368fbe2 100644
--- a/src/core/transport/chttp2/hpack_encoder.h
+++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,15 +31,16 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_ENCODER_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_ENCODER_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H
 
-#include "src/core/transport/chttp2/frame.h"
-#include "src/core/transport/metadata.h"
-#include "src/core/transport/metadata_batch.h"
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/lib/transport/transport.h"
 
 #define GRPC_CHTTP2_HPACKC_NUM_FILTERS 256
 #define GRPC_CHTTP2_HPACKC_NUM_VALUES 256
@@ -90,6 +91,7 @@ void grpc_chttp2_hpack_compressor_set_max_usable_size(
 
 void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c, uint32_t id,
                                grpc_metadata_batch *metadata, int is_eof,
+                               grpc_transport_one_way_stats *stats,
                                gpr_slice_buffer *outbuf);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_ENCODER_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H */
diff --git a/src/core/transport/chttp2/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c
index a63c7db1f6..259452fa42 100644
--- a/src/core/transport/chttp2/hpack_parser.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c
@@ -31,12 +31,12 @@
  *
  */
 
-#include "src/core/transport/chttp2/hpack_parser.h"
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_parser.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
+#include <assert.h>
 #include <stddef.h>
 #include <string.h>
-#include <assert.h>
 
 /* This is here for grpc_is_binary_header
  * TODO(murgatroid99): Remove this
@@ -48,9 +48,11 @@
 #include <grpc/support/port_platform.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/transport/chttp2/bin_encoder.h"
+#include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+
+extern int grpc_http_trace;
 
 typedef enum {
   NOT_BINARY,
@@ -723,7 +725,9 @@ static int finish_indexed_field(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
                                 const uint8_t *end) {
   grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   if (md == NULL) {
-    gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index);
+    if (grpc_http_trace) {
+      gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index);
+    }
     return 0;
   }
   GRPC_MDELEM_REF(md);
@@ -919,7 +923,9 @@ static int parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
 /* finish parsing a max table size change */
 static int finish_max_tbl_size(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
                                const uint8_t *end) {
-  gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index);
+  if (grpc_http_trace) {
+    gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index);
+  }
   return grpc_chttp2_hptbl_set_current_table_size(&p->table, p->index) &&
          parse_begin(p, cur, end);
 }
@@ -960,7 +966,9 @@ static int parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 static int parse_illegal_op(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
                             const uint8_t *end) {
   GPR_ASSERT(cur != end);
-  gpr_log(GPR_DEBUG, "Illegal hpack op code %d", *cur);
+  if (grpc_http_trace) {
+    gpr_log(GPR_DEBUG, "Illegal hpack op code %d", *cur);
+  }
   return parse_error(p, cur, end);
 }
 
@@ -1069,10 +1077,12 @@ static int parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
   }
 
 error:
-  gpr_log(GPR_ERROR,
-          "integer overflow in hpack integer decoding: have 0x%08x, "
-          "got byte 0x%02x on byte 5",
-          *p->parsing.value, *cur);
+  if (grpc_http_trace) {
+    gpr_log(GPR_ERROR,
+            "integer overflow in hpack integer decoding: have 0x%08x, "
+            "got byte 0x%02x on byte 5",
+            *p->parsing.value, *cur);
+  }
   return parse_error(p, cur, end);
 }
 
@@ -1094,10 +1104,12 @@ static int parse_value5up(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
     return parse_next(p, cur + 1, end);
   }
 
-  gpr_log(GPR_ERROR,
-          "integer overflow in hpack integer decoding: have 0x%08x, "
-          "got byte 0x%02x sometime after byte 5",
-          *p->parsing.value, *cur);
+  if (grpc_http_trace) {
+    gpr_log(GPR_ERROR,
+            "integer overflow in hpack integer decoding: have 0x%08x, "
+            "got byte 0x%02x sometime after byte 5",
+            *p->parsing.value, *cur);
+  }
   return parse_error(p, cur, end);
 }
 
@@ -1329,7 +1341,9 @@ static is_binary_header is_binary_literal_header(grpc_chttp2_hpack_parser *p) {
 static is_binary_header is_binary_indexed_header(grpc_chttp2_hpack_parser *p) {
   grpc_mdelem *elem = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   if (!elem) {
-    gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index);
+    if (grpc_http_trace) {
+      gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index);
+    }
     return ERROR_HEADER;
   }
   return grpc_is_binary_header(
@@ -1412,6 +1426,9 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
   grpc_chttp2_hpack_parser *parser = hpack_parser;
   GPR_TIMER_BEGIN("grpc_chttp2_hpack_parser_parse", 0);
+  if (stream_parsing != NULL) {
+    stream_parsing->stats.incoming.header_bytes += GPR_SLICE_LENGTH(slice);
+  }
   if (!grpc_chttp2_hpack_parser_parse(parser, GPR_SLICE_START_PTR(slice),
                                       GPR_SLICE_END_PTR(slice))) {
     GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0);
diff --git a/src/core/transport/chttp2/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h
index 1ad0c64fb9..0aaddc8b9c 100644
--- a/src/core/transport/chttp2/hpack_parser.h
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,16 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_PARSER_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_PARSER_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_PARSER_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_PARSER_H
 
 #include <stddef.h>
 
 #include <grpc/support/port_platform.h>
-#include "src/core/iomgr/exec_ctx.h"
-#include "src/core/transport/chttp2/frame.h"
-#include "src/core/transport/chttp2/hpack_table.h"
-#include "src/core/transport/metadata.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_table.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/transport/metadata.h"
 
 typedef struct grpc_chttp2_hpack_parser grpc_chttp2_hpack_parser;
 
@@ -113,4 +113,4 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_PARSER_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_PARSER_H */
diff --git a/src/core/transport/chttp2/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c
index f1ce3b84fd..67cd1bb10a 100644
--- a/src/core/transport/chttp2/hpack_table.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_table.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/hpack_table.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_table.h"
 
 #include <assert.h>
 #include <string.h>
@@ -39,7 +39,9 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/support/murmur_hash.h"
+#include "src/core/lib/support/murmur_hash.h"
+
+extern int grpc_http_trace;
 
 static struct {
   const char *key;
@@ -264,12 +266,16 @@ int grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl,
     return 1;
   }
   if (bytes > tbl->max_bytes) {
-    gpr_log(GPR_ERROR,
-            "Attempt to make hpack table %d bytes when max is %d bytes", bytes,
-            tbl->max_bytes);
+    if (grpc_http_trace) {
+      gpr_log(GPR_ERROR,
+              "Attempt to make hpack table %d bytes when max is %d bytes",
+              bytes, tbl->max_bytes);
+    }
     return 0;
   }
-  gpr_log(GPR_DEBUG, "Update hpack parser table size to %d", bytes);
+  if (grpc_http_trace) {
+    gpr_log(GPR_DEBUG, "Update hpack parser table size to %d", bytes);
+  }
   while (tbl->mem_used > bytes) {
     evict1(tbl);
   }
@@ -293,10 +299,12 @@ int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) {
                       GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD;
 
   if (tbl->current_table_bytes > tbl->max_bytes) {
-    gpr_log(GPR_ERROR,
-            "HPACK max table size reduced to %d but not reflected by hpack "
-            "stream (still at %d)",
-            tbl->max_bytes, tbl->current_table_bytes);
+    if (grpc_http_trace) {
+      gpr_log(GPR_ERROR,
+              "HPACK max table size reduced to %d but not reflected by hpack "
+              "stream (still at %d)",
+              tbl->max_bytes, tbl->current_table_bytes);
+    }
     return 0;
   }
 
diff --git a/src/core/transport/chttp2/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h
index e7431255fc..b3475c8f5c 100644
--- a/src/core/transport/chttp2/hpack_table.h
+++ b/src/core/ext/transport/chttp2/transport/hpack_table.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_TABLE_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_TABLE_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H
 
-#include "src/core/transport/metadata.h"
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
+#include "src/core/lib/transport/metadata.h"
 
 /* HPACK header table */
 
@@ -105,4 +105,4 @@ typedef struct {
 grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find(
     const grpc_chttp2_hptbl *tbl, grpc_mdelem *md);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_TABLE_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H */
diff --git a/src/core/transport/chttp2/hpack_tables.txt b/src/core/ext/transport/chttp2/transport/hpack_tables.txt
index 08842a0267..08842a0267 100644
--- a/src/core/transport/chttp2/hpack_tables.txt
+++ b/src/core/ext/transport/chttp2/transport/hpack_tables.txt
diff --git a/src/core/transport/chttp2/http2_errors.h b/src/core/ext/transport/chttp2/transport/http2_errors.h
index a4f309e056..85542e2337 100644
--- a/src/core/transport/chttp2/http2_errors.h
+++ b/src/core/ext/transport/chttp2/transport/http2_errors.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HTTP2_ERRORS_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HTTP2_ERRORS_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_ERRORS_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_ERRORS_H
 
 /* error codes for RST_STREAM from http2 draft 14 section 7 */
 typedef enum {
@@ -53,4 +53,4 @@ typedef enum {
   GRPC_CHTTP2__ERROR_DO_NOT_USE = -1
 } grpc_chttp2_error_code;
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HTTP2_ERRORS_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_ERRORS_H */
diff --git a/src/core/ext/transport/chttp2/transport/huffsyms.c b/src/core/ext/transport/chttp2/transport/huffsyms.c
new file mode 100644
index 0000000000..91f62bf34b
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/huffsyms.c
@@ -0,0 +1,105 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/ext/transport/chttp2/transport/huffsyms.h"
+
+/* Constants pulled from the HPACK spec, and converted to C using the vim
+   command:
+   :%s/.*   \([0-9a-f]\+\)  \[ *\([0-9]\+\)\]/{0x\1, \2},/g */
+const grpc_chttp2_huffsym grpc_chttp2_huffsyms[GRPC_CHTTP2_NUM_HUFFSYMS] = {
+    {0x1ff8, 13},     {0x7fffd8, 23},   {0xfffffe2, 28},  {0xfffffe3, 28},
+    {0xfffffe4, 28},  {0xfffffe5, 28},  {0xfffffe6, 28},  {0xfffffe7, 28},
+    {0xfffffe8, 28},  {0xffffea, 24},   {0x3ffffffc, 30}, {0xfffffe9, 28},
+    {0xfffffea, 28},  {0x3ffffffd, 30}, {0xfffffeb, 28},  {0xfffffec, 28},
+    {0xfffffed, 28},  {0xfffffee, 28},  {0xfffffef, 28},  {0xffffff0, 28},
+    {0xffffff1, 28},  {0xffffff2, 28},  {0x3ffffffe, 30}, {0xffffff3, 28},
+    {0xffffff4, 28},  {0xffffff5, 28},  {0xffffff6, 28},  {0xffffff7, 28},
+    {0xffffff8, 28},  {0xffffff9, 28},  {0xffffffa, 28},  {0xffffffb, 28},
+    {0x14, 6},        {0x3f8, 10},      {0x3f9, 10},      {0xffa, 12},
+    {0x1ff9, 13},     {0x15, 6},        {0xf8, 8},        {0x7fa, 11},
+    {0x3fa, 10},      {0x3fb, 10},      {0xf9, 8},        {0x7fb, 11},
+    {0xfa, 8},        {0x16, 6},        {0x17, 6},        {0x18, 6},
+    {0x0, 5},         {0x1, 5},         {0x2, 5},         {0x19, 6},
+    {0x1a, 6},        {0x1b, 6},        {0x1c, 6},        {0x1d, 6},
+    {0x1e, 6},        {0x1f, 6},        {0x5c, 7},        {0xfb, 8},
+    {0x7ffc, 15},     {0x20, 6},        {0xffb, 12},      {0x3fc, 10},
+    {0x1ffa, 13},     {0x21, 6},        {0x5d, 7},        {0x5e, 7},
+    {0x5f, 7},        {0x60, 7},        {0x61, 7},        {0x62, 7},
+    {0x63, 7},        {0x64, 7},        {0x65, 7},        {0x66, 7},
+    {0x67, 7},        {0x68, 7},        {0x69, 7},        {0x6a, 7},
+    {0x6b, 7},        {0x6c, 7},        {0x6d, 7},        {0x6e, 7},
+    {0x6f, 7},        {0x70, 7},        {0x71, 7},        {0x72, 7},
+    {0xfc, 8},        {0x73, 7},        {0xfd, 8},        {0x1ffb, 13},
+    {0x7fff0, 19},    {0x1ffc, 13},     {0x3ffc, 14},     {0x22, 6},
+    {0x7ffd, 15},     {0x3, 5},         {0x23, 6},        {0x4, 5},
+    {0x24, 6},        {0x5, 5},         {0x25, 6},        {0x26, 6},
+    {0x27, 6},        {0x6, 5},         {0x74, 7},        {0x75, 7},
+    {0x28, 6},        {0x29, 6},        {0x2a, 6},        {0x7, 5},
+    {0x2b, 6},        {0x76, 7},        {0x2c, 6},        {0x8, 5},
+    {0x9, 5},         {0x2d, 6},        {0x77, 7},        {0x78, 7},
+    {0x79, 7},        {0x7a, 7},        {0x7b, 7},        {0x7ffe, 15},
+    {0x7fc, 11},      {0x3ffd, 14},     {0x1ffd, 13},     {0xffffffc, 28},
+    {0xfffe6, 20},    {0x3fffd2, 22},   {0xfffe7, 20},    {0xfffe8, 20},
+    {0x3fffd3, 22},   {0x3fffd4, 22},   {0x3fffd5, 22},   {0x7fffd9, 23},
+    {0x3fffd6, 22},   {0x7fffda, 23},   {0x7fffdb, 23},   {0x7fffdc, 23},
+    {0x7fffdd, 23},   {0x7fffde, 23},   {0xffffeb, 24},   {0x7fffdf, 23},
+    {0xffffec, 24},   {0xffffed, 24},   {0x3fffd7, 22},   {0x7fffe0, 23},
+    {0xffffee, 24},   {0x7fffe1, 23},   {0x7fffe2, 23},   {0x7fffe3, 23},
+    {0x7fffe4, 23},   {0x1fffdc, 21},   {0x3fffd8, 22},   {0x7fffe5, 23},
+    {0x3fffd9, 22},   {0x7fffe6, 23},   {0x7fffe7, 23},   {0xffffef, 24},
+    {0x3fffda, 22},   {0x1fffdd, 21},   {0xfffe9, 20},    {0x3fffdb, 22},
+    {0x3fffdc, 22},   {0x7fffe8, 23},   {0x7fffe9, 23},   {0x1fffde, 21},
+    {0x7fffea, 23},   {0x3fffdd, 22},   {0x3fffde, 22},   {0xfffff0, 24},
+    {0x1fffdf, 21},   {0x3fffdf, 22},   {0x7fffeb, 23},   {0x7fffec, 23},
+    {0x1fffe0, 21},   {0x1fffe1, 21},   {0x3fffe0, 22},   {0x1fffe2, 21},
+    {0x7fffed, 23},   {0x3fffe1, 22},   {0x7fffee, 23},   {0x7fffef, 23},
+    {0xfffea, 20},    {0x3fffe2, 22},   {0x3fffe3, 22},   {0x3fffe4, 22},
+    {0x7ffff0, 23},   {0x3fffe5, 22},   {0x3fffe6, 22},   {0x7ffff1, 23},
+    {0x3ffffe0, 26},  {0x3ffffe1, 26},  {0xfffeb, 20},    {0x7fff1, 19},
+    {0x3fffe7, 22},   {0x7ffff2, 23},   {0x3fffe8, 22},   {0x1ffffec, 25},
+    {0x3ffffe2, 26},  {0x3ffffe3, 26},  {0x3ffffe4, 26},  {0x7ffffde, 27},
+    {0x7ffffdf, 27},  {0x3ffffe5, 26},  {0xfffff1, 24},   {0x1ffffed, 25},
+    {0x7fff2, 19},    {0x1fffe3, 21},   {0x3ffffe6, 26},  {0x7ffffe0, 27},
+    {0x7ffffe1, 27},  {0x3ffffe7, 26},  {0x7ffffe2, 27},  {0xfffff2, 24},
+    {0x1fffe4, 21},   {0x1fffe5, 21},   {0x3ffffe8, 26},  {0x3ffffe9, 26},
+    {0xffffffd, 28},  {0x7ffffe3, 27},  {0x7ffffe4, 27},  {0x7ffffe5, 27},
+    {0xfffec, 20},    {0xfffff3, 24},   {0xfffed, 20},    {0x1fffe6, 21},
+    {0x3fffe9, 22},   {0x1fffe7, 21},   {0x1fffe8, 21},   {0x7ffff3, 23},
+    {0x3fffea, 22},   {0x3fffeb, 22},   {0x1ffffee, 25},  {0x1ffffef, 25},
+    {0xfffff4, 24},   {0xfffff5, 24},   {0x3ffffea, 26},  {0x7ffff4, 23},
+    {0x3ffffeb, 26},  {0x7ffffe6, 27},  {0x3ffffec, 26},  {0x3ffffed, 26},
+    {0x7ffffe7, 27},  {0x7ffffe8, 27},  {0x7ffffe9, 27},  {0x7ffffea, 27},
+    {0x7ffffeb, 27},  {0xffffffe, 28},  {0x7ffffec, 27},  {0x7ffffed, 27},
+    {0x7ffffee, 27},  {0x7ffffef, 27},  {0x7fffff0, 27},  {0x3ffffee, 26},
+    {0x3fffffff, 30},
+};
diff --git a/src/core/transport/chttp2/huffsyms.h b/src/core/ext/transport/chttp2/transport/huffsyms.h
index a3cdba8235..780baeaf55 100644
--- a/src/core/transport/chttp2/huffsyms.h
+++ b/src/core/ext/transport/chttp2/transport/huffsyms.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HUFFSYMS_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HUFFSYMS_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HUFFSYMS_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HUFFSYMS_H
 
 /* HPACK static huffman table */
 
@@ -45,4 +45,4 @@ typedef struct {
 
 extern const grpc_chttp2_huffsym grpc_chttp2_huffsyms[GRPC_CHTTP2_NUM_HUFFSYMS];
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HUFFSYMS_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HUFFSYMS_H */
diff --git a/src/core/transport/chttp2/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c
index 315bc2faa1..ef5fd4fe03 100644
--- a/src/core/transport/chttp2/incoming_metadata.c
+++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#include "src/core/transport/chttp2/incoming_metadata.h"
+#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 
 #include <string.h>
 
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/transport/chttp2/incoming_metadata.h b/src/core/ext/transport/chttp2/transport/incoming_metadata.h
index ea74cfc64b..5e1dc72389 100644
--- a/src/core/transport/chttp2/incoming_metadata.h
+++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHTTP2_INCOMING_METADATA_H
-#define GRPC_INTERNAL_CORE_CHTTP2_INCOMING_METADATA_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INCOMING_METADATA_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INCOMING_METADATA_H
 
-#include "src/core/transport/transport.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef struct {
   grpc_linked_mdelem *elems;
@@ -57,4 +57,4 @@ void grpc_chttp2_incoming_metadata_buffer_add(
 void grpc_chttp2_incoming_metadata_buffer_set_deadline(
     grpc_chttp2_incoming_metadata_buffer *buffer, gpr_timespec deadline);
 
-#endif /* GRPC_INTERNAL_CORE_CHTTP2_INCOMING_METADATA_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INCOMING_METADATA_H */
diff --git a/src/core/transport/chttp2/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index b720d1ab3e..2c3c7a3820 100644
--- a/src/core/transport/chttp2/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -31,26 +31,26 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHTTP2_INTERNAL_H
-#define GRPC_INTERNAL_CORE_CHTTP2_INTERNAL_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H
 
 #include <assert.h>
 #include <stdbool.h>
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/transport/chttp2/frame.h"
-#include "src/core/transport/chttp2/frame_data.h"
-#include "src/core/transport/chttp2/frame_goaway.h"
-#include "src/core/transport/chttp2/frame_ping.h"
-#include "src/core/transport/chttp2/frame_rst_stream.h"
-#include "src/core/transport/chttp2/frame_settings.h"
-#include "src/core/transport/chttp2/frame_window_update.h"
-#include "src/core/transport/chttp2/hpack_encoder.h"
-#include "src/core/transport/chttp2/hpack_parser.h"
-#include "src/core/transport/chttp2/incoming_metadata.h"
-#include "src/core/transport/chttp2/stream_map.h"
-#include "src/core/transport/connectivity_state.h"
-#include "src/core/transport/transport_impl.h"
+#include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/ext/transport/chttp2/transport/frame_data.h"
+#include "src/core/ext/transport/chttp2/transport/frame_goaway.h"
+#include "src/core/ext/transport/chttp2/transport/frame_ping.h"
+#include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h"
+#include "src/core/ext/transport/chttp2/transport/frame_settings.h"
+#include "src/core/ext/transport/chttp2/transport/frame_window_update.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_encoder.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_parser.h"
+#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
+#include "src/core/ext/transport/chttp2/transport/stream_map.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/transport_impl.h"
 
 typedef struct grpc_chttp2_transport grpc_chttp2_transport;
 typedef struct grpc_chttp2_stream grpc_chttp2_stream;
@@ -394,6 +394,9 @@ typedef struct {
   grpc_metadata_batch *recv_trailing_metadata;
   grpc_closure *recv_trailing_metadata_finished;
 
+  grpc_transport_stream_stats *collecting_stats;
+  grpc_transport_stream_stats stats;
+
   /** when the application requests writes be closed, the write_closed is
       'queued'; when the close is flow controlled into the send path, we are
       'sending' it; when the write has been performed it is 'sent' */
@@ -435,6 +438,8 @@ typedef struct {
   gpr_slice fetching_slice;
   size_t stream_fetched;
   grpc_closure finished_fetch;
+  /** stats gathered during the write */
+  grpc_transport_one_way_stats stats;
 } grpc_chttp2_stream_writing;
 
 struct grpc_chttp2_stream_parsing {
@@ -460,6 +465,8 @@ struct grpc_chttp2_stream_parsing {
   int64_t outgoing_window;
   /** number of bytes received - reset at end of parse thread execution */
   int64_t received_bytes;
+  /** stats gathered during the parse */
+  grpc_transport_stream_stats stats;
 
   /** incoming metadata */
   grpc_chttp2_incoming_metadata_buffer metadata_buffer[2];
@@ -635,6 +642,7 @@ void grpc_chttp2_parsing_become_skip_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
 
 void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_stream_global *stream_global,
                                        grpc_closure **pclosure, int success);
 
 #define GRPC_CHTTP2_CLIENT_CONNECT_STRING "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
@@ -777,4 +785,4 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx,
 void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global,
                                  grpc_chttp2_stream_global *stream_global);
 
-#endif
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H */
diff --git a/src/core/transport/chttp2/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c
index 0516f39fa9..1e815e096b 100644
--- a/src/core/transport/chttp2/parsing.c
+++ b/src/core/ext/transport/chttp2/transport/parsing.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <string.h>
 
@@ -39,11 +39,11 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/profiling/timers.h"
-#include "src/core/transport/chttp2/http2_errors.h"
-#include "src/core/transport/chttp2/status_conversion.h"
-#include "src/core/transport/chttp2/timeout_encoding.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/ext/transport/chttp2/transport/http2_errors.h"
+#include "src/core/ext/transport/chttp2/transport/status_conversion.h"
+#include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 static int init_frame_parser(grpc_exec_ctx *exec_ctx,
                              grpc_chttp2_transport_parsing *transport_parsing);
@@ -171,6 +171,9 @@ void grpc_chttp2_publish_reads(
       grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
     }
 
+    /* flush stats to global stream state */
+    grpc_transport_move_stats(&stream_parsing->stats, &stream_global->stats);
+
     /* update outgoing flow control window */
     was_zero = stream_global->outgoing_window <= 0;
     GRPC_CHTTP2_FLOW_MOVE_STREAM("parsed", transport_global, stream_global,
@@ -544,8 +547,13 @@ static int init_data_frame_parser(
       grpc_chttp2_parsing_lookup_stream(transport_parsing,
                                         transport_parsing->incoming_stream_id);
   grpc_chttp2_parse_error err = GRPC_CHTTP2_PARSE_OK;
-  if (!stream_parsing || stream_parsing->received_close)
+  if (stream_parsing == NULL) {
+    return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
+  }
+  stream_parsing->stats.incoming.framing_bytes += 9;
+  if (stream_parsing->received_close) {
     return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
+  }
   if (err == GRPC_CHTTP2_PARSE_OK) {
     err = update_incoming_window(exec_ctx, transport_parsing, stream_parsing);
   }
@@ -566,7 +574,8 @@ static int init_data_frame_parser(
       gpr_slice_buffer_add(
           &transport_parsing->qbuf,
           grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id,
-                                        GRPC_CHTTP2_PROTOCOL_ERROR));
+                                        GRPC_CHTTP2_PROTOCOL_ERROR,
+                                        &stream_parsing->stats.outgoing));
       return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
     case GRPC_CHTTP2_CONNECTION_ERROR:
       return 0;
@@ -717,6 +726,7 @@ static int init_header_frame_parser(
     transport_parsing->incoming_stream = stream_parsing;
   }
   GPR_ASSERT(stream_parsing != NULL && (via_accept == 0 || via_accept == 1));
+  stream_parsing->stats.incoming.framing_bytes += 9;
   if (stream_parsing->received_close) {
     gpr_log(GPR_ERROR, "skipping already closed grpc_chttp2_stream header");
     transport_parsing->incoming_stream = NULL;
@@ -752,9 +762,14 @@ static int init_window_update_frame_parser(
                                        &transport_parsing->simple.window_update,
                                        transport_parsing->incoming_frame_size,
                                        transport_parsing->incoming_frame_flags);
-  if (transport_parsing->incoming_stream_id) {
-    transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream(
-        transport_parsing, transport_parsing->incoming_stream_id);
+  if (transport_parsing->incoming_stream_id != 0) {
+    grpc_chttp2_stream_parsing *stream_parsing =
+        transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream(
+            transport_parsing, transport_parsing->incoming_stream_id);
+    if (stream_parsing == NULL) {
+      return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
+    }
+    stream_parsing->stats.incoming.framing_bytes += 9;
   }
   transport_parsing->parser = grpc_chttp2_window_update_parser_parse;
   transport_parsing->parser_data = &transport_parsing->simple.window_update;
@@ -778,11 +793,13 @@ static int init_rst_stream_parser(
                                        &transport_parsing->simple.rst_stream,
                                        transport_parsing->incoming_frame_size,
                                        transport_parsing->incoming_frame_flags);
-  transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream(
-      transport_parsing, transport_parsing->incoming_stream_id);
+  grpc_chttp2_stream_parsing *stream_parsing =
+      transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream(
+          transport_parsing, transport_parsing->incoming_stream_id);
   if (!transport_parsing->incoming_stream) {
     return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
   }
+  stream_parsing->stats.incoming.framing_bytes += 9;
   transport_parsing->parser = grpc_chttp2_rst_stream_parser_parse;
   transport_parsing->parser_data = &transport_parsing->simple.rst_stream;
   return ok;
@@ -856,7 +873,8 @@ static int parse_frame_slice(grpc_exec_ctx *exec_ctx,
         gpr_slice_buffer_add(
             &transport_parsing->qbuf,
             grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id,
-                                          GRPC_CHTTP2_PROTOCOL_ERROR));
+                                          GRPC_CHTTP2_PROTOCOL_ERROR,
+                                          &stream_parsing->stats.outgoing));
       }
       return 1;
     case GRPC_CHTTP2_CONNECTION_ERROR:
diff --git a/src/core/transport/chttp2/status_conversion.c b/src/core/ext/transport/chttp2/transport/status_conversion.c
index bf214b017a..5a79579989 100644
--- a/src/core/transport/chttp2/status_conversion.c
+++ b/src/core/ext/transport/chttp2/transport/status_conversion.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/status_conversion.h"
+#include "src/core/ext/transport/chttp2/transport/status_conversion.h"
 
 int grpc_chttp2_grpc_status_to_http2_error(grpc_status_code status) {
   switch (status) {
diff --git a/src/core/transport/chttp2/status_conversion.h b/src/core/ext/transport/chttp2/transport/status_conversion.h
index 0ec5b560b8..e92a5f6702 100644
--- a/src/core/transport/chttp2/status_conversion.h
+++ b/src/core/ext/transport/chttp2/transport/status_conversion.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STATUS_CONVERSION_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STATUS_CONVERSION_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_STATUS_CONVERSION_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_STATUS_CONVERSION_H
 
 #include <grpc/grpc.h>
-#include "src/core/transport/chttp2/http2_errors.h"
+#include "src/core/ext/transport/chttp2/transport/http2_errors.h"
 
 /* Conversion of grpc status codes to http2 error codes (for RST_STREAM) */
 grpc_chttp2_error_code grpc_chttp2_grpc_status_to_http2_error(
@@ -47,4 +47,4 @@ grpc_status_code grpc_chttp2_http2_error_to_grpc_status(
 grpc_status_code grpc_chttp2_http2_status_to_grpc_status(int status);
 int grpc_chttp2_grpc_status_to_http2_status(grpc_status_code status);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STATUS_CONVERSION_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_STATUS_CONVERSION_H */
diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c
index 60fe735cfc..01ed49b1ec 100644
--- a/src/core/transport/chttp2/stream_lists.c
+++ b/src/core/ext/transport/chttp2/transport/stream_lists.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <grpc/support/log.h>
 
diff --git a/src/core/transport/chttp2/stream_map.c b/src/core/ext/transport/chttp2/transport/stream_map.c
index 555a16fb72..3fb1389650 100644
--- a/src/core/transport/chttp2/stream_map.c
+++ b/src/core/ext/transport/chttp2/transport/stream_map.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/stream_map.h"
+#include "src/core/ext/transport/chttp2/transport/stream_map.h"
 
 #include <string.h>
 
diff --git a/src/core/transport/chttp2/stream_map.h b/src/core/ext/transport/chttp2/transport/stream_map.h
index 7a0e45fab2..4e8586fe46 100644
--- a/src/core/transport/chttp2/stream_map.h
+++ b/src/core/ext/transport/chttp2/transport/stream_map.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STREAM_MAP_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STREAM_MAP_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_STREAM_MAP_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_STREAM_MAP_H
 
 #include <grpc/support/port_platform.h>
 
@@ -81,4 +81,4 @@ void grpc_chttp2_stream_map_for_each(grpc_chttp2_stream_map *map,
                                                void *value),
                                      void *user_data);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STREAM_MAP_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_STREAM_MAP_H */
diff --git a/src/core/transport/chttp2/timeout_encoding.c b/src/core/ext/transport/chttp2/transport/timeout_encoding.c
index a6f7081d21..d5b9da9252 100644
--- a/src/core/transport/chttp2/timeout_encoding.c
+++ b/src/core/ext/transport/chttp2/transport/timeout_encoding.c
@@ -31,13 +31,13 @@
  *
  */
 
-#include "src/core/transport/chttp2/timeout_encoding.h"
+#include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
 
 #include <stdio.h>
 #include <string.h>
 
 #include <grpc/support/port_platform.h>
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 static int64_t round_up(int64_t x, int64_t divisor) {
   return (x / divisor + (x % divisor != 0)) * divisor;
@@ -150,7 +150,7 @@ int grpc_chttp2_decode_timeout(const char *buffer, gpr_timespec *timeout) {
     /* spec allows max. 8 digits, but we allow values up to 1,000,000,000 */
     if (x >= (100 * 1000 * 1000)) {
       if (x != (100 * 1000 * 1000) || digit != 0) {
-        *timeout = gpr_inf_future(GPR_CLOCK_REALTIME);
+        *timeout = gpr_inf_future(GPR_TIMESPAN);
         return 1;
       }
     }
diff --git a/src/core/transport/chttp2/timeout_encoding.h b/src/core/ext/transport/chttp2/transport/timeout_encoding.h
index 9d8756e799..dc64f9cc3f 100644
--- a/src/core/transport/chttp2/timeout_encoding.h
+++ b/src/core/ext/transport/chttp2/transport/timeout_encoding.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TIMEOUT_ENCODING_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TIMEOUT_ENCODING_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_TIMEOUT_ENCODING_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_TIMEOUT_ENCODING_H
 
-#include "src/core/support/string.h"
 #include <grpc/support/time.h>
+#include "src/core/lib/support/string.h"
 
 #define GRPC_CHTTP2_TIMEOUT_ENCODE_MIN_BUFSIZE (GPR_LTOA_MIN_BUFSIZE + 1)
 
@@ -44,4 +44,4 @@
 void grpc_chttp2_encode_timeout(gpr_timespec timeout, char *buffer);
 int grpc_chttp2_decode_timeout(const char *buffer, gpr_timespec *timeout);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TIMEOUT_ENCODING_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_TIMEOUT_ENCODING_H */
diff --git a/src/core/transport/chttp2/varint.c b/src/core/ext/transport/chttp2/transport/varint.c
index 1cc235e989..6721d042a2 100644
--- a/src/core/transport/chttp2/varint.c
+++ b/src/core/ext/transport/chttp2/transport/varint.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/chttp2/varint.h"
+#include "src/core/ext/transport/chttp2/transport/varint.h"
 
 uint32_t grpc_chttp2_hpack_varint_length(uint32_t tail_value) {
   if (tail_value < (1 << 7)) {
diff --git a/src/core/transport/chttp2/varint.h b/src/core/ext/transport/chttp2/transport/varint.h
index 2d92b6693e..6442ea3c5a 100644
--- a/src/core/transport/chttp2/varint.h
+++ b/src/core/ext/transport/chttp2/transport/varint.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_VARINT_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_VARINT_H
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_VARINT_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_VARINT_H
 
 #include <grpc/support/port_platform.h>
 
@@ -72,4 +72,4 @@ void grpc_chttp2_hpack_write_varint_tail(uint32_t tail_value, uint8_t* target,
     }                                                                         \
   } while (0)
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_VARINT_H */
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_VARINT_H */
diff --git a/src/core/transport/chttp2/writing.c b/src/core/ext/transport/chttp2/transport/writing.c
index 107725cbc7..6bb188f1da 100644
--- a/src/core/transport/chttp2/writing.c
+++ b/src/core/ext/transport/chttp2/transport/writing.c
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/transport/chttp2/internal.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 
 #include <limits.h>
 
 #include <grpc/support/log.h>
 
-#include "src/core/profiling/timers.h"
-#include "src/core/transport/chttp2/http2_errors.h"
+#include "src/core/ext/transport/chttp2/transport/http2_errors.h"
+#include "src/core/lib/profiling/timers.h"
 
 static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
                             grpc_chttp2_transport_writing *transport_writing);
@@ -161,8 +161,10 @@ int grpc_chttp2_unlocking_check_writes(
         transport_global->announce_incoming_window, UINT32_MAX);
     GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", transport_global,
                                      announce_incoming_window, announced);
-    gpr_slice_buffer_add(&transport_writing->outbuf,
-                         grpc_chttp2_window_update_create(0, announced));
+    grpc_transport_one_way_stats throwaway_stats;
+    gpr_slice_buffer_add(
+        &transport_writing->outbuf,
+        grpc_chttp2_window_update_create(0, announced, &throwaway_stats));
   }
 
   GPR_TIMER_END("grpc_chttp2_unlocking_check_writes", 0);
@@ -205,7 +207,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
     if (stream_writing->send_initial_metadata != NULL) {
       grpc_chttp2_encode_header(
           &transport_writing->hpack_compressor, stream_writing->id,
-          stream_writing->send_initial_metadata, 0, &transport_writing->outbuf);
+          stream_writing->send_initial_metadata, 0, &stream_writing->stats,
+          &transport_writing->outbuf);
       stream_writing->send_initial_metadata = NULL;
       stream_writing->sent_initial_metadata = 1;
     }
@@ -216,7 +219,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
       gpr_slice_buffer_add(
           &transport_writing->outbuf,
           grpc_chttp2_window_update_create(stream_writing->id,
-                                           stream_writing->announce_window));
+                                           stream_writing->announce_window,
+                                           &stream_writing->stats));
       GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", transport_writing, stream_writing,
                                     announce_window, announce);
       stream_writing->announce_window = 0;
@@ -255,7 +259,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
                                 stream_writing->send_trailing_metadata);
         grpc_chttp2_encode_data(
             stream_writing->id, &stream_writing->flow_controlled_buffer,
-            send_bytes, is_last_frame, &transport_writing->outbuf);
+            send_bytes, is_last_frame, &stream_writing->stats,
+            &transport_writing->outbuf);
         GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", transport_writing,
                                       stream_writing, outgoing_window,
                                       send_bytes);
@@ -281,19 +286,20 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
         stream_writing->send_trailing_metadata != NULL) {
       if (grpc_metadata_batch_is_empty(
               stream_writing->send_trailing_metadata)) {
-        grpc_chttp2_encode_data(stream_writing->id,
-                                &stream_writing->flow_controlled_buffer, 0, 1,
-                                &transport_writing->outbuf);
+        grpc_chttp2_encode_data(
+            stream_writing->id, &stream_writing->flow_controlled_buffer, 0, 1,
+            &stream_writing->stats, &transport_writing->outbuf);
       } else {
-        grpc_chttp2_encode_header(&transport_writing->hpack_compressor,
-                                  stream_writing->id,
-                                  stream_writing->send_trailing_metadata, 1,
-                                  &transport_writing->outbuf);
+        grpc_chttp2_encode_header(
+            &transport_writing->hpack_compressor, stream_writing->id,
+            stream_writing->send_trailing_metadata, 1, &stream_writing->stats,
+            &transport_writing->outbuf);
       }
       if (!transport_writing->is_client && !stream_writing->read_closed) {
         gpr_slice_buffer_add(&transport_writing->outbuf,
                              grpc_chttp2_rst_stream_create(
-                                 stream_writing->id, GRPC_CHTTP2_NO_ERROR));
+                                 stream_writing->id, GRPC_CHTTP2_NO_ERROR,
+                                 &stream_writing->stats));
       }
       stream_writing->send_trailing_metadata = NULL;
       stream_writing->sent_trailing_metadata = 1;
@@ -328,17 +334,21 @@ void grpc_chttp2_cleanup_writing(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
     if (stream_writing->sent_initial_metadata) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->send_initial_metadata_finished, 1);
+          exec_ctx, stream_global,
+          &stream_global->send_initial_metadata_finished, 1);
     }
+    grpc_transport_move_one_way_stats(&stream_writing->stats,
+                                      &stream_global->stats.outgoing);
     if (stream_writing->sent_message) {
       GPR_ASSERT(stream_writing->send_message == NULL);
       grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->send_message_finished, 1);
+          exec_ctx, stream_global, &stream_global->send_message_finished, 1);
       stream_writing->sent_message = 0;
     }
     if (stream_writing->sent_trailing_metadata) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->send_trailing_metadata_finished, 1);
+          exec_ctx, stream_global,
+          &stream_global->send_trailing_metadata_finished, 1);
     }
     if (stream_writing->sent_trailing_metadata) {
       grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
diff --git a/src/core/httpcli/parser.c b/src/core/httpcli/parser.c
deleted file mode 100644
index c314f025a0..0000000000
--- a/src/core/httpcli/parser.c
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/httpcli/parser.h"
-
-#include <string.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/useful.h>
-
-static int handle_response_line(grpc_httpcli_parser *parser) {
-  uint8_t *beg = parser->cur_line;
-  uint8_t *cur = beg;
-  uint8_t *end = beg + parser->cur_line_length;
-
-  if (cur == end || *cur++ != 'H') goto error;
-  if (cur == end || *cur++ != 'T') goto error;
-  if (cur == end || *cur++ != 'T') goto error;
-  if (cur == end || *cur++ != 'P') goto error;
-  if (cur == end || *cur++ != '/') goto error;
-  if (cur == end || *cur++ != '1') goto error;
-  if (cur == end || *cur++ != '.') goto error;
-  if (cur == end || *cur < '0' || *cur++ > '1') goto error;
-  if (cur == end || *cur++ != ' ') goto error;
-  if (cur == end || *cur < '1' || *cur++ > '9') goto error;
-  if (cur == end || *cur < '0' || *cur++ > '9') goto error;
-  if (cur == end || *cur < '0' || *cur++ > '9') goto error;
-  parser->r.status =
-      (cur[-3] - '0') * 100 + (cur[-2] - '0') * 10 + (cur[-1] - '0');
-  if (cur == end || *cur++ != ' ') goto error;
-
-  /* we don't really care about the status code message */
-
-  return 1;
-
-error:
-  gpr_log(GPR_ERROR, "Failed parsing response line");
-  return 0;
-}
-
-static char *buf2str(void *buffer, size_t length) {
-  char *out = gpr_malloc(length + 1);
-  memcpy(out, buffer, length);
-  out[length] = 0;
-  return out;
-}
-
-static int add_header(grpc_httpcli_parser *parser) {
-  uint8_t *beg = parser->cur_line;
-  uint8_t *cur = beg;
-  uint8_t *end = beg + parser->cur_line_length;
-  grpc_httpcli_header hdr = {NULL, NULL};
-
-  GPR_ASSERT(cur != end);
-
-  if (*cur == ' ' || *cur == '\t') {
-    gpr_log(GPR_ERROR, "Continued header lines not supported yet");
-    goto error;
-  }
-
-  while (cur != end && *cur != ':') {
-    cur++;
-  }
-  if (cur == end) {
-    gpr_log(GPR_ERROR, "Didn't find ':' in header string");
-    goto error;
-  }
-  GPR_ASSERT(cur >= beg);
-  hdr.key = buf2str(beg, (size_t)(cur - beg));
-  cur++; /* skip : */
-
-  while (cur != end && (*cur == ' ' || *cur == '\t')) {
-    cur++;
-  }
-  GPR_ASSERT(end - cur >= 2);
-  hdr.value = buf2str(cur, (size_t)(end - cur) - 2);
-
-  if (parser->r.hdr_count == parser->hdr_capacity) {
-    parser->hdr_capacity =
-        GPR_MAX(parser->hdr_capacity + 1, parser->hdr_capacity * 3 / 2);
-    parser->r.hdrs = gpr_realloc(
-        parser->r.hdrs, parser->hdr_capacity * sizeof(*parser->r.hdrs));
-  }
-  parser->r.hdrs[parser->r.hdr_count++] = hdr;
-  return 1;
-
-error:
-  gpr_free(hdr.key);
-  gpr_free(hdr.value);
-  return 0;
-}
-
-static int finish_line(grpc_httpcli_parser *parser) {
-  switch (parser->state) {
-    case GRPC_HTTPCLI_INITIAL_RESPONSE:
-      if (!handle_response_line(parser)) {
-        return 0;
-      }
-      parser->state = GRPC_HTTPCLI_HEADERS;
-      break;
-    case GRPC_HTTPCLI_HEADERS:
-      if (parser->cur_line_length == 2) {
-        parser->state = GRPC_HTTPCLI_BODY;
-        break;
-      }
-      if (!add_header(parser)) {
-        return 0;
-      }
-      break;
-    case GRPC_HTTPCLI_BODY:
-      GPR_UNREACHABLE_CODE(return 0);
-  }
-
-  parser->cur_line_length = 0;
-  return 1;
-}
-
-static int addbyte(grpc_httpcli_parser *parser, uint8_t byte) {
-  switch (parser->state) {
-    case GRPC_HTTPCLI_INITIAL_RESPONSE:
-    case GRPC_HTTPCLI_HEADERS:
-      if (parser->cur_line_length >= GRPC_HTTPCLI_MAX_HEADER_LENGTH) {
-        gpr_log(GPR_ERROR, "HTTP client max line length (%d) exceeded",
-                GRPC_HTTPCLI_MAX_HEADER_LENGTH);
-        return 0;
-      }
-      parser->cur_line[parser->cur_line_length] = byte;
-      parser->cur_line_length++;
-      if (parser->cur_line_length >= 2 &&
-          parser->cur_line[parser->cur_line_length - 2] == '\r' &&
-          parser->cur_line[parser->cur_line_length - 1] == '\n') {
-        return finish_line(parser);
-      } else {
-        return 1;
-      }
-      GPR_UNREACHABLE_CODE(return 0);
-    case GRPC_HTTPCLI_BODY:
-      if (parser->r.body_length == parser->body_capacity) {
-        parser->body_capacity = GPR_MAX(8, parser->body_capacity * 3 / 2);
-        parser->r.body =
-            gpr_realloc((void *)parser->r.body, parser->body_capacity);
-      }
-      parser->r.body[parser->r.body_length] = (char)byte;
-      parser->r.body_length++;
-      return 1;
-  }
-  GPR_UNREACHABLE_CODE(return 0);
-}
-
-void grpc_httpcli_parser_init(grpc_httpcli_parser *parser) {
-  memset(parser, 0, sizeof(*parser));
-  parser->state = GRPC_HTTPCLI_INITIAL_RESPONSE;
-  parser->r.status = 500;
-}
-
-void grpc_httpcli_parser_destroy(grpc_httpcli_parser *parser) {
-  size_t i;
-  gpr_free(parser->r.body);
-  for (i = 0; i < parser->r.hdr_count; i++) {
-    gpr_free(parser->r.hdrs[i].key);
-    gpr_free(parser->r.hdrs[i].value);
-  }
-  gpr_free(parser->r.hdrs);
-}
-
-int grpc_httpcli_parser_parse(grpc_httpcli_parser *parser, gpr_slice slice) {
-  size_t i;
-
-  for (i = 0; i < GPR_SLICE_LENGTH(slice); i++) {
-    if (!addbyte(parser, GPR_SLICE_START_PTR(slice)[i])) {
-      return 0;
-    }
-  }
-
-  return 1;
-}
-
-int grpc_httpcli_parser_eof(grpc_httpcli_parser *parser) {
-  return parser->state == GRPC_HTTPCLI_BODY;
-}
diff --git a/src/core/census/README.md b/src/core/lib/census/README.md
index fb615a2194..fb615a2194 100644
--- a/src/core/census/README.md
+++ b/src/core/lib/census/README.md
diff --git a/src/core/census/aggregation.h b/src/core/lib/census/aggregation.h
index e9bc6ada96..f353368b97 100644
--- a/src/core/census/aggregation.h
+++ b/src/core/lib/census/aggregation.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,8 +33,8 @@
 
 #include <stddef.h>
 
-#ifndef GRPC_INTERNAL_CORE_CENSUS_AGGREGATION_H
-#define GRPC_INTERNAL_CORE_CENSUS_AGGREGATION_H
+#ifndef GRPC_CORE_LIB_CENSUS_AGGREGATION_H
+#define GRPC_CORE_LIB_CENSUS_AGGREGATION_H
 
 /** Structure used to describe an aggregation type. */
 struct census_aggregation_ops {
@@ -63,4 +63,4 @@ struct census_aggregation_ops {
   size_t (*print)(const void *aggregation, char *buffer, size_t n);
 };
 
-#endif /* GRPC_INTERNAL_CORE_CENSUS_AGGREGATION_H */
+#endif /* GRPC_CORE_LIB_CENSUS_AGGREGATION_H */
diff --git a/src/core/census/context.c b/src/core/lib/census/context.c
index 89b8ee0b39..5a118f46a9 100644
--- a/src/core/census/context.c
+++ b/src/core/lib/census/context.c
@@ -38,7 +38,7 @@
 #include <grpc/support/useful.h>
 #include <stdbool.h>
 #include <string.h>
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 // Functions in this file support the public context API, including
 // encoding/decoding as part of context propagation across RPC's. The overall
diff --git a/src/core/census/grpc_context.c b/src/core/lib/census/grpc_context.c
index 4b61382a2c..457c176355 100644
--- a/src/core/census/grpc_context.c
+++ b/src/core/lib/census/grpc_context.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,8 +33,8 @@
 
 #include <grpc/census.h>
 #include <grpc/grpc.h>
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
 
 void grpc_census_call_set_context(grpc_call *call, census_context *context) {
   GRPC_API_TRACE("grpc_census_call_set_context(call=%p, census_context=%p)", 2,
diff --git a/src/core/census/grpc_filter.c b/src/core/lib/census/grpc_filter.c
index c8aaf31e2d..d27d789aa1 100644
--- a/src/core/census/grpc_filter.c
+++ b/src/core/lib/census/grpc_filter.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/census/grpc_filter.h"
+#include "src/core/lib/census/grpc_filter.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -42,10 +42,10 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/time.h>
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/statistics/census_interface.h"
-#include "src/core/statistics/census_rpc_stats.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/statistics/census_interface.h"
+#include "src/core/lib/statistics/census_rpc_stats.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 typedef struct call_data {
   census_op_id op_id;
@@ -172,13 +172,27 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 }
 
 const grpc_channel_filter grpc_client_census_filter = {
-    client_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    client_init_call_elem, grpc_call_stack_ignore_set_pollset,
-    client_destroy_call_elem, sizeof(channel_data), init_channel_elem,
-    destroy_channel_elem, grpc_call_next_get_peer, "census-client"};
+    client_start_transport_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    client_init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    client_destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "census-client"};
 
 const grpc_channel_filter grpc_server_census_filter = {
-    server_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    server_init_call_elem, grpc_call_stack_ignore_set_pollset,
-    server_destroy_call_elem, sizeof(channel_data), init_channel_elem,
-    destroy_channel_elem, grpc_call_next_get_peer, "census-server"};
+    server_start_transport_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    server_init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    server_destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "census-server"};
diff --git a/src/core/census/grpc_filter.h b/src/core/lib/census/grpc_filter.h
index b3de3adc94..7ceafe56e4 100644
--- a/src/core/census/grpc_filter.h
+++ b/src/core/lib/census/grpc_filter.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CENSUS_GRPC_FILTER_H
-#define GRPC_INTERNAL_CORE_CENSUS_GRPC_FILTER_H
+#ifndef GRPC_CORE_LIB_CENSUS_GRPC_FILTER_H
+#define GRPC_CORE_LIB_CENSUS_GRPC_FILTER_H
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 
 /* Census filters: provides tracing and stats collection functionalities. It
    needs to reside right below the surface filter in the channel stack. */
 extern const grpc_channel_filter grpc_client_census_filter;
 extern const grpc_channel_filter grpc_server_census_filter;
 
-#endif /* GRPC_INTERNAL_CORE_CENSUS_GRPC_FILTER_H */
+#endif /* GRPC_CORE_LIB_CENSUS_GRPC_FILTER_H */
diff --git a/src/core/lib/census/grpc_plugin.c b/src/core/lib/census/grpc_plugin.c
new file mode 100644
index 0000000000..12aca76745
--- /dev/null
+++ b/src/core/lib/census/grpc_plugin.c
@@ -0,0 +1,70 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/census/grpc_plugin.h"
+
+#include <limits.h>
+
+#include <grpc/census.h>
+
+#include "src/core/lib/census/grpc_filter.h"
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_init.h"
+
+static bool maybe_add_census_filter(grpc_channel_stack_builder *builder,
+                                    void *arg_must_be_null) {
+  const grpc_channel_args *args =
+      grpc_channel_stack_builder_get_channel_arguments(builder);
+  if (grpc_channel_args_is_census_enabled(args)) {
+    return grpc_channel_stack_builder_prepend_filter(
+        builder, &grpc_client_census_filter, NULL, NULL);
+  }
+  return true;
+}
+
+void census_grpc_plugin_init(void) {
+  /* Only initialize census if no one else has and some features are
+   * available. */
+  if (census_enabled() == CENSUS_FEATURE_NONE &&
+      census_supported() != CENSUS_FEATURE_NONE) {
+    if (census_initialize(census_supported())) { /* enable all features. */
+      gpr_log(GPR_ERROR, "Could not initialize census.");
+    }
+  }
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
+                                   maybe_add_census_filter, NULL);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_census_filter, NULL);
+}
+
+void census_grpc_plugin_destroy(void) { census_shutdown(); }
diff --git a/src/core/surface/init_secure.c b/src/core/lib/census/grpc_plugin.h
index fa20e91583..33e5f0b701 100644
--- a/src/core/surface/init_secure.c
+++ b/src/core/lib/census/grpc_plugin.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,10 @@
  *
  */
 
-#include "src/core/surface/init.h"
-#include "src/core/debug/trace.h"
-#include "src/core/security/secure_endpoint.h"
-#include "src/core/tsi/transport_security_interface.h"
+#ifndef GRPC_CORE_LIB_CENSUS_GRPC_PLUGIN_H
+#define GRPC_CORE_LIB_CENSUS_GRPC_PLUGIN_H
 
-void grpc_security_pre_init(void) {
-  grpc_register_tracer("secure_endpoint", &grpc_trace_secure_endpoint);
-  grpc_register_tracer("transport_security", &tsi_tracing_enabled);
-}
+void census_grpc_plugin_init(void);
+void census_grpc_plugin_destroy(void);
+
+#endif /* GRPC_CORE_LIB_CENSUS_GRPC_PLUGIN_H */
diff --git a/src/core/census/initialize.c b/src/core/lib/census/initialize.c
index ce7ec09b89..ce7ec09b89 100644
--- a/src/core/census/initialize.c
+++ b/src/core/lib/census/initialize.c
diff --git a/src/core/census/mlog.c b/src/core/lib/census/mlog.c
index a2cc46d3f2..9d47e80297 100644
--- a/src/core/census/mlog.c
+++ b/src/core/lib/census/mlog.c
@@ -88,7 +88,7 @@
 // include the name of the structure, which will be passed as the first
 // argument. E.g. cl_block_initialize() will initialize a cl_block.
 
-#include "src/core/census/mlog.h"
+#include "src/core/lib/census/mlog.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/cpu.h>
diff --git a/src/core/census/mlog.h b/src/core/lib/census/mlog.h
index aaba9e1535..7fbdeda986 100644
--- a/src/core/census/mlog.h
+++ b/src/core/lib/census/mlog.h
@@ -33,8 +33,8 @@
 
 /* A very fast in-memory log, optimized for multiple writers. */
 
-#ifndef GRPC_INTERNAL_CORE_CENSUS_MLOG_H
-#define GRPC_INTERNAL_CORE_CENSUS_MLOG_H
+#ifndef GRPC_CORE_LIB_CENSUS_MLOG_H
+#define GRPC_CORE_LIB_CENSUS_MLOG_H
 
 #include <grpc/support/port_platform.h>
 #include <stddef.h>
@@ -92,4 +92,4 @@ size_t census_log_remaining_space(void);
    out-of-space. */
 int64_t census_log_out_of_space_count(void);
 
-#endif /* GRPC_INTERNAL_CORE_CENSUS_LOG_H */
+#endif /* GRPC_CORE_LIB_CENSUS_MLOG_H */
diff --git a/src/core/census/operation.c b/src/core/lib/census/operation.c
index 5c58704372..315f9c3534 100644
--- a/src/core/census/operation.c
+++ b/src/core/lib/census/operation.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/census/placeholders.c b/src/core/lib/census/placeholders.c
index fe23d13971..fe23d13971 100644
--- a/src/core/census/placeholders.c
+++ b/src/core/lib/census/placeholders.c
diff --git a/src/core/census/rpc_metric_id.h b/src/core/lib/census/rpc_metric_id.h
index f199839511..aad0588fb3 100644
--- a/src/core/census/rpc_metric_id.h
+++ b/src/core/lib/census/rpc_metric_id.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef CENSUS_RPC_METRIC_ID_H
-#define CENSUS_RPC_METRIC_ID_H
+#ifndef GRPC_CORE_LIB_CENSUS_RPC_METRIC_ID_H
+#define GRPC_CORE_LIB_CENSUS_RPC_METRIC_ID_H
 
 /* Metric ID's used for RPC measurements. */
 /* Count of client requests sent. */
@@ -48,4 +48,4 @@
 /* Server side request latency. */
 #define CENSUS_METRIC_RPC_SERVER_LATENCY ((uint32_t)5)
 
-#endif /* CENSUS_RPC_METRIC_ID_H */
+#endif /* GRPC_CORE_LIB_CENSUS_RPC_METRIC_ID_H */
diff --git a/src/core/census/tracing.c b/src/core/lib/census/tracing.c
index 3b5d6dab2b..e508996af3 100644
--- a/src/core/census/tracing.c
+++ b/src/core/lib/census/tracing.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index bae7a90a01..1a02f1f4aa 100644
--- a/src/core/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -31,9 +31,9 @@
  *
  */
 
+#include "src/core/lib/channel/channel_args.h"
 #include <grpc/grpc.h>
-#include "src/core/channel/channel_args.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 #include <grpc/census.h>
 #include <grpc/support/alloc.h>
diff --git a/src/core/channel/channel_args.h b/src/core/lib/channel/channel_args.h
index b3a7c9f434..1ea202c543 100644
--- a/src/core/channel/channel_args.h
+++ b/src/core/lib/channel/channel_args.h
@@ -31,29 +31,29 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_ARGS_H
-#define GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_ARGS_H
+#ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_ARGS_H
+#define GRPC_CORE_LIB_CHANNEL_CHANNEL_ARGS_H
 
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 
-/* Copy some arguments */
+/** Copy the arguments in \a src into a new instance */
 grpc_channel_args *grpc_channel_args_copy(const grpc_channel_args *src);
 
-/* Copy some arguments, stably sorting keys */
-grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a);
+/** Copy the arguments in \a src into a new instance, stably sorting keys */
+grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *src);
 
-/** Copy some arguments and add the to_add parameter in the end.
-   If to_add is NULL, it is equivalent to call grpc_channel_args_copy. */
+/** Copy the arguments in \a src and append \a to_add. If \a to_add is NULL, it
+ * is equivalent to calling \a grpc_channel_args_copy. */
 grpc_channel_args *grpc_channel_args_copy_and_add(const grpc_channel_args *src,
                                                   const grpc_arg *to_add,
                                                   size_t num_to_add);
 
-/** Copy args from a then args from b into a new channel args */
+/** Concatenate args from \a a and \a b into a new instance */
 grpc_channel_args *grpc_channel_args_merge(const grpc_channel_args *a,
                                            const grpc_channel_args *b);
 
-/** Destroy arguments created by grpc_channel_args_copy */
+/** Destroy arguments created by \a grpc_channel_args_copy */
 void grpc_channel_args_destroy(grpc_channel_args *a);
 
 /** Reads census_enabled settings from channel args. Returns 1 if census_enabled
@@ -91,4 +91,4 @@ int grpc_channel_args_compression_algorithm_get_states(
 int grpc_channel_args_compare(const grpc_channel_args *a,
                               const grpc_channel_args *b);
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_ARGS_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_ARGS_H */
diff --git a/src/core/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c
index 3e61688364..52283e35fa 100644
--- a/src/core/channel/channel_stack.c
+++ b/src/core/lib/channel/channel_stack.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 #include <grpc/support/log.h>
 
 #include <stdlib.h>
diff --git a/src/core/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index c01050e717..b29bee411d 100644
--- a/src/core/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H
-#define GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H
+#ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H
+#define GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H
 
 /* A channel filter defines how operations on a channel are implemented.
    Channel filters are chained together to create full channels, and if those
@@ -45,8 +45,8 @@
 
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
-#include "src/core/debug/trace.h"
-#include "src/core/transport/transport.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef struct grpc_channel_element grpc_channel_element;
 typedef struct grpc_call_element grpc_call_element;
@@ -257,4 +257,4 @@ extern int grpc_trace_channel;
 #define GRPC_CALL_LOG_OP(sev, elem, op) \
   if (grpc_trace_channel) grpc_call_log_op(sev, elem, op)
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H */
diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c
new file mode 100644
index 0000000000..1ce0c4e07f
--- /dev/null
+++ b/src/core/lib/channel/channel_stack_builder.c
@@ -0,0 +1,258 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/channel/channel_stack_builder.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+
+int grpc_trace_channel_stack_builder = 0;
+
+typedef struct filter_node {
+  struct filter_node *next;
+  struct filter_node *prev;
+  const grpc_channel_filter *filter;
+  grpc_post_filter_create_init_func init;
+  void *init_arg;
+} filter_node;
+
+struct grpc_channel_stack_builder {
+  // sentinel nodes for filters that have been added
+  filter_node begin;
+  filter_node end;
+  // various set/get-able parameters
+  const grpc_channel_args *args;
+  grpc_transport *transport;
+  const char *name;
+};
+
+struct grpc_channel_stack_builder_iterator {
+  grpc_channel_stack_builder *builder;
+  filter_node *node;
+};
+
+grpc_channel_stack_builder *grpc_channel_stack_builder_create(void) {
+  grpc_channel_stack_builder *b = gpr_malloc(sizeof(*b));
+  memset(b, 0, sizeof(*b));
+
+  b->begin.filter = NULL;
+  b->end.filter = NULL;
+  b->begin.next = &b->end;
+  b->begin.prev = &b->end;
+  b->end.next = &b->begin;
+  b->end.prev = &b->begin;
+
+  return b;
+}
+
+static grpc_channel_stack_builder_iterator *create_iterator_at_filter_node(
+    grpc_channel_stack_builder *builder, filter_node *node) {
+  grpc_channel_stack_builder_iterator *it = gpr_malloc(sizeof(*it));
+  it->builder = builder;
+  it->node = node;
+  return it;
+}
+
+void grpc_channel_stack_builder_iterator_destroy(
+    grpc_channel_stack_builder_iterator *it) {
+  gpr_free(it);
+}
+
+grpc_channel_stack_builder_iterator *
+grpc_channel_stack_builder_create_iterator_at_first(
+    grpc_channel_stack_builder *builder) {
+  return create_iterator_at_filter_node(builder, &builder->begin);
+}
+
+grpc_channel_stack_builder_iterator *
+grpc_channel_stack_builder_create_iterator_at_last(
+    grpc_channel_stack_builder *builder) {
+  return create_iterator_at_filter_node(builder, &builder->end);
+}
+
+bool grpc_channel_stack_builder_move_next(
+    grpc_channel_stack_builder_iterator *iterator) {
+  if (iterator->node == &iterator->builder->end) return false;
+  iterator->node = iterator->node->next;
+  return true;
+}
+
+bool grpc_channel_stack_builder_move_prev(
+    grpc_channel_stack_builder_iterator *iterator) {
+  if (iterator->node == &iterator->builder->begin) return false;
+  iterator->node = iterator->node->prev;
+  return true;
+}
+
+bool grpc_channel_stack_builder_move_prev(
+    grpc_channel_stack_builder_iterator *iterator);
+
+void grpc_channel_stack_builder_set_name(grpc_channel_stack_builder *builder,
+                                         const char *name) {
+  GPR_ASSERT(builder->name == NULL);
+  builder->name = name;
+}
+
+void grpc_channel_stack_builder_set_channel_arguments(
+    grpc_channel_stack_builder *builder, const grpc_channel_args *args) {
+  GPR_ASSERT(builder->args == NULL);
+  builder->args = args;
+}
+
+void grpc_channel_stack_builder_set_transport(
+    grpc_channel_stack_builder *builder, grpc_transport *transport) {
+  GPR_ASSERT(builder->transport == NULL);
+  builder->transport = transport;
+}
+
+grpc_transport *grpc_channel_stack_builder_get_transport(
+    grpc_channel_stack_builder *builder) {
+  return builder->transport;
+}
+
+const grpc_channel_args *grpc_channel_stack_builder_get_channel_arguments(
+    grpc_channel_stack_builder *builder) {
+  return builder->args;
+}
+
+bool grpc_channel_stack_builder_append_filter(
+    grpc_channel_stack_builder *builder, const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func, void *user_data) {
+  grpc_channel_stack_builder_iterator *it =
+      grpc_channel_stack_builder_create_iterator_at_last(builder);
+  bool ok = grpc_channel_stack_builder_add_filter_before(
+      it, filter, post_init_func, user_data);
+  grpc_channel_stack_builder_iterator_destroy(it);
+  return ok;
+}
+
+bool grpc_channel_stack_builder_prepend_filter(
+    grpc_channel_stack_builder *builder, const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func, void *user_data) {
+  grpc_channel_stack_builder_iterator *it =
+      grpc_channel_stack_builder_create_iterator_at_first(builder);
+  bool ok = grpc_channel_stack_builder_add_filter_after(
+      it, filter, post_init_func, user_data);
+  grpc_channel_stack_builder_iterator_destroy(it);
+  return ok;
+}
+
+static void add_after(filter_node *before, const grpc_channel_filter *filter,
+                      grpc_post_filter_create_init_func post_init_func,
+                      void *user_data) {
+  filter_node *new = gpr_malloc(sizeof(*new));
+  new->next = before->next;
+  new->prev = before;
+  new->next->prev = new->prev->next = new;
+  new->filter = filter;
+  new->init = post_init_func;
+  new->init_arg = user_data;
+}
+
+bool grpc_channel_stack_builder_add_filter_before(
+    grpc_channel_stack_builder_iterator *iterator,
+    const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func, void *user_data) {
+  if (iterator->node == &iterator->builder->begin) return false;
+  add_after(iterator->node->prev, filter, post_init_func, user_data);
+  return true;
+}
+
+bool grpc_channel_stack_builder_add_filter_after(
+    grpc_channel_stack_builder_iterator *iterator,
+    const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func, void *user_data) {
+  if (iterator->node == &iterator->builder->end) return false;
+  add_after(iterator->node, filter, post_init_func, user_data);
+  return true;
+}
+
+void grpc_channel_stack_builder_destroy(grpc_channel_stack_builder *builder) {
+  filter_node *p = builder->begin.next;
+  while (p != &builder->end) {
+    filter_node *next = p->next;
+    gpr_free(p);
+    p = next;
+  }
+  gpr_free(builder);
+}
+
+void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
+                                        grpc_channel_stack_builder *builder,
+                                        size_t prefix_bytes, int initial_refs,
+                                        grpc_iomgr_cb_func destroy,
+                                        void *destroy_arg) {
+  // count the number of filters
+  size_t num_filters = 0;
+  for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) {
+    num_filters++;
+  }
+
+  // create an array of filters
+  const grpc_channel_filter **filters =
+      gpr_malloc(sizeof(*filters) * num_filters);
+  size_t i = 0;
+  for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) {
+    filters[i++] = p->filter;
+  }
+
+  // calculate the size of the channel stack
+  size_t channel_stack_size = grpc_channel_stack_size(filters, num_filters);
+
+  // allocate memory, with prefix_bytes followed by channel_stack_size
+  char *result = gpr_malloc(prefix_bytes + channel_stack_size);
+  // fetch a pointer to the channel stack
+  grpc_channel_stack *channel_stack =
+      (grpc_channel_stack *)(result + prefix_bytes);
+  // and initialize it
+  grpc_channel_stack_init(exec_ctx, initial_refs, destroy,
+                          destroy_arg == NULL ? result : destroy_arg, filters,
+                          num_filters, builder->args, builder->name,
+                          channel_stack);
+
+  // run post-initialization functions
+  i = 0;
+  for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) {
+    if (p->init != NULL) {
+      p->init(channel_stack, grpc_channel_stack_element(channel_stack, i),
+              p->init_arg);
+    }
+    i++;
+  }
+
+  grpc_channel_stack_builder_destroy(builder);
+  gpr_free((grpc_channel_filter **)filters);
+
+  return result;
+}
diff --git a/src/core/lib/channel/channel_stack_builder.h b/src/core/lib/channel/channel_stack_builder.h
new file mode 100644
index 0000000000..8532c4462a
--- /dev/null
+++ b/src/core/lib/channel/channel_stack_builder.h
@@ -0,0 +1,155 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_BUILDER_H
+#define GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_BUILDER_H
+
+#include <stdbool.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_stack.h"
+
+/// grpc_channel_stack_builder offers a programmatic interface to selected
+/// and order channel filters
+typedef struct grpc_channel_stack_builder grpc_channel_stack_builder;
+typedef struct grpc_channel_stack_builder_iterator
+    grpc_channel_stack_builder_iterator;
+
+/// Create a new channel stack builder
+grpc_channel_stack_builder *grpc_channel_stack_builder_create(void);
+
+/// Assign a name to the channel stack: \a name must be statically allocated
+void grpc_channel_stack_builder_set_name(grpc_channel_stack_builder *builder,
+                                         const char *name);
+
+/// Attach \a transport to the builder (does not take ownership)
+void grpc_channel_stack_builder_set_transport(
+    grpc_channel_stack_builder *builder, grpc_transport *transport);
+
+/// Fetch attached transport
+grpc_transport *grpc_channel_stack_builder_get_transport(
+    grpc_channel_stack_builder *builder);
+
+/// Set channel arguments: \a args must continue to exist until after
+/// grpc_channel_stack_builder_finish returns
+void grpc_channel_stack_builder_set_channel_arguments(
+    grpc_channel_stack_builder *builder, const grpc_channel_args *args);
+
+/// Return a borrowed pointer to the channel arguments
+const grpc_channel_args *grpc_channel_stack_builder_get_channel_arguments(
+    grpc_channel_stack_builder *builder);
+
+/// Begin iterating over already defined filters in the builder at the beginning
+grpc_channel_stack_builder_iterator *
+grpc_channel_stack_builder_create_iterator_at_first(
+    grpc_channel_stack_builder *builder);
+
+/// Begin iterating over already defined filters in the builder at the end
+grpc_channel_stack_builder_iterator *
+grpc_channel_stack_builder_create_iterator_at_last(
+    grpc_channel_stack_builder *builder);
+
+/// Is an iterator at the first element?
+bool grpc_channel_stack_builder_iterator_is_first(
+    grpc_channel_stack_builder_iterator *iterator);
+
+/// Is an iterator at the end?
+bool grpc_channel_stack_builder_iterator_is_end(
+    grpc_channel_stack_builder_iterator *iterator);
+
+/// Move an iterator to the next item
+bool grpc_channel_stack_builder_move_next(
+    grpc_channel_stack_builder_iterator *iterator);
+
+/// Move an iterator to the previous item
+bool grpc_channel_stack_builder_move_prev(
+    grpc_channel_stack_builder_iterator *iterator);
+
+typedef void (*grpc_post_filter_create_init_func)(
+    grpc_channel_stack *channel_stack, grpc_channel_element *elem, void *arg);
+
+/// Add \a filter to the stack, after \a iterator.
+/// Call \a post_init_func(..., \a user_data) once the channel stack is
+/// created.
+bool grpc_channel_stack_builder_add_filter_after(
+    grpc_channel_stack_builder_iterator *iterator,
+    const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func,
+    void *user_data) GRPC_MUST_USE_RESULT;
+
+/// Add \a filter to the stack, before \a iterator.
+/// Call \a post_init_func(..., \a user_data) once the channel stack is
+/// created.
+bool grpc_channel_stack_builder_add_filter_before(
+    grpc_channel_stack_builder_iterator *iterator,
+    const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func,
+    void *user_data) GRPC_MUST_USE_RESULT;
+
+/// Add \a filter to the beginning of the filter list.
+/// Call \a post_init_func(..., \a user_data) once the channel stack is
+/// created.
+bool grpc_channel_stack_builder_prepend_filter(
+    grpc_channel_stack_builder *builder, const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func,
+    void *user_data) GRPC_MUST_USE_RESULT;
+
+/// Add \a filter to the end of the filter list.
+/// Call \a post_init_func(..., \a user_data) once the channel stack is
+/// created.
+bool grpc_channel_stack_builder_append_filter(
+    grpc_channel_stack_builder *builder, const grpc_channel_filter *filter,
+    grpc_post_filter_create_init_func post_init_func,
+    void *user_data) GRPC_MUST_USE_RESULT;
+
+/// Terminate iteration and destroy \a iterator
+void grpc_channel_stack_builder_iterator_destroy(
+    grpc_channel_stack_builder_iterator *iterator);
+
+/// Destroy the builder, return the freshly minted channel stack
+/// Allocates \a prefix_bytes bytes before the channel stack
+/// Returns the base pointer of the allocated block
+/// \a initial_refs, \a destroy, \a destroy_arg are as per
+/// grpc_channel_stack_init
+void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
+                                        grpc_channel_stack_builder *builder,
+                                        size_t prefix_bytes, int initial_refs,
+                                        grpc_iomgr_cb_func destroy,
+                                        void *destroy_arg);
+
+/// Destroy the builder without creating a channel stack
+void grpc_channel_stack_builder_destroy(grpc_channel_stack_builder *builder);
+
+extern int grpc_trace_channel_stack_builder;
+
+#endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_BUILDER_H */
diff --git a/src/core/channel/client_channel.c b/src/core/lib/channel/client_channel.c
index d4ba950818..9fdf803ecf 100644
--- a/src/core/channel/client_channel.c
+++ b/src/core/lib/channel/client_channel.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/channel/client_channel.h"
+#include "src/core/lib/channel/client_channel.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -41,14 +41,14 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/connected_channel.h"
-#include "src/core/channel/subchannel_call_holder.h"
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/channel.h"
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/channel/subchannel_call_holder.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 /* Client channel implementation */
 
@@ -165,7 +165,6 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
   channel_data *chand = arg;
   grpc_lb_policy *lb_policy = NULL;
   grpc_lb_policy *old_lb_policy;
-  grpc_resolver *old_resolver;
   grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE;
   int exit_idle = 0;
 
@@ -201,28 +200,25 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
   }
 
   if (iomgr_success && chand->resolver) {
-    grpc_resolver *resolver = chand->resolver;
-    GRPC_RESOLVER_REF(resolver, "channel-next");
     grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state,
                                 "new_lb+resolver");
     if (lb_policy != NULL) {
       watch_lb_policy(exec_ctx, chand, lb_policy, state);
     }
-    gpr_mu_unlock(&chand->mu_config);
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-    grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration,
+    grpc_resolver_next(exec_ctx, chand->resolver,
+                       &chand->incoming_configuration,
                        &chand->on_config_changed);
-    GRPC_RESOLVER_UNREF(exec_ctx, resolver, "channel-next");
+    gpr_mu_unlock(&chand->mu_config);
   } else {
-    old_resolver = chand->resolver;
-    chand->resolver = NULL;
+    if (chand->resolver != NULL) {
+      grpc_resolver_shutdown(exec_ctx, chand->resolver);
+      GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
+      chand->resolver = NULL;
+    }
     grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
                                 GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone");
     gpr_mu_unlock(&chand->mu_config);
-    if (old_resolver != NULL) {
-      grpc_resolver_shutdown(exec_ctx, old_resolver);
-      GRPC_RESOLVER_UNREF(exec_ctx, old_resolver, "channel");
-    }
   }
 
   if (exit_idle) {
@@ -247,7 +243,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                   grpc_channel_element *elem,
                                   grpc_transport_op *op) {
   channel_data *chand = elem->channel_data;
-  grpc_resolver *destroy_resolver = NULL;
 
   grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL);
 
@@ -279,7 +274,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
   if (op->disconnect && chand->resolver != NULL) {
     grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
                                 GRPC_CHANNEL_FATAL_FAILURE, "disconnect");
-    destroy_resolver = chand->resolver;
+    grpc_resolver_shutdown(exec_ctx, chand->resolver);
+    GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
     chand->resolver = NULL;
     if (chand->lb_policy != NULL) {
       grpc_pollset_set_del_pollset_set(exec_ctx,
@@ -290,11 +286,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
     }
   }
   gpr_mu_unlock(&chand->mu_config);
-
-  if (destroy_resolver) {
-    grpc_resolver_shutdown(exec_ctx, destroy_resolver);
-    GRPC_RESOLVER_UNREF(exec_ctx, destroy_resolver, "channel");
-  }
 }
 
 typedef struct {
@@ -440,9 +431,17 @@ static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 const grpc_channel_filter grpc_client_channel_filter = {
-    cc_start_transport_stream_op, cc_start_transport_op, sizeof(call_data),
-    init_call_elem, cc_set_pollset, destroy_call_elem, sizeof(channel_data),
-    init_channel_elem, destroy_channel_elem, cc_get_peer, "client-channel",
+    cc_start_transport_stream_op,
+    cc_start_transport_op,
+    sizeof(call_data),
+    init_call_elem,
+    cc_set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    cc_get_peer,
+    "client-channel",
 };
 
 void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/channel/client_channel.h b/src/core/lib/channel/client_channel.h
index d9bc4971f1..8777796fb6 100644
--- a/src/core/channel/client_channel.h
+++ b/src/core/lib/channel/client_channel.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_CLIENT_CHANNEL_H
-#define GRPC_INTERNAL_CORE_CHANNEL_CLIENT_CHANNEL_H
+#ifndef GRPC_CORE_LIB_CHANNEL_CLIENT_CHANNEL_H
+#define GRPC_CORE_LIB_CHANNEL_CLIENT_CHANNEL_H
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/client_config/resolver.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/client_config/resolver.h"
 
 /* A client channel is a channel that begins disconnected, and can connect
    to some endpoint on demand. If that endpoint disconnects, it will be
@@ -60,4 +60,4 @@ void grpc_client_channel_watch_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
     grpc_connectivity_state *state, grpc_closure *on_complete);
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_CLIENT_CHANNEL_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_CLIENT_CHANNEL_H */
diff --git a/src/core/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c
index 3e7ca08fd2..04bb7cc76f 100644
--- a/src/core/channel/compress_filter.c
+++ b/src/core/lib/channel/compress_filter.c
@@ -39,13 +39,13 @@
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
 
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/compress_filter.h"
-#include "src/core/compression/algorithm_metadata.h"
-#include "src/core/compression/message_compress.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/compress_filter.h"
+#include "src/core/lib/compression/algorithm_metadata.h"
+#include "src/core/lib/compression/message_compress.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 typedef struct call_data {
   gpr_slice_buffer slices; /**< Buffers up input slices to be compressed */
@@ -291,7 +291,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {}
 
 const grpc_channel_filter grpc_compress_filter = {
-    compress_start_transport_stream_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem,
-    sizeof(channel_data), init_channel_elem, destroy_channel_elem,
-    grpc_call_next_get_peer, "compress"};
+    compress_start_transport_stream_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "compress"};
diff --git a/src/core/channel/compress_filter.h b/src/core/lib/channel/compress_filter.h
index 415459bca6..9010074335 100644
--- a/src/core/channel/compress_filter.h
+++ b/src/core/lib/channel/compress_filter.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_COMPRESS_FILTER_H
-#define GRPC_INTERNAL_CORE_CHANNEL_COMPRESS_FILTER_H
+#ifndef GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H
+#define GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 
 #define GRPC_COMPRESS_REQUEST_ALGORITHM_KEY "grpc-internal-encoding-request"
 
@@ -62,4 +62,4 @@
 
 extern const grpc_channel_filter grpc_compress_filter;
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_COMPRESS_FILTER_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H */
diff --git a/src/core/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c
index e8eb9dcfc5..5e3a8974ce 100644
--- a/src/core/channel/connected_channel.c
+++ b/src/core/lib/channel/connected_channel.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,19 +31,19 @@
  *
  */
 
-#include "src/core/channel/connected_channel.h"
+#include "src/core/lib/channel/connected_channel.h"
 
 #include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/support/string.h"
-#include "src/core/transport/transport.h"
-#include "src/core/profiling/timers.h"
 #include <grpc/byte_buffer.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/transport.h"
 
 #define MAX_BUFFER_LENGTH 8192
 
@@ -67,7 +67,6 @@ static void con_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
                                           grpc_transport_stream_op *op) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
   GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
 
   grpc_transport_perform_stream_op(exec_ctx, chand->transport,
@@ -88,7 +87,6 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   channel_data *chand = elem->channel_data;
   int r;
 
-  GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
   r = grpc_transport_init_stream(
       exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld),
       &args->call_stack->refcount, args->server_transport_data);
@@ -108,7 +106,6 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx,
                               grpc_call_element *elem) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
   grpc_transport_destroy_stream(exec_ctx, chand->transport,
                                 TRANSPORT_STREAM_FROM_CALL_DATA(calld));
 }
@@ -119,7 +116,6 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               grpc_channel_element_args *args) {
   channel_data *cd = (channel_data *)elem->channel_data;
   GPR_ASSERT(args->is_last);
-  GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
   cd->transport = NULL;
 }
 
@@ -127,7 +123,6 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {
   channel_data *cd = (channel_data *)elem->channel_data;
-  GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
   grpc_transport_destroy(exec_ctx, cd->transport);
 }
 
@@ -136,21 +131,26 @@ static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
   return grpc_transport_get_peer(exec_ctx, chand->transport);
 }
 
-const grpc_channel_filter grpc_connected_channel_filter = {
-    con_start_transport_stream_op, con_start_transport_op, sizeof(call_data),
-    init_call_elem, set_pollset, destroy_call_elem, sizeof(channel_data),
-    init_channel_elem, destroy_channel_elem, con_get_peer, "connected",
+static const grpc_channel_filter connected_channel_filter = {
+    con_start_transport_stream_op,
+    con_start_transport_op,
+    sizeof(call_data),
+    init_call_elem,
+    set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    con_get_peer,
+    "connected",
 };
 
-void grpc_connected_channel_bind_transport(grpc_channel_stack *channel_stack,
-                                           grpc_transport *transport) {
-  /* Assumes that the connected channel filter is always the last filter
-     in a channel stack */
-  grpc_channel_element *elem = grpc_channel_stack_last_element(channel_stack);
+static void bind_transport(grpc_channel_stack *channel_stack,
+                           grpc_channel_element *elem, void *t) {
   channel_data *cd = (channel_data *)elem->channel_data;
-  GPR_ASSERT(elem->filter == &grpc_connected_channel_filter);
+  GPR_ASSERT(elem->filter == &connected_channel_filter);
   GPR_ASSERT(cd->transport == NULL);
-  cd->transport = transport;
+  cd->transport = t;
 
   /* HACK(ctiller): increase call stack size for the channel to make space
      for channel data. We need a cleaner (but performant) way to do this,
@@ -158,7 +158,16 @@ void grpc_connected_channel_bind_transport(grpc_channel_stack *channel_stack,
      This is only "safe" because call stacks place no additional data after
      the last call element, and the last call element MUST be the connected
      channel. */
-  channel_stack->call_stack_size += grpc_transport_stream_size(transport);
+  channel_stack->call_stack_size += grpc_transport_stream_size(t);
+}
+
+bool grpc_add_connected_filter(grpc_channel_stack_builder *builder,
+                               void *arg_must_be_null) {
+  GPR_ASSERT(arg_must_be_null == NULL);
+  grpc_transport *t = grpc_channel_stack_builder_get_transport(builder);
+  GPR_ASSERT(t != NULL);
+  return grpc_channel_stack_builder_append_filter(
+      builder, &connected_channel_filter, bind_transport, t);
 }
 
 grpc_stream *grpc_connected_channel_get_stream(grpc_call_element *elem) {
diff --git a/src/core/surface/server_create.c b/src/core/lib/channel/connected_channel.h
index 5e37e80948..4f20b751cc 100644
--- a/src/core/surface/server_create.c
+++ b/src/core/lib/channel/connected_channel.h
@@ -31,18 +31,12 @@
  *
  */
 
-#include <grpc/grpc.h>
-#include "src/core/census/grpc_filter.h"
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/compress_filter.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/completion_queue.h"
-#include "src/core/surface/server.h"
+#ifndef GRPC_CORE_LIB_CHANNEL_CONNECTED_CHANNEL_H
+#define GRPC_CORE_LIB_CHANNEL_CONNECTED_CHANNEL_H
 
-grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
-  const grpc_channel_filter *filters[3];
-  size_t num_filters = 0;
-  filters[num_filters++] = &grpc_compress_filter;
-  GRPC_API_TRACE("grpc_server_create(%p, %p)", 2, (args, reserved));
-  return grpc_server_create_from_filters(filters, num_filters, args);
-}
+#include "src/core/lib/channel/channel_stack_builder.h"
+
+bool grpc_add_connected_filter(grpc_channel_stack_builder *builder,
+                               void *arg_must_be_null);
+
+#endif /* GRPC_CORE_LIB_CHANNEL_CONNECTED_CHANNEL_H */
diff --git a/src/core/channel/context.h b/src/core/lib/channel/context.h
index ac5796b9ef..bca102da9a 100644
--- a/src/core/channel/context.h
+++ b/src/core/lib/channel/context.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_CONTEXT_H
-#define GRPC_INTERNAL_CORE_CHANNEL_CONTEXT_H
+#ifndef GRPC_CORE_LIB_CHANNEL_CONTEXT_H
+#define GRPC_CORE_LIB_CHANNEL_CONTEXT_H
 
 /* Call object context pointers */
 typedef enum {
@@ -46,4 +46,4 @@ typedef struct {
   void (*destroy)(void *);
 } grpc_call_context_element;
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_CONTEXT_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_CONTEXT_H */
diff --git a/src/core/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c
index 1aa27208c2..b4e58b7f29 100644
--- a/src/core/channel/http_client_filter.c
+++ b/src/core/lib/channel/http_client_filter.c
@@ -30,14 +30,14 @@
  *
  */
 
-#include "src/core/channel/http_client_filter.h"
+#include "src/core/lib/channel/http_client_filter.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <string.h>
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 typedef struct call_data {
   grpc_linked_mdelem method;
@@ -112,7 +112,9 @@ static void hc_mutate_op(grpc_call_element *elem,
     /* Send : prefixed headers, which have to be before any application
        layer headers. */
     grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->method,
-                                 GRPC_MDELEM_METHOD_POST);
+                                 op->send_idempotent_request
+                                     ? GRPC_MDELEM_METHOD_PUT
+                                     : GRPC_MDELEM_METHOD_POST);
     grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->scheme,
                                  channeld->static_scheme);
     grpc_metadata_batch_add_tail(op->send_initial_metadata, &calld->te_trailers,
@@ -242,7 +244,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 }
 
 const grpc_channel_filter grpc_http_client_filter = {
-    hc_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem,
-    sizeof(channel_data), init_channel_elem, destroy_channel_elem,
-    grpc_call_next_get_peer, "http-client"};
+    hc_start_transport_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "http-client"};
diff --git a/src/core/channel/http_client_filter.h b/src/core/lib/channel/http_client_filter.h
index 21c66b9b8e..418426e9cc 100644
--- a/src/core/channel/http_client_filter.h
+++ b/src/core/lib/channel/http_client_filter.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_HTTP_CLIENT_FILTER_H
-#define GRPC_INTERNAL_CORE_CHANNEL_HTTP_CLIENT_FILTER_H
+#ifndef GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H
+#define GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 
 /* Processes metadata on the client side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_client_filter;
 
 #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme"
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_HTTP_CLIENT_FILTER_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H */
diff --git a/src/core/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c
index 370f8dbe42..db1a3d5010 100644
--- a/src/core/channel/http_server_filter.c
+++ b/src/core/lib/channel/http_server_filter.c
@@ -31,17 +31,17 @@
  *
  */
 
-#include "src/core/channel/http_server_filter.h"
+#include "src/core/lib/channel/http_server_filter.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <string.h>
-#include "src/core/profiling/timers.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 typedef struct call_data {
   uint8_t seen_path;
-  uint8_t seen_post;
+  uint8_t seen_method;
   uint8_t sent_status;
   uint8_t seen_scheme;
   uint8_t seen_te_trailers;
@@ -50,6 +50,7 @@ typedef struct call_data {
   grpc_linked_mdelem content_type;
 
   grpc_metadata_batch *recv_initial_metadata;
+  bool *recv_idempotent_request;
   /** Closure to call when finished with the hs_on_recv hook */
   grpc_closure *on_done_recv;
   /** Receive closures are chained: we inject this closure as the on_done_recv
@@ -72,11 +73,16 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
 
   /* Check if it is one of the headers we care about. */
   if (md == GRPC_MDELEM_TE_TRAILERS || md == GRPC_MDELEM_METHOD_POST ||
-      md == GRPC_MDELEM_SCHEME_HTTP || md == GRPC_MDELEM_SCHEME_HTTPS ||
+      md == GRPC_MDELEM_METHOD_PUT || md == GRPC_MDELEM_SCHEME_HTTP ||
+      md == GRPC_MDELEM_SCHEME_HTTPS ||
       md == GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC) {
     /* swallow it */
     if (md == GRPC_MDELEM_METHOD_POST) {
-      calld->seen_post = 1;
+      calld->seen_method = 1;
+      *calld->recv_idempotent_request = false;
+    } else if (md == GRPC_MDELEM_METHOD_PUT) {
+      calld->seen_method = 1;
+      *calld->recv_idempotent_request = true;
     } else if (md->key == GRPC_MDSTR_SCHEME) {
       calld->seen_scheme = 1;
     } else if (md == GRPC_MDELEM_TE_TRAILERS) {
@@ -142,7 +148,7 @@ static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
     /* Have we seen the required http2 transport headers?
        (:method, :scheme, content-type, with :path and :authority covered
        at the channel level right now) */
-    if (calld->seen_post && calld->seen_scheme && calld->seen_te_trailers &&
+    if (calld->seen_method && calld->seen_scheme && calld->seen_te_trailers &&
         calld->seen_path && calld->seen_authority) {
       /* do nothing */
     } else {
@@ -152,7 +158,7 @@ static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
       if (!calld->seen_authority) {
         gpr_log(GPR_ERROR, "Missing :authority header");
       }
-      if (!calld->seen_post) {
+      if (!calld->seen_method) {
         gpr_log(GPR_ERROR, "Missing :method header");
       }
       if (!calld->seen_scheme) {
@@ -185,7 +191,9 @@ static void hs_mutate_op(grpc_call_element *elem,
 
   if (op->recv_initial_metadata) {
     /* substitute our callback for the higher callback */
+    GPR_ASSERT(op->recv_idempotent_request != NULL);
     calld->recv_initial_metadata = op->recv_initial_metadata;
+    calld->recv_idempotent_request = op->recv_idempotent_request;
     calld->on_done_recv = op->recv_initial_metadata_ready;
     op->recv_initial_metadata_ready = &calld->hs_on_recv;
   }
@@ -227,7 +235,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {}
 
 const grpc_channel_filter grpc_http_server_filter = {
-    hs_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem,
-    sizeof(channel_data), init_channel_elem, destroy_channel_elem,
-    grpc_call_next_get_peer, "http-server"};
+    hs_start_transport_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "http-server"};
diff --git a/src/core/channel/http_server_filter.h b/src/core/lib/channel/http_server_filter.h
index f219d4e66f..c8cf920ded 100644
--- a/src/core/channel/http_server_filter.h
+++ b/src/core/lib/channel/http_server_filter.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_HTTP_SERVER_FILTER_H
-#define GRPC_INTERNAL_CORE_CHANNEL_HTTP_SERVER_FILTER_H
+#ifndef GRPC_CORE_LIB_CHANNEL_HTTP_SERVER_FILTER_H
+#define GRPC_CORE_LIB_CHANNEL_HTTP_SERVER_FILTER_H
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 
 /* Processes metadata on the client side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_server_filter;
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_HTTP_SERVER_FILTER_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_HTTP_SERVER_FILTER_H */
diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/lib/channel/subchannel_call_holder.c
index 8f46885a04..6c6d42dd73 100644
--- a/src/core/channel/subchannel_call_holder.c
+++ b/src/core/lib/channel/subchannel_call_holder.c
@@ -31,11 +31,11 @@
  *
  */
 
-#include "src/core/channel/subchannel_call_holder.h"
+#include "src/core/lib/channel/subchannel_call_holder.h"
 
 #include <grpc/support/alloc.h>
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
 #define GET_CALL(holder) \
   ((grpc_subchannel_call *)(gpr_atm_acq_load(&(holder)->subchannel_call)))
@@ -174,17 +174,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
   if (holder->connected_subchannel == NULL) {
     fail_locked(exec_ctx, holder);
+  } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) {
+    /* already cancelled before subchannel became ready */
+    fail_locked(exec_ctx, holder);
   } else {
-    if (!gpr_atm_rel_cas(
-            &holder->subchannel_call, 0,
-            (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
-                exec_ctx, holder->connected_subchannel, holder->pollset))) {
-      GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1);
-      /* if this cas fails, the call was cancelled before the pick completed */
-      fail_locked(exec_ctx, holder);
-    } else {
-      retry_waiting_locked(exec_ctx, holder);
-    }
+    gpr_atm_rel_store(
+        &holder->subchannel_call,
+        (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
+            exec_ctx, holder->connected_subchannel, holder->pollset));
+    retry_waiting_locked(exec_ctx, holder);
   }
   gpr_mu_unlock(&holder->mu);
   GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");
diff --git a/src/core/channel/subchannel_call_holder.h b/src/core/lib/channel/subchannel_call_holder.h
index 9cf72c6cf7..882f366792 100644
--- a/src/core/channel/subchannel_call_holder.h
+++ b/src/core/lib/channel/subchannel_call_holder.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_SUBCHANNEL_CALL_HOLDER_H
-#define GRPC_INTERNAL_CORE_CHANNEL_SUBCHANNEL_CALL_HOLDER_H
+#ifndef GRPC_CORE_LIB_CHANNEL_SUBCHANNEL_CALL_HOLDER_H
+#define GRPC_CORE_LIB_CHANNEL_SUBCHANNEL_CALL_HOLDER_H
 
-#include "src/core/client_config/subchannel.h"
+#include "src/core/lib/client_config/subchannel.h"
 
 /** Pick a subchannel for grpc_subchannel_call_holder;
     Return 1 if subchannel is available immediately (in which case on_ready
@@ -55,15 +55,14 @@ typedef enum {
     for initial metadata before trying to create a call object,
     and handling cancellation gracefully.
 
-    Both the channel and uchannel filter use this as their call_data. */
+    The channel filter uses this as their call_data. */
 typedef struct grpc_subchannel_call_holder {
   /** either 0 for no call, 1 for cancelled, or a pointer to a
       grpc_subchannel_call */
   gpr_atm subchannel_call;
   /** Helper function to choose the subchannel on which to create
       the call object. Channel filter delegates to the load
-      balancing policy (once it's ready); uchannel returns
-      immediately */
+      balancing policy (once it's ready). */
   grpc_subchannel_call_holder_pick_subchannel pick_subchannel;
   void *pick_subchannel_arg;
 
@@ -95,4 +94,4 @@ void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
 char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx,
                                            grpc_subchannel_call_holder *holder);
 
-#endif
+#endif /* GRPC_CORE_LIB_CHANNEL_SUBCHANNEL_CALL_HOLDER_H */
diff --git a/src/core/client_config/README.md b/src/core/lib/client_config/README.md
index fff7a5af5b..fff7a5af5b 100644
--- a/src/core/client_config/README.md
+++ b/src/core/lib/client_config/README.md
diff --git a/src/core/client_config/client_config.c b/src/core/lib/client_config/client_config.c
index 6ecffb3854..82c8d68099 100644
--- a/src/core/client_config/client_config.c
+++ b/src/core/lib/client_config/client_config.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/client_config.h"
+#include "src/core/lib/client_config/client_config.h"
 
 #include <string.h>
 
@@ -53,7 +53,9 @@ void grpc_client_config_ref(grpc_client_config *c) { gpr_ref(&c->refs); }
 
 void grpc_client_config_unref(grpc_exec_ctx *exec_ctx, grpc_client_config *c) {
   if (gpr_unref(&c->refs)) {
-    GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config");
+    if (c->lb_policy != NULL) {
+      GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config");
+    }
     gpr_free(c);
   }
 }
diff --git a/src/core/client_config/client_config.h b/src/core/lib/client_config/client_config.h
index 04bf036b00..404ec0d3a5 100644
--- a/src/core/client_config/client_config.h
+++ b/src/core/lib/client_config/client_config.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_CLIENT_CONFIG_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_CLIENT_CONFIG_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_CLIENT_CONFIG_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_CLIENT_CONFIG_H
 
-#include "src/core/client_config/lb_policy.h"
+#include "src/core/lib/client_config/lb_policy.h"
 
 /** Total configuration for a client. Provided, and updated, by
     grpc_resolver */
@@ -50,4 +50,4 @@ void grpc_client_config_set_lb_policy(grpc_client_config *client_config,
 grpc_lb_policy *grpc_client_config_get_lb_policy(
     grpc_client_config *client_config);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_CLIENT_CONFIG_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_CLIENT_CONFIG_H */
diff --git a/src/core/client_config/connector.c b/src/core/lib/client_config/connector.c
index aa34aa7fab..f51d862c6d 100644
--- a/src/core/client_config/connector.c
+++ b/src/core/lib/client_config/connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/connector.h"
+#include "src/core/lib/client_config/connector.h"
 
 grpc_connector* grpc_connector_ref(grpc_connector* connector) {
   connector->vtable->ref(connector);
diff --git a/src/core/client_config/connector.h b/src/core/lib/client_config/connector.h
index b91eb512c3..21b925aade 100644
--- a/src/core/client_config/connector.h
+++ b/src/core/lib/client_config/connector.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_CONNECTOR_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_CONNECTOR_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_CONNECTOR_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_CONNECTOR_H
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/iomgr/sockaddr.h"
-#include "src/core/transport/transport.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/iomgr/sockaddr.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef struct grpc_connector grpc_connector;
 typedef struct grpc_connector_vtable grpc_connector_vtable;
@@ -62,9 +62,6 @@ typedef struct {
 typedef struct {
   /** the connected transport */
   grpc_transport *transport;
-  /** any additional filters (owned by the caller of connect) */
-  const grpc_channel_filter **filters;
-  size_t num_filters;
 
   /** channel arguments (to be passed to the filters) */
   const grpc_channel_args *channel_args;
@@ -92,4 +89,4 @@ void grpc_connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *connector,
 void grpc_connector_shutdown(grpc_exec_ctx *exec_ctx,
                              grpc_connector *connector);
 
-#endif
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_CONNECTOR_H */
diff --git a/src/core/client_config/default_initial_connect_string.c b/src/core/lib/client_config/default_initial_connect_string.c
index 6a4e23e6f2..86eb37de77 100644
--- a/src/core/client_config/default_initial_connect_string.c
+++ b/src/core/lib/client_config/default_initial_connect_string.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,7 +32,7 @@
  */
 
 #include <grpc/support/slice.h>
-#include "src/core/iomgr/sockaddr.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
 void grpc_set_default_initial_connect_string(struct sockaddr **addr,
                                              size_t *addr_len,
diff --git a/src/core/client_config/initial_connect_string.c b/src/core/lib/client_config/initial_connect_string.c
index 19afa1675a..95ae728316 100644
--- a/src/core/client_config/initial_connect_string.c
+++ b/src/core/lib/client_config/initial_connect_string.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/initial_connect_string.h"
+#include "src/core/lib/client_config/initial_connect_string.h"
 
 #include <stddef.h>
 
diff --git a/src/core/client_config/initial_connect_string.h b/src/core/lib/client_config/initial_connect_string.h
index b6dca7134a..eec42fa240 100644
--- a/src/core/client_config/initial_connect_string.h
+++ b/src/core/lib/client_config/initial_connect_string.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H
 
 #include <grpc/support/slice.h>
-#include "src/core/iomgr/sockaddr.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
 typedef void (*grpc_set_initial_connect_string_func)(struct sockaddr **addr,
                                                      size_t *addr_len,
@@ -47,4 +47,4 @@ void grpc_test_set_initial_connect_string_function(
 void grpc_set_initial_connect_string(struct sockaddr **addr, size_t *addr_len,
                                      gpr_slice *connect_string);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H */
diff --git a/src/core/client_config/lb_policies/load_balancer_api.c b/src/core/lib/client_config/lb_policies/load_balancer_api.c
index a6b5785fe4..4cbed200df 100644
--- a/src/core/client_config/lb_policies/load_balancer_api.c
+++ b/src/core/lib/client_config/lb_policies/load_balancer_api.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/lb_policies/load_balancer_api.h"
+#include "src/core/lib/client_config/lb_policies/load_balancer_api.h"
 #include "third_party/nanopb/pb_decode.h"
 #include "third_party/nanopb/pb_encode.h"
 
diff --git a/src/core/client_config/lb_policies/load_balancer_api.h b/src/core/lib/client_config/lb_policies/load_balancer_api.h
index 4dbe1d6c22..83299adfa9 100644
--- a/src/core/client_config/lb_policies/load_balancer_api.h
+++ b/src/core/lib/client_config/lb_policies/load_balancer_api.h
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H
 
 #include <grpc/support/slice_buffer.h>
 
-#include "src/core/client_config/lb_policy_factory.h"
-#include "src/core/proto/grpc/lb/v0/load_balancer.pb.h"
+#include "src/core/lib/client_config/lb_policy_factory.h"
+#include "src/core/lib/proto/grpc/lb/v0/load_balancer.pb.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -82,4 +82,4 @@ void grpc_grpclb_response_destroy(grpc_grpclb_response *response);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H */
diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/lib/client_config/lb_policies/pick_first.c
index 81167b31c8..9ac550d9e7 100644
--- a/src/core/client_config/lb_policies/pick_first.c
+++ b/src/core/lib/client_config/lb_policies/pick_first.c
@@ -31,13 +31,13 @@
  *
  */
 
-#include "src/core/client_config/lb_policies/pick_first.h"
-#include "src/core/client_config/lb_policy_factory.h"
+#include "src/core/lib/client_config/lb_policies/pick_first.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/client_config/lb_policy_factory.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 typedef struct pending_pick {
   struct pending_pick *next;
@@ -378,26 +378,55 @@ void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
 }
 
 static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = {
-    pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_ping_one, pf_exit_idle,
-    pf_check_connectivity, pf_notify_on_state_change};
+    pf_destroy,
+    pf_shutdown,
+    pf_pick,
+    pf_cancel_pick,
+    pf_ping_one,
+    pf_exit_idle,
+    pf_check_connectivity,
+    pf_notify_on_state_change};
 
 static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {}
 
 static void pick_first_factory_unref(grpc_lb_policy_factory *factory) {}
 
-static grpc_lb_policy *create_pick_first(grpc_lb_policy_factory *factory,
+static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx,
+                                         grpc_lb_policy_factory *factory,
                                          grpc_lb_policy_args *args) {
+  GPR_ASSERT(args->addresses != NULL);
+  GPR_ASSERT(args->subchannel_factory != NULL);
+
+  if (args->addresses->naddrs == 0) return NULL;
+
   pick_first_lb_policy *p = gpr_malloc(sizeof(*p));
-  GPR_ASSERT(args->num_subchannels > 0);
   memset(p, 0, sizeof(*p));
-  grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable);
+
   p->subchannels =
-      gpr_malloc(sizeof(grpc_subchannel *) * args->num_subchannels);
-  p->num_subchannels = args->num_subchannels;
-  grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
-                               "pick_first");
-  memcpy(p->subchannels, args->subchannels,
-         sizeof(grpc_subchannel *) * args->num_subchannels);
+      gpr_malloc(sizeof(grpc_subchannel *) * args->addresses->naddrs);
+  memset(p->subchannels, 0, sizeof(*p->subchannels) * args->addresses->naddrs);
+  grpc_subchannel_args sc_args;
+  size_t subchannel_idx = 0;
+  for (size_t i = 0; i < args->addresses->naddrs; i++) {
+    memset(&sc_args, 0, sizeof(grpc_subchannel_args));
+    sc_args.addr = (struct sockaddr *)(args->addresses->addrs[i].addr);
+    sc_args.addr_len = (size_t)args->addresses->addrs[i].len;
+
+    grpc_subchannel *subchannel = grpc_subchannel_factory_create_subchannel(
+        exec_ctx, args->subchannel_factory, &sc_args);
+
+    if (subchannel != NULL) {
+      p->subchannels[subchannel_idx++] = subchannel;
+    }
+  }
+  if (subchannel_idx == 0) {
+    gpr_free(p->subchannels);
+    gpr_free(p);
+    return NULL;
+  }
+  p->num_subchannels = subchannel_idx;
+
+  grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable);
   grpc_closure_init(&p->connectivity_changed, pf_connectivity_changed, p);
   gpr_mu_init(&p->mu);
   return &p->base;
diff --git a/src/core/client_config/lb_policies/pick_first.h b/src/core/lib/client_config/lb_policies/pick_first.h
index 3ca53ad42a..dba86ea7ad 100644
--- a/src/core/client_config/lb_policies/pick_first.h
+++ b/src/core/lib/client_config/lb_policies/pick_first.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_PICK_FIRST_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_PICK_FIRST_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_PICK_FIRST_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_PICK_FIRST_H
 
-#include "src/core/client_config/lb_policy_factory.h"
+#include "src/core/lib/client_config/lb_policy_factory.h"
 
 /** Returns a load balancing factory for the pick first policy, which picks up
  * the first subchannel from \a subchannels to succesfully connect */
 grpc_lb_policy_factory *grpc_pick_first_lb_factory_create();
 
-#endif
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_PICK_FIRST_H */
diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/lib/client_config/lb_policies/round_robin.c
index 98d9acc75b..a4bc2c4786 100644
--- a/src/core/client_config/lb_policies/round_robin.c
+++ b/src/core/lib/client_config/lb_policies/round_robin.c
@@ -31,12 +31,12 @@
  *
  */
 
-#include "src/core/client_config/lb_policies/round_robin.h"
+#include "src/core/lib/client_config/lb_policies/round_robin.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 typedef struct round_robin_lb_policy round_robin_lb_policy;
 
@@ -483,37 +483,60 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
 }
 
 static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = {
-    rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_ping_one, rr_exit_idle,
-    rr_check_connectivity, rr_notify_on_state_change};
+    rr_destroy,
+    rr_shutdown,
+    rr_pick,
+    rr_cancel_pick,
+    rr_ping_one,
+    rr_exit_idle,
+    rr_check_connectivity,
+    rr_notify_on_state_change};
 
 static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {}
 
 static void round_robin_factory_unref(grpc_lb_policy_factory *factory) {}
 
-static grpc_lb_policy *create_round_robin(grpc_lb_policy_factory *factory,
+static grpc_lb_policy *create_round_robin(grpc_exec_ctx *exec_ctx,
+                                          grpc_lb_policy_factory *factory,
                                           grpc_lb_policy_args *args) {
-  size_t i;
+  GPR_ASSERT(args->addresses != NULL);
+  GPR_ASSERT(args->subchannel_factory != NULL);
+
   round_robin_lb_policy *p = gpr_malloc(sizeof(*p));
-  GPR_ASSERT(args->num_subchannels > 0);
   memset(p, 0, sizeof(*p));
-  grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable);
-  p->num_subchannels = args->num_subchannels;
-  p->subchannels = gpr_malloc(sizeof(*p->subchannels) * p->num_subchannels);
-  memset(p->subchannels, 0, sizeof(*p->subchannels) * p->num_subchannels);
-  grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
-                               "round_robin");
 
-  gpr_mu_init(&p->mu);
-  for (i = 0; i < args->num_subchannels; i++) {
-    subchannel_data *sd = gpr_malloc(sizeof(*sd));
-    memset(sd, 0, sizeof(*sd));
-    p->subchannels[i] = sd;
-    sd->policy = p;
-    sd->index = i;
-    sd->subchannel = args->subchannels[i];
-    grpc_closure_init(&sd->connectivity_changed_closure,
-                      rr_connectivity_changed, sd);
+  p->subchannels =
+      gpr_malloc(sizeof(*p->subchannels) * args->addresses->naddrs);
+  memset(p->subchannels, 0, sizeof(*p->subchannels) * args->addresses->naddrs);
+
+  grpc_subchannel_args sc_args;
+  size_t subchannel_idx = 0;
+  for (size_t i = 0; i < args->addresses->naddrs; i++) {
+    memset(&sc_args, 0, sizeof(grpc_subchannel_args));
+    sc_args.addr = (struct sockaddr *)(args->addresses->addrs[i].addr);
+    sc_args.addr_len = (size_t)args->addresses->addrs[i].len;
+
+    grpc_subchannel *subchannel = grpc_subchannel_factory_create_subchannel(
+        exec_ctx, args->subchannel_factory, &sc_args);
+
+    if (subchannel != NULL) {
+      subchannel_data *sd = gpr_malloc(sizeof(*sd));
+      memset(sd, 0, sizeof(*sd));
+      p->subchannels[subchannel_idx] = sd;
+      sd->policy = p;
+      sd->index = subchannel_idx;
+      sd->subchannel = subchannel;
+      ++subchannel_idx;
+      grpc_closure_init(&sd->connectivity_changed_closure,
+                        rr_connectivity_changed, sd);
+    }
+  }
+  if (subchannel_idx == 0) {
+    gpr_free(p->subchannels);
+    gpr_free(p);
+    return NULL;
   }
+  p->num_subchannels = subchannel_idx;
 
   /* The (dummy node) root of the ready list */
   p->ready_list.subchannel = NULL;
@@ -521,6 +544,10 @@ static grpc_lb_policy *create_round_robin(grpc_lb_policy_factory *factory,
   p->ready_list.next = NULL;
   p->ready_list_last_pick = &p->ready_list;
 
+  grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable);
+  grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
+                               "round_robin");
+  gpr_mu_init(&p->mu);
   return &p->base;
 }
 
diff --git a/src/core/client_config/lb_policies/round_robin.h b/src/core/lib/client_config/lb_policies/round_robin.h
index cf1f69c85f..52db1caa0c 100644
--- a/src/core/client_config/lb_policies/round_robin.h
+++ b/src/core/lib/client_config/lb_policies/round_robin.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,16 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_ROUND_ROBIN_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_ROUND_ROBIN_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_ROUND_ROBIN_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_ROUND_ROBIN_H
 
-#include "src/core/client_config/lb_policy.h"
+#include "src/core/lib/client_config/lb_policy.h"
 
 extern int grpc_lb_round_robin_trace;
 
-#include "src/core/client_config/lb_policy_factory.h"
+#include "src/core/lib/client_config/lb_policy_factory.h"
 
 /** Returns a load balancing factory for the round robin policy */
 grpc_lb_policy_factory *grpc_round_robin_lb_factory_create();
 
-#endif
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICIES_ROUND_ROBIN_H */
diff --git a/src/core/client_config/lb_policy.c b/src/core/lib/client_config/lb_policy.c
index 0d8b007336..ee20ccd76a 100644
--- a/src/core/client_config/lb_policy.c
+++ b/src/core/lib/client_config/lb_policy.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/lb_policy.h"
+#include "src/core/lib/client_config/lb_policy.h"
 
 #define WEAK_REF_BITS 16
 
diff --git a/src/core/client_config/lb_policy.h b/src/core/lib/client_config/lb_policy.h
index 3457390606..58a0a04d85 100644
--- a/src/core/client_config/lb_policy.h
+++ b/src/core/lib/client_config/lb_policy.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_H
 
-#include "src/core/client_config/subchannel.h"
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/client_config/subchannel.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 /** A load balancing policy: specified by a vtable and a struct (which
     is expected to be extended to contain some parameters) */
@@ -141,4 +141,4 @@ void grpc_lb_policy_notify_on_state_change(grpc_exec_ctx *exec_ctx,
 grpc_connectivity_state grpc_lb_policy_check_connectivity(
     grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 
-#endif /* GRPC_INTERNAL_CORE_CONFIG_LB_POLICY_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_H */
diff --git a/src/core/client_config/lb_policy_factory.c b/src/core/lib/client_config/lb_policy_factory.c
index e49de544e3..ede1d624af 100644
--- a/src/core/client_config/lb_policy_factory.c
+++ b/src/core/lib/client_config/lb_policy_factory.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/lb_policy_factory.h"
+#include "src/core/lib/client_config/lb_policy_factory.h"
 
 void grpc_lb_policy_factory_ref(grpc_lb_policy_factory* factory) {
   factory->vtable->ref(factory);
@@ -42,7 +42,8 @@ void grpc_lb_policy_factory_unref(grpc_lb_policy_factory* factory) {
 }
 
 grpc_lb_policy* grpc_lb_policy_factory_create_lb_policy(
-    grpc_lb_policy_factory* factory, grpc_lb_policy_args* args) {
+    grpc_exec_ctx* exec_ctx, grpc_lb_policy_factory* factory,
+    grpc_lb_policy_args* args) {
   if (factory == NULL) return NULL;
-  return factory->vtable->create_lb_policy(factory, args);
+  return factory->vtable->create_lb_policy(exec_ctx, factory, args);
 }
diff --git a/src/core/client_config/lb_policy_factory.h b/src/core/lib/client_config/lb_policy_factory.h
index 04610316ee..9a93a8ca3f 100644
--- a/src/core/client_config/lb_policy_factory.h
+++ b/src/core/lib/client_config/lb_policy_factory.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_FACTORY_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_FACTORY_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_FACTORY_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_FACTORY_H
 
-#include "src/core/client_config/lb_policy.h"
-#include "src/core/client_config/subchannel.h"
+#include "src/core/lib/client_config/lb_policy.h"
+#include "src/core/lib/client_config/subchannel_factory.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef struct grpc_lb_policy_factory grpc_lb_policy_factory;
 typedef struct grpc_lb_policy_factory_vtable grpc_lb_policy_factory_vtable;
@@ -47,8 +50,8 @@ struct grpc_lb_policy_factory {
 };
 
 typedef struct grpc_lb_policy_args {
-  grpc_subchannel **subchannels;
-  size_t num_subchannels;
+  grpc_resolved_addresses *addresses;
+  grpc_subchannel_factory *subchannel_factory;
 } grpc_lb_policy_args;
 
 struct grpc_lb_policy_factory_vtable {
@@ -56,7 +59,8 @@ struct grpc_lb_policy_factory_vtable {
   void (*unref)(grpc_lb_policy_factory *factory);
 
   /** Implementation of grpc_lb_policy_factory_create_lb_policy */
-  grpc_lb_policy *(*create_lb_policy)(grpc_lb_policy_factory *factory,
+  grpc_lb_policy *(*create_lb_policy)(grpc_exec_ctx *exec_ctx,
+                                      grpc_lb_policy_factory *factory,
                                       grpc_lb_policy_args *args);
 
   /** Name for the LB policy this factory implements */
@@ -68,6 +72,7 @@ void grpc_lb_policy_factory_unref(grpc_lb_policy_factory *factory);
 
 /** Create a lb_policy instance. */
 grpc_lb_policy *grpc_lb_policy_factory_create_lb_policy(
-    grpc_lb_policy_factory *factory, grpc_lb_policy_args *args);
+    grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory,
+    grpc_lb_policy_args *args);
 
-#endif /* GRPC_INTERNAL_CORE_CONFIG_LB_POLICY_FACTORY_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_FACTORY_H */
diff --git a/src/core/client_config/lb_policy_registry.c b/src/core/lib/client_config/lb_policy_registry.c
index fc302e82d7..f703e630a0 100644
--- a/src/core/client_config/lb_policy_registry.c
+++ b/src/core/lib/client_config/lb_policy_registry.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/lb_policy_registry.h"
+#include "src/core/lib/client_config/lb_policy_registry.h"
 
 #include <string.h>
 
@@ -79,10 +79,10 @@ static grpc_lb_policy_factory *lookup_factory(const char *name) {
   return NULL;
 }
 
-grpc_lb_policy *grpc_lb_policy_create(const char *name,
+grpc_lb_policy *grpc_lb_policy_create(grpc_exec_ctx *exec_ctx, const char *name,
                                       grpc_lb_policy_args *args) {
   grpc_lb_policy_factory *factory = lookup_factory(name);
   grpc_lb_policy *lb_policy =
-      grpc_lb_policy_factory_create_lb_policy(factory, args);
+      grpc_lb_policy_factory_create_lb_policy(exec_ctx, factory, args);
   return lb_policy;
 }
diff --git a/src/core/client_config/lb_policy_registry.h b/src/core/lib/client_config/lb_policy_registry.h
index 96fc2a1628..789854bd20 100644
--- a/src/core/client_config/lb_policy_registry.h
+++ b/src/core/lib/client_config/lb_policy_registry.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_REGISTRY_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_REGISTRY_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_REGISTRY_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_REGISTRY_H
 
-#include "src/core/client_config/lb_policy_factory.h"
+#include "src/core/lib/client_config/lb_policy_factory.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 /** Initialize the registry and set \a default_factory as the factory to be
  * returned when no name is provided in a lookup */
@@ -48,7 +49,7 @@ void grpc_register_lb_policy(grpc_lb_policy_factory *factory);
  *
  * If \a name is NULL, the default factory from \a grpc_lb_policy_registry_init
  * will be returned. */
-grpc_lb_policy *grpc_lb_policy_create(const char *name,
+grpc_lb_policy *grpc_lb_policy_create(grpc_exec_ctx *exec_ctx, const char *name,
                                       grpc_lb_policy_args *args);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICY_REGISTRY_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_LB_POLICY_REGISTRY_H */
diff --git a/src/core/client_config/resolver.c b/src/core/lib/client_config/resolver.c
index eda01e72ba..32f0643adb 100644
--- a/src/core/client_config/resolver.c
+++ b/src/core/lib/client_config/resolver.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/resolver.h"
+#include "src/core/lib/client_config/resolver.h"
 
 void grpc_resolver_init(grpc_resolver *resolver,
                         const grpc_resolver_vtable *vtable) {
diff --git a/src/core/client_config/resolver.h b/src/core/lib/client_config/resolver.h
index e612eaf3b3..1ee879293a 100644
--- a/src/core/client_config/resolver.h
+++ b/src/core/lib/client_config/resolver.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_H
 
-#include "src/core/client_config/client_config.h"
-#include "src/core/client_config/subchannel.h"
-#include "src/core/iomgr/iomgr.h"
+#include "src/core/lib/client_config/client_config.h"
+#include "src/core/lib/client_config/subchannel.h"
+#include "src/core/lib/iomgr/iomgr.h"
 
 typedef struct grpc_resolver grpc_resolver;
 typedef struct grpc_resolver_vtable grpc_resolver_vtable;
@@ -91,4 +91,4 @@ void grpc_resolver_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
                         grpc_client_config **target_config,
                         grpc_closure *on_complete);
 
-#endif /* GRPC_INTERNAL_CORE_CONFIG_RESOLVER_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_H */
diff --git a/src/core/client_config/resolver_factory.c b/src/core/lib/client_config/resolver_factory.c
index e7e9196ac4..0f76c664fa 100644
--- a/src/core/client_config/resolver_factory.c
+++ b/src/core/lib/client_config/resolver_factory.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/resolver_factory.h"
+#include "src/core/lib/client_config/resolver_factory.h"
 
 void grpc_resolver_factory_ref(grpc_resolver_factory* factory) {
   factory->vtable->ref(factory);
diff --git a/src/core/client_config/resolver_factory.h b/src/core/lib/client_config/resolver_factory.h
index 4c4df353f7..7765c3c844 100644
--- a/src/core/client_config/resolver_factory.h
+++ b/src/core/lib/client_config/resolver_factory.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_FACTORY_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_FACTORY_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_FACTORY_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_FACTORY_H
 
-#include "src/core/client_config/resolver.h"
-#include "src/core/client_config/subchannel_factory.h"
-#include "src/core/client_config/uri_parser.h"
+#include "src/core/lib/client_config/resolver.h"
+#include "src/core/lib/client_config/subchannel_factory.h"
+#include "src/core/lib/client_config/uri_parser.h"
 
 typedef struct grpc_resolver_factory grpc_resolver_factory;
 typedef struct grpc_resolver_factory_vtable grpc_resolver_factory_vtable;
@@ -79,4 +79,4 @@ grpc_resolver *grpc_resolver_factory_create_resolver(
 char *grpc_resolver_factory_get_default_authority(
     grpc_resolver_factory *factory, grpc_uri *uri);
 
-#endif /* GRPC_INTERNAL_CORE_CONFIG_RESOLVER_FACTORY_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_FACTORY_H */
diff --git a/src/core/client_config/resolver_registry.c b/src/core/lib/client_config/resolver_registry.c
index 89a945c2d3..29bd00c284 100644
--- a/src/core/client_config/resolver_registry.c
+++ b/src/core/lib/client_config/resolver_registry.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/resolver_registry.h"
+#include "src/core/lib/client_config/resolver_registry.h"
 
 #include <string.h>
 
diff --git a/src/core/client_config/resolver_registry.h b/src/core/lib/client_config/resolver_registry.h
index 5a7193b7ae..22289ca6bd 100644
--- a/src/core/client_config/resolver_registry.h
+++ b/src/core/lib/client_config/resolver_registry.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_REGISTRY_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_REGISTRY_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_REGISTRY_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_REGISTRY_H
 
-#include "src/core/client_config/resolver_factory.h"
+#include "src/core/lib/client_config/resolver_factory.h"
 
 void grpc_resolver_registry_init(const char *default_prefix);
 void grpc_resolver_registry_shutdown(void);
@@ -62,4 +62,4 @@ grpc_resolver *grpc_resolver_create(
     representing the default authority to pass from a client. */
 char *grpc_get_default_authority(const char *target);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVER_REGISTRY_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVER_REGISTRY_H */
diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/lib/client_config/resolvers/dns_resolver.c
index 376b6b3d76..62bccdd045 100644
--- a/src/core/client_config/resolvers/dns_resolver.c
+++ b/src/core/lib/client_config/resolvers/dns_resolver.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/resolvers/dns_resolver.h"
+#include "src/core/lib/client_config/resolvers/dns_resolver.h"
 
 #include <string.h>
 
@@ -39,9 +39,16 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/client_config/lb_policy_registry.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/client_config/lb_policy_registry.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/support/backoff.h"
+#include "src/core/lib/support/string.h"
+
+#define BACKOFF_MULTIPLIER 1.6
+#define BACKOFF_JITTER 0.2
+#define BACKOFF_MIN_SECONDS 1
+#define BACKOFF_MAX_SECONDS 120
 
 typedef struct {
   /** base class: must be first */
@@ -71,6 +78,11 @@ typedef struct {
   grpc_client_config **target_config;
   /** current (fully resolved) config */
   grpc_client_config *resolved_config;
+  /** retry timer */
+  bool have_retry_timer;
+  grpc_timer retry_timer;
+  /** retry backoff state */
+  gpr_backoff backoff_state;
 } dns_resolver;
 
 static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
@@ -91,6 +103,9 @@ static const grpc_resolver_vtable dns_resolver_vtable = {
 static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
   dns_resolver *r = (dns_resolver *)resolver;
   gpr_mu_lock(&r->mu);
+  if (r->have_retry_timer) {
+    grpc_timer_cancel(exec_ctx, &r->retry_timer);
+  }
   if (r->next_completion != NULL) {
     *r->target_config = NULL;
     grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL);
@@ -104,6 +119,7 @@ static void dns_channel_saw_error(grpc_exec_ctx *exec_ctx,
   dns_resolver *r = (dns_resolver *)resolver;
   gpr_mu_lock(&r->mu);
   if (!r->resolving) {
+    gpr_backoff_reset(&r->backoff_state);
     dns_start_resolving_locked(r);
   }
   gpr_mu_unlock(&r->mu);
@@ -118,6 +134,7 @@ static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
   r->next_completion = on_complete;
   r->target_config = target_config;
   if (r->resolved_version == 0 && !r->resolving) {
+    gpr_backoff_reset(&r->backoff_state);
     dns_start_resolving_locked(r);
   } else {
     dns_maybe_finish_next_locked(exec_ctx, r);
@@ -125,37 +142,55 @@ static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
   gpr_mu_unlock(&r->mu);
 }
 
+static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg,
+                               bool success) {
+  dns_resolver *r = arg;
+
+  gpr_mu_lock(&r->mu);
+  r->have_retry_timer = false;
+  if (success) {
+    if (!r->resolving) {
+      dns_start_resolving_locked(r);
+    }
+  }
+  gpr_mu_unlock(&r->mu);
+
+  GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer");
+}
+
 static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
                             grpc_resolved_addresses *addresses) {
   dns_resolver *r = arg;
   grpc_client_config *config = NULL;
-  grpc_subchannel **subchannels;
-  grpc_subchannel_args args;
   grpc_lb_policy *lb_policy;
-  size_t i;
-  if (addresses) {
+  gpr_mu_lock(&r->mu);
+  GPR_ASSERT(r->resolving);
+  r->resolving = 0;
+  if (addresses != NULL) {
     grpc_lb_policy_args lb_policy_args;
     config = grpc_client_config_create();
-    subchannels = gpr_malloc(sizeof(grpc_subchannel *) * addresses->naddrs);
-    for (i = 0; i < addresses->naddrs; i++) {
-      memset(&args, 0, sizeof(args));
-      args.addr = (struct sockaddr *)(addresses->addrs[i].addr);
-      args.addr_len = (size_t)addresses->addrs[i].len;
-      subchannels[i] = grpc_subchannel_factory_create_subchannel(
-          exec_ctx, r->subchannel_factory, &args);
-    }
     memset(&lb_policy_args, 0, sizeof(lb_policy_args));
-    lb_policy_args.subchannels = subchannels;
-    lb_policy_args.num_subchannels = addresses->naddrs;
-    lb_policy = grpc_lb_policy_create(r->lb_policy_name, &lb_policy_args);
-    grpc_client_config_set_lb_policy(config, lb_policy);
-    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction");
+    lb_policy_args.addresses = addresses;
+    lb_policy_args.subchannel_factory = r->subchannel_factory;
+    lb_policy =
+        grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args);
+    if (lb_policy != NULL) {
+      grpc_client_config_set_lb_policy(config, lb_policy);
+      GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction");
+    }
     grpc_resolved_addresses_destroy(addresses);
-    gpr_free(subchannels);
+  } else {
+    gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+    gpr_timespec next_try = gpr_backoff_step(&r->backoff_state, now);
+    gpr_timespec timeout = gpr_time_sub(next_try, now);
+    gpr_log(GPR_DEBUG, "dns resolution failed: retrying in %d.%09d seconds",
+            timeout.tv_sec, timeout.tv_nsec);
+    GPR_ASSERT(!r->have_retry_timer);
+    r->have_retry_timer = true;
+    GRPC_RESOLVER_REF(&r->base, "retry-timer");
+    grpc_timer_init(exec_ctx, &r->retry_timer, next_try, dns_on_retry_timer, r,
+                    now);
   }
-  gpr_mu_lock(&r->mu);
-  GPR_ASSERT(r->resolving);
-  r->resolving = 0;
   if (r->resolved_config) {
     grpc_client_config_unref(exec_ctx, r->resolved_config);
   }
@@ -222,6 +257,8 @@ static grpc_resolver *dns_create(grpc_resolver_args *args,
   r->name = gpr_strdup(path);
   r->default_port = gpr_strdup(default_port);
   r->subchannel_factory = args->subchannel_factory;
+  gpr_backoff_init(&r->backoff_state, BACKOFF_MULTIPLIER, BACKOFF_JITTER,
+                   BACKOFF_MIN_SECONDS * 1000, BACKOFF_MAX_SECONDS * 1000);
   grpc_subchannel_factory_ref(r->subchannel_factory);
   r->lb_policy_name = gpr_strdup(lb_policy_name);
   return &r->base;
diff --git a/src/core/client_config/resolvers/dns_resolver.h b/src/core/lib/client_config/resolvers/dns_resolver.h
index a3ef3161a6..eb46e41c77 100644
--- a/src/core/client_config/resolvers/dns_resolver.h
+++ b/src/core/lib/client_config/resolvers/dns_resolver.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_DNS_RESOLVER_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_DNS_RESOLVER_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_DNS_RESOLVER_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_DNS_RESOLVER_H
 
-#include "src/core/client_config/resolver_factory.h"
+#include "src/core/lib/client_config/resolver_factory.h"
 
 /** Create a dns resolver factory */
 grpc_resolver_factory *grpc_dns_resolver_factory_create(void);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_DNS_RESOLVER_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_DNS_RESOLVER_H */
diff --git a/src/core/client_config/resolvers/sockaddr_resolver.c b/src/core/lib/client_config/resolvers/sockaddr_resolver.c
index 68910ad975..c787bd57d6 100644
--- a/src/core/client_config/resolvers/sockaddr_resolver.c
+++ b/src/core/lib/client_config/resolvers/sockaddr_resolver.c
@@ -33,21 +33,19 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/client_config/resolvers/sockaddr_resolver.h"
+#include "src/core/lib/client_config/resolvers/sockaddr_resolver.h"
 
 #include <stdio.h>
 #include <string.h>
-#ifdef GPR_POSIX_SOCKET
-#include <sys/un.h>
-#endif
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/client_config/lb_policy_registry.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/client_config/lb_policy_registry.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/support/string.h"
 
 typedef struct {
   /** base class: must be first */
@@ -60,11 +58,7 @@ typedef struct {
   char *lb_policy_name;
 
   /** the addresses that we've 'resolved' */
-  struct sockaddr_storage *addrs;
-  /** the corresponding length of the addresses */
-  size_t *addrs_len;
-  /** how many elements in \a addrs */
-  size_t num_addrs;
+  grpc_resolved_addresses *addresses;
 
   /** mutex guarding the rest of the state */
   gpr_mu mu;
@@ -127,28 +121,14 @@ static void sockaddr_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
 
 static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
                                               sockaddr_resolver *r) {
-  grpc_client_config *cfg;
-  grpc_lb_policy *lb_policy;
-  grpc_lb_policy_args lb_policy_args;
-  grpc_subchannel **subchannels;
-  grpc_subchannel_args args;
-
   if (r->next_completion != NULL && !r->published) {
-    size_t i;
-    cfg = grpc_client_config_create();
-    subchannels = gpr_malloc(sizeof(grpc_subchannel *) * r->num_addrs);
-    for (i = 0; i < r->num_addrs; i++) {
-      memset(&args, 0, sizeof(args));
-      args.addr = (struct sockaddr *)&r->addrs[i];
-      args.addr_len = r->addrs_len[i];
-      subchannels[i] = grpc_subchannel_factory_create_subchannel(
-          exec_ctx, r->subchannel_factory, &args);
-    }
+    grpc_client_config *cfg = grpc_client_config_create();
+    grpc_lb_policy_args lb_policy_args;
     memset(&lb_policy_args, 0, sizeof(lb_policy_args));
-    lb_policy_args.subchannels = subchannels;
-    lb_policy_args.num_subchannels = r->num_addrs;
-    lb_policy = grpc_lb_policy_create(r->lb_policy_name, &lb_policy_args);
-    gpr_free(subchannels);
+    lb_policy_args.addresses = r->addresses;
+    lb_policy_args.subchannel_factory = r->subchannel_factory;
+    grpc_lb_policy *lb_policy =
+        grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args);
     grpc_client_config_set_lb_policy(cfg, lb_policy);
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "sockaddr");
     r->published = 1;
@@ -162,30 +142,11 @@ static void sockaddr_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) {
   sockaddr_resolver *r = (sockaddr_resolver *)gr;
   gpr_mu_destroy(&r->mu);
   grpc_subchannel_factory_unref(exec_ctx, r->subchannel_factory);
-  gpr_free(r->addrs);
-  gpr_free(r->addrs_len);
+  grpc_resolved_addresses_destroy(r->addresses);
   gpr_free(r->lb_policy_name);
   gpr_free(r);
 }
 
-#ifdef GPR_POSIX_SOCKET
-static int parse_unix(grpc_uri *uri, struct sockaddr_storage *addr,
-                      size_t *len) {
-  struct sockaddr_un *un = (struct sockaddr_un *)addr;
-
-  un->sun_family = AF_UNIX;
-  strcpy(un->sun_path, uri->path);
-  *len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
-
-  return 1;
-}
-
-static char *unix_get_default_authority(grpc_resolver_factory *factory,
-                                        grpc_uri *uri) {
-  return gpr_strdup("localhost");
-}
-#endif
-
 static char *ip_get_default_authority(grpc_uri *uri) {
   const char *path = uri->path;
   if (path[0] == '/') ++path;
@@ -289,7 +250,6 @@ static void do_nothing(void *ignored) {}
 static grpc_resolver *sockaddr_create(
     grpc_resolver_args *args, const char *default_lb_policy_name,
     int parse(grpc_uri *uri, struct sockaddr_storage *dst, size_t *len)) {
-  size_t i;
   int errors_found = 0; /* GPR_FALSE */
   sockaddr_resolver *r;
   gpr_slice path_slice;
@@ -329,15 +289,18 @@ static grpc_resolver *sockaddr_create(
   gpr_slice_buffer_init(&path_parts);
 
   gpr_slice_split(path_slice, ",", &path_parts);
-  r->num_addrs = path_parts.count;
-  r->addrs = gpr_malloc(sizeof(struct sockaddr_storage) * r->num_addrs);
-  r->addrs_len = gpr_malloc(sizeof(*r->addrs_len) * r->num_addrs);
+  r->addresses = gpr_malloc(sizeof(grpc_resolved_addresses));
+  r->addresses->naddrs = path_parts.count;
+  r->addresses->addrs =
+      gpr_malloc(sizeof(grpc_resolved_address) * r->addresses->naddrs);
 
-  for (i = 0; i < r->num_addrs; i++) {
+  for (size_t i = 0; i < r->addresses->naddrs; i++) {
     grpc_uri ith_uri = *args->uri;
     char *part_str = gpr_dump_slice(path_parts.slices[i], GPR_DUMP_ASCII);
     ith_uri.path = part_str;
-    if (!parse(&ith_uri, &r->addrs[i], &r->addrs_len[i])) {
+    if (!parse(&ith_uri,
+               (struct sockaddr_storage *)(&r->addresses->addrs[i].addr),
+               &r->addresses->addrs[i].len)) {
       errors_found = 1; /* GPR_TRUE */
     }
     gpr_free(part_str);
@@ -348,8 +311,7 @@ static grpc_resolver *sockaddr_create(
   gpr_slice_unref(path_slice);
   if (errors_found) {
     gpr_free(r->lb_policy_name);
-    gpr_free(r->addrs);
-    gpr_free(r->addrs_len);
+    grpc_resolved_addresses_destroy(r->addresses);
     gpr_free(r);
     return NULL;
   }
@@ -371,21 +333,22 @@ static void sockaddr_factory_ref(grpc_resolver_factory *factory) {}
 
 static void sockaddr_factory_unref(grpc_resolver_factory *factory) {}
 
-#define DECL_FACTORY(name)                                                  \
+#define DECL_FACTORY(name, prefix)                                          \
   static grpc_resolver *name##_factory_create_resolver(                     \
       grpc_resolver_factory *factory, grpc_resolver_args *args) {           \
-    return sockaddr_create(args, "pick_first", parse_##name);               \
+    return sockaddr_create(args, "pick_first", prefix##parse_##name);       \
   }                                                                         \
   static const grpc_resolver_factory_vtable name##_factory_vtable = {       \
       sockaddr_factory_ref, sockaddr_factory_unref,                         \
-      name##_factory_create_resolver, name##_get_default_authority, #name}; \
+      name##_factory_create_resolver, prefix##name##_get_default_authority, \
+      #name};                                                               \
   static grpc_resolver_factory name##_resolver_factory = {                  \
       &name##_factory_vtable};                                              \
   grpc_resolver_factory *grpc_##name##_resolver_factory_create() {          \
     return &name##_resolver_factory;                                        \
   }
 
-#ifdef GPR_POSIX_SOCKET
-DECL_FACTORY(unix)
+#ifdef GPR_HAVE_UNIX_SOCKET
+DECL_FACTORY(unix, grpc_)
 #endif
-DECL_FACTORY(ipv4) DECL_FACTORY(ipv6)
+DECL_FACTORY(ipv4, ) DECL_FACTORY(ipv6, )
diff --git a/src/core/client_config/resolvers/sockaddr_resolver.h b/src/core/lib/client_config/resolvers/sockaddr_resolver.h
index 1b7a18f9c2..45c55bd160 100644
--- a/src/core/client_config/resolvers/sockaddr_resolver.h
+++ b/src/core/lib/client_config/resolvers/sockaddr_resolver.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_UNIX_RESOLVER_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_UNIX_RESOLVER_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_SOCKADDR_RESOLVER_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_SOCKADDR_RESOLVER_H
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/client_config/resolver_factory.h"
+#include "src/core/lib/client_config/resolver_factory.h"
 
 grpc_resolver_factory *grpc_ipv4_resolver_factory_create(void);
 
@@ -47,4 +47,4 @@ grpc_resolver_factory *grpc_ipv6_resolver_factory_create(void);
 grpc_resolver_factory *grpc_unix_resolver_factory_create(void);
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_UNIX_RESOLVER_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_SOCKADDR_RESOLVER_H */
diff --git a/src/core/client_config/resolvers/zookeeper_resolver.c b/src/core/lib/client_config/resolvers/zookeeper_resolver.c
index 166738e768..404dfcd423 100644
--- a/src/core/client_config/resolvers/zookeeper_resolver.c
+++ b/src/core/lib/client_config/resolvers/zookeeper_resolver.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/resolvers/zookeeper_resolver.h"
+#include "src/core/lib/client_config/resolvers/zookeeper_resolver.h"
 
 #include <string.h>
 
@@ -41,12 +41,12 @@
 #include <grpc/grpc_zookeeper.h>
 #include <zookeeper/zookeeper.h>
 
-#include "src/core/client_config/lb_policy_registry.h"
-#include "src/core/client_config/resolver_registry.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/json/json.h"
+#include "src/core/lib/client_config/lb_policy_registry.h"
+#include "src/core/lib/client_config/resolver_registry.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
 
 /** Zookeeper session expiration time in milliseconds */
 #define GRPC_ZOOKEEPER_SESSION_TIMEOUT 15000
@@ -184,28 +184,22 @@ static void zookeeper_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
                                   grpc_resolved_addresses *addresses) {
   zookeeper_resolver *r = arg;
   grpc_client_config *config = NULL;
-  grpc_subchannel **subchannels;
-  grpc_subchannel_args args;
   grpc_lb_policy *lb_policy;
-  size_t i;
+
   if (addresses != NULL) {
     grpc_lb_policy_args lb_policy_args;
     config = grpc_client_config_create();
-    subchannels = gpr_malloc(sizeof(grpc_subchannel *) * addresses->naddrs);
-    for (i = 0; i < addresses->naddrs; i++) {
-      memset(&args, 0, sizeof(args));
-      args.addr = (struct sockaddr *)(addresses->addrs[i].addr);
-      args.addr_len = addresses->addrs[i].len;
-      subchannels[i] = grpc_subchannel_factory_create_subchannel(
-          exec_ctx, r->subchannel_factory, &args);
+
+    lb_policy_args.addresses = addresses;
+    lb_policy_args.subchannel_factory = r->subchannel_factory;
+    lb_policy =
+        grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args);
+
+    if (lb_policy != NULL) {
+      grpc_client_config_set_lb_policy(config, lb_policy);
+      GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction");
     }
-    lb_policy_args.subchannels = subchannels;
-    lb_policy_args.num_subchannels = addresses->naddrs;
-    lb_policy = grpc_lb_policy_create(r->lb_policy_name, &lb_policy_args);
-    grpc_client_config_set_lb_policy(config, lb_policy);
-    GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction");
     grpc_resolved_addresses_destroy(addresses);
-    gpr_free(subchannels);
   }
   gpr_mu_lock(&r->mu);
   GPR_ASSERT(r->resolving == 1);
diff --git a/src/core/client_config/resolvers/zookeeper_resolver.h b/src/core/lib/client_config/resolvers/zookeeper_resolver.h
index a6f002dd6d..7ee7604360 100644
--- a/src/core/client_config/resolvers/zookeeper_resolver.h
+++ b/src/core/lib/client_config/resolvers/zookeeper_resolver.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_ZOOKEEPER_RESOLVER_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_ZOOKEEPER_RESOLVER_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_ZOOKEEPER_RESOLVER_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_ZOOKEEPER_RESOLVER_H
 
-#include "src/core/client_config/resolver_factory.h"
+#include "src/core/lib/client_config/resolver_factory.h"
 
 /** Create a zookeeper resolver factory */
 grpc_resolver_factory *grpc_zookeeper_resolver_factory_create(void);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_RESOLVERS_ZOOKEEPER_RESOLVER_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_RESOLVERS_ZOOKEEPER_RESOLVER_H */
diff --git a/src/core/client_config/subchannel.c b/src/core/lib/client_config/subchannel.c
index 055f69c70e..5203ad2143 100644
--- a/src/core/client_config/subchannel.c
+++ b/src/core/lib/client_config/subchannel.c
@@ -31,22 +31,24 @@
  *
  */
 
-#include "src/core/client_config/subchannel.h"
+#include "src/core/lib/client_config/subchannel.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/avl.h>
 
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/client_channel.h"
-#include "src/core/channel/connected_channel.h"
-#include "src/core/client_config/initial_connect_string.h"
-#include "src/core/client_config/subchannel_index.h"
-#include "src/core/iomgr/timer.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/surface/channel.h"
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/client_channel.h"
+#include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/client_config/initial_connect_string.h"
+#include "src/core/lib/client_config/subchannel_index.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/backoff.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 #define INTERNAL_REF_BITS 16
 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1))
@@ -127,8 +129,8 @@ struct grpc_subchannel {
 
   /** next connect attempt time */
   gpr_timespec next_attempt;
-  /** amount to backoff each failure */
-  gpr_timespec backoff_delta;
+  /** backoff state */
+  gpr_backoff backoff_state;
   /** do we have an active alarm? */
   int have_alarm;
   /** our alarm */
@@ -146,7 +148,6 @@ struct grpc_subchannel_call {
 #define CALLSTACK_TO_SUBCHANNEL_CALL(callstack) \
   (((grpc_subchannel_call *)(callstack)) - 1)
 
-static gpr_timespec compute_connect_deadline(grpc_subchannel *c);
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *subchannel,
                                  bool iomgr_success);
 
@@ -184,8 +185,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_free(c);
 }
 
-void grpc_connected_subchannel_ref(grpc_connected_subchannel *c
-                                       GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+void grpc_connected_subchannel_ref(
+    grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON);
 }
 
@@ -226,8 +227,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta,
   return old_val;
 }
 
-grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c
-                                         GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+grpc_subchannel *grpc_subchannel_ref(
+    grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   gpr_atm old_refs;
   old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS),
                         0 REF_MUTATE_PURPOSE("STRONG_REF"));
@@ -235,8 +236,8 @@ grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c
   return c;
 }
 
-grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c
-                                              GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+grpc_subchannel *grpc_subchannel_weak_ref(
+    grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   gpr_atm old_refs;
   old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF"));
   GPR_ASSERT(old_refs != 0);
@@ -337,6 +338,41 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
   grpc_closure_init(&c->connected, subchannel_connected, c);
   grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE,
                                "subchannel");
+  gpr_backoff_init(&c->backoff_state,
+                   GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER,
+                   GRPC_SUBCHANNEL_RECONNECT_JITTER,
+                   GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS * 1000,
+                   GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000);
+  if (c->args) {
+    for (size_t i = 0; i < c->args->num_args; i++) {
+      if (0 == strcmp(c->args->args[i].key,
+                      "grpc.testing.fixed_reconnect_backoff")) {
+        GPR_ASSERT(c->args->args[i].type == GRPC_ARG_INTEGER);
+        gpr_backoff_init(&c->backoff_state, 1.0, 0.0,
+                         c->args->args[i].value.integer,
+                         c->args->args[i].value.integer);
+      }
+      if (0 ==
+          strcmp(c->args->args[i].key, GRPC_ARG_MAX_RECONNECT_BACKOFF_MS)) {
+        if (c->args->args[i].type == GRPC_ARG_INTEGER) {
+          if (c->args->args[i].value.integer >= 0) {
+            gpr_backoff_init(
+                &c->backoff_state, GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER,
+                GRPC_SUBCHANNEL_RECONNECT_JITTER,
+                GPR_MIN(c->args->args[i].value.integer,
+                        GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS * 1000),
+                c->args->args[i].value.integer);
+          } else {
+            gpr_log(GPR_ERROR, GRPC_ARG_MAX_RECONNECT_BACKOFF_MS
+                    " : must be non-negative");
+          }
+        } else {
+          gpr_log(GPR_ERROR,
+                  GRPC_ARG_MAX_RECONNECT_BACKOFF_MS " : must be an integer");
+        }
+      }
+    }
+  }
   gpr_mu_init(&c->mu);
 
   return grpc_subchannel_index_register(exec_ctx, key, c);
@@ -348,7 +384,7 @@ static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   args.interested_parties = c->pollset_set;
   args.addr = c->addr;
   args.addr_len = c->addr_len;
-  args.deadline = compute_connect_deadline(c);
+  args.deadline = c->next_attempt;
   args.channel_args = c->args;
   args.initial_connect_string = c->initial_connect_string;
 
@@ -359,10 +395,8 @@ static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
 }
 
 static void start_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
-  c->backoff_delta = gpr_time_from_seconds(
-      GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS, GPR_TIMESPAN);
   c->next_attempt =
-      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), c->backoff_delta);
+      gpr_backoff_begin(&c->backoff_state, gpr_now(GPR_CLOCK_MONOTONIC));
   continue_connect(exec_ctx, c);
 }
 
@@ -505,33 +539,17 @@ void grpc_connected_subchannel_ping(grpc_exec_ctx *exec_ctx,
   elem->filter->start_transport_op(exec_ctx, elem, &op);
 }
 
-static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
-  size_t channel_stack_size;
+static void publish_transport_locked(grpc_exec_ctx *exec_ctx,
+                                     grpc_subchannel *c) {
   grpc_connected_subchannel *con;
   grpc_channel_stack *stk;
-  size_t num_filters;
-  const grpc_channel_filter **filters;
   state_watcher *sw_subchannel;
 
-  /* build final filter list */
-  num_filters = c->num_filters + c->connecting_result.num_filters + 1;
-  filters = gpr_malloc(sizeof(*filters) * num_filters);
-  if (c->num_filters > 0) {
-    memcpy((void *)filters, c->filters, sizeof(*filters) * c->num_filters);
-  }
-  memcpy((void *)(filters + c->num_filters), c->connecting_result.filters,
-         sizeof(*filters) * c->connecting_result.num_filters);
-  filters[num_filters - 1] = &grpc_connected_channel_filter;
-
   /* construct channel stack */
-  channel_stack_size = grpc_channel_stack_size(filters, num_filters);
-  con = gpr_malloc(channel_stack_size);
+  con = grpc_channel_init_create_stack(
+      exec_ctx, GRPC_CLIENT_SUBCHANNEL, 0, c->connecting_result.channel_args, 1,
+      connection_destroy, NULL, c->connecting_result.transport);
   stk = CHANNEL_STACK_FROM_CONNECTION(con);
-  grpc_channel_stack_init(exec_ctx, 1, connection_destroy, con, filters,
-                          num_filters, c->connecting_result.channel_args,
-                          "CONNECTED_SUBCHANNEL", stk);
-  grpc_connected_channel_bind_transport(stk, c->connecting_result.transport);
-  gpr_free((void *)c->connecting_result.filters);
   memset(&c->connecting_result, 0, sizeof(c->connecting_result));
 
   /* initialize state watcher */
@@ -541,12 +559,8 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   grpc_closure_init(&sw_subchannel->closure, subchannel_on_child_state_changed,
                     sw_subchannel);
 
-  gpr_mu_lock(&c->mu);
-
   if (c->disconnected) {
-    gpr_mu_unlock(&c->mu);
     gpr_free(sw_subchannel);
-    gpr_free((void *)filters);
     grpc_channel_stack_destroy(exec_ctx, stk);
     gpr_free(con);
     GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
@@ -574,67 +588,6 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   /* signal completion */
   grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_READY,
                               "connected");
-
-  gpr_mu_unlock(&c->mu);
-  gpr_free((void *)filters);
-}
-
-/* Generate a random number between 0 and 1. */
-static double generate_uniform_random_number(grpc_subchannel *c) {
-  c->random = (1103515245 * c->random + 12345) % ((uint32_t)1 << 31);
-  return c->random / (double)((uint32_t)1 << 31);
-}
-
-/* Update backoff_delta and next_attempt in subchannel */
-static void update_reconnect_parameters(grpc_subchannel *c) {
-  size_t i;
-  int32_t backoff_delta_millis, jitter;
-  int32_t max_backoff_millis =
-      GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000;
-  double jitter_range;
-
-  if (c->args) {
-    for (i = 0; i < c->args->num_args; i++) {
-      if (0 == strcmp(c->args->args[i].key,
-                      "grpc.testing.fixed_reconnect_backoff")) {
-        GPR_ASSERT(c->args->args[i].type == GRPC_ARG_INTEGER);
-        c->next_attempt = gpr_time_add(
-            gpr_now(GPR_CLOCK_MONOTONIC),
-            gpr_time_from_millis(c->args->args[i].value.integer, GPR_TIMESPAN));
-        return;
-      }
-      if (0 ==
-          strcmp(c->args->args[i].key, GRPC_ARG_MAX_RECONNECT_BACKOFF_MS)) {
-        if (c->args->args[i].type == GRPC_ARG_INTEGER) {
-          if (c->args->args[i].value.integer >= 0) {
-            max_backoff_millis = c->args->args[i].value.integer;
-          } else {
-            gpr_log(GPR_ERROR, GRPC_ARG_MAX_RECONNECT_BACKOFF_MS
-                    " : must be non-negative");
-          }
-        } else {
-          gpr_log(GPR_ERROR,
-                  GRPC_ARG_MAX_RECONNECT_BACKOFF_MS " : must be an integer");
-        }
-      }
-    }
-  }
-
-  backoff_delta_millis =
-      (int32_t)(gpr_time_to_millis(c->backoff_delta) *
-                GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER);
-  if (backoff_delta_millis > max_backoff_millis) {
-    backoff_delta_millis = max_backoff_millis;
-  }
-  c->backoff_delta = gpr_time_from_millis(backoff_delta_millis, GPR_TIMESPAN);
-  c->next_attempt =
-      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), c->backoff_delta);
-
-  jitter_range = GRPC_SUBCHANNEL_RECONNECT_JITTER * backoff_delta_millis;
-  jitter =
-      (int32_t)((2 * generate_uniform_random_number(c) - 1) * jitter_range);
-  c->next_attempt =
-      gpr_time_add(c->next_attempt, gpr_time_from_millis(jitter, GPR_TIMESPAN));
 }
 
 static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) {
@@ -645,7 +598,8 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) {
     iomgr_success = 0;
   }
   if (iomgr_success) {
-    update_reconnect_parameters(c);
+    c->next_attempt =
+        gpr_backoff_step(&c->backoff_state, gpr_now(GPR_CLOCK_MONOTONIC));
     continue_connect(exec_ctx, c);
     gpr_mu_unlock(&c->mu);
   } else {
@@ -658,32 +612,23 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
                                  bool iomgr_success) {
   grpc_subchannel *c = arg;
 
+  GRPC_SUBCHANNEL_WEAK_REF(c, "connected");
+  gpr_mu_lock(&c->mu);
   if (c->connecting_result.transport != NULL) {
-    publish_transport(exec_ctx, c);
+    publish_transport_locked(exec_ctx, c);
   } else if (c->disconnected) {
     GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   } else {
     gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
-    gpr_mu_lock(&c->mu);
     GPR_ASSERT(!c->have_alarm);
     c->have_alarm = 1;
     grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
                                 GRPC_CHANNEL_TRANSIENT_FAILURE,
                                 "connect_failed");
     grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now);
-    gpr_mu_unlock(&c->mu);
   }
-}
-
-static gpr_timespec compute_connect_deadline(grpc_subchannel *c) {
-  gpr_timespec current_deadline =
-      gpr_time_add(c->next_attempt, c->backoff_delta);
-  gpr_timespec min_deadline = gpr_time_add(
-      gpr_now(GPR_CLOCK_MONOTONIC),
-      gpr_time_from_seconds(GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS,
-                            GPR_TIMESPAN));
-  return gpr_time_cmp(current_deadline, min_deadline) > 0 ? current_deadline
-                                                          : min_deadline;
+  gpr_mu_unlock(&c->mu);
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
 }
 
 /*
@@ -700,8 +645,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call,
   GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0);
 }
 
-void grpc_subchannel_call_ref(grpc_subchannel_call *c
-                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+void grpc_subchannel_call_ref(
+    grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON);
 }
 
diff --git a/src/core/client_config/subchannel.h b/src/core/lib/client_config/subchannel.h
index 313e63c75c..b4f545be52 100644
--- a/src/core/client_config/subchannel.h
+++ b/src/core/lib/client_config/subchannel.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_H
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/client_config/connector.h"
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/client_config/connector.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 /** A (sub-)channel that knows how to connect to exactly one target
     address. Provides a target for load balancing. */
@@ -83,25 +83,25 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
 #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS
 #endif
 
-grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *channel
-                                         GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+grpc_subchannel *grpc_subchannel_ref(
+    grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 grpc_subchannel *grpc_subchannel_ref_from_weak_ref(
     grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx,
                            grpc_subchannel *channel
                                GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *channel
-                                              GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+grpc_subchannel *grpc_subchannel_weak_ref(
+    grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel *channel
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_connected_subchannel_ref(grpc_connected_subchannel *channel
-                                       GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_connected_subchannel_ref(
+    grpc_connected_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx,
                                      grpc_connected_subchannel *channel
                                          GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_subchannel_call_ref(grpc_subchannel_call *call
-                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_subchannel_call_ref(
+    grpc_subchannel_call *call GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel_call *call
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
@@ -171,4 +171,4 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
                                         grpc_connector *connector,
                                         grpc_subchannel_args *args);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_H */
diff --git a/src/core/client_config/subchannel_factory.c b/src/core/lib/client_config/subchannel_factory.c
index 2c64219e8b..727a48a6c8 100644
--- a/src/core/client_config/subchannel_factory.c
+++ b/src/core/lib/client_config/subchannel_factory.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/subchannel_factory.h"
+#include "src/core/lib/client_config/subchannel_factory.h"
 
 void grpc_subchannel_factory_ref(grpc_subchannel_factory* factory) {
   factory->vtable->ref(factory);
diff --git a/src/core/client_config/subchannel_factory.h b/src/core/lib/client_config/subchannel_factory.h
index c6d8cc90be..3ba2f860fe 100644
--- a/src/core/client_config/subchannel_factory.h
+++ b/src/core/lib/client_config/subchannel_factory.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_FACTORY_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_FACTORY_H
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/client_config/subchannel.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/client_config/subchannel.h"
 
 typedef struct grpc_subchannel_factory grpc_subchannel_factory;
 typedef struct grpc_subchannel_factory_vtable grpc_subchannel_factory_vtable;
@@ -63,4 +63,4 @@ grpc_subchannel *grpc_subchannel_factory_create_subchannel(
     grpc_exec_ctx *exec_ctx, grpc_subchannel_factory *factory,
     grpc_subchannel_args *args);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_FACTORY_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_FACTORY_H */
diff --git a/src/core/client_config/subchannel_index.c b/src/core/lib/client_config/subchannel_index.c
index 3f948998f9..2c545002a2 100644
--- a/src/core/client_config/subchannel_index.c
+++ b/src/core/lib/client_config/subchannel_index.c
@@ -31,7 +31,7 @@
 //
 //
 
-#include "src/core/client_config/subchannel_index.h"
+#include "src/core/lib/client_config/subchannel_index.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -40,7 +40,7 @@
 #include <grpc/support/avl.h>
 #include <grpc/support/tls.h>
 
-#include "src/core/channel/channel_args.h"
+#include "src/core/lib/channel/channel_args.h"
 
 // a map of subchannel_key --> subchannel, used for detecting connections
 // to the same destination in order to share them
@@ -108,6 +108,7 @@ static int subchannel_key_compare(grpc_subchannel_key *a,
   if (c != 0) return c;
   c = memcmp(a->args.filters, b->args.filters,
              a->args.filter_count * sizeof(*a->args.filters));
+  if (c != 0) return c;
   return grpc_channel_args_compare(a->args.args, b->args.args);
 }
 
diff --git a/src/core/client_config/subchannel_index.h b/src/core/lib/client_config/subchannel_index.h
index 095ef17819..bc5f03beb4 100644
--- a/src/core/client_config/subchannel_index.h
+++ b/src/core/lib/client_config/subchannel_index.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_INDEX_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_INDEX_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_INDEX_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_INDEX_H
 
-#include "src/core/client_config/connector.h"
-#include "src/core/client_config/subchannel.h"
+#include "src/core/lib/client_config/connector.h"
+#include "src/core/lib/client_config/subchannel.h"
 
 /** \file Provides an index of active subchannels so that they can be
     shared amongst channels */
@@ -74,4 +74,4 @@ void grpc_subchannel_index_init(void);
 /** Shutdown the subchannel index (global) */
 void grpc_subchannel_index_shutdown(void);
 
-#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_INDEX_H */
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_SUBCHANNEL_INDEX_H */
diff --git a/src/core/client_config/uri_parser.c b/src/core/lib/client_config/uri_parser.c
index cbdfffcf8e..d3228dec5f 100644
--- a/src/core/client_config/uri_parser.c
+++ b/src/core/lib/client_config/uri_parser.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/client_config/uri_parser.h"
+#include "src/core/lib/client_config/uri_parser.h"
 
 #include <string.h>
 
diff --git a/src/core/client_config/uri_parser.h b/src/core/lib/client_config/uri_parser.h
index b8daa13bd4..d70d451e60 100644
--- a/src/core/client_config/uri_parser.h
+++ b/src/core/lib/client_config/uri_parser.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_URI_PARSER_H
-#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_URI_PARSER_H
+#ifndef GRPC_CORE_LIB_CLIENT_CONFIG_URI_PARSER_H
+#define GRPC_CORE_LIB_CLIENT_CONFIG_URI_PARSER_H
 
 typedef struct {
   char *scheme;
@@ -48,4 +48,4 @@ grpc_uri *grpc_uri_parse(const char *uri_text, int suppress_errors);
 /** destroy a uri */
 void grpc_uri_destroy(grpc_uri *uri);
 
-#endif
+#endif /* GRPC_CORE_LIB_CLIENT_CONFIG_URI_PARSER_H */
diff --git a/src/core/compression/algorithm_metadata.h b/src/core/lib/compression/algorithm_metadata.h
index 882633c307..47f33abdc7 100644
--- a/src/core/compression/algorithm_metadata.h
+++ b/src/core/lib/compression/algorithm_metadata.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_COMPRESSION_ALGORITHM_METADATA_H
-#define GRPC_INTERNAL_CORE_COMPRESSION_ALGORITHM_METADATA_H
+#ifndef GRPC_CORE_LIB_COMPRESSION_ALGORITHM_METADATA_H
+#define GRPC_CORE_LIB_COMPRESSION_ALGORITHM_METADATA_H
 
 #include <grpc/compression.h>
-#include "src/core/transport/metadata.h"
+#include "src/core/lib/transport/metadata.h"
 
 /** Return compression algorithm based metadata value */
 grpc_mdstr *grpc_compression_algorithm_mdstr(
@@ -50,4 +50,4 @@ grpc_mdelem *grpc_compression_encoding_mdelem(
 grpc_compression_algorithm grpc_compression_algorithm_from_mdstr(
     grpc_mdstr *str);
 
-#endif /* GRPC_INTERNAL_CORE_COMPRESSION_ALGORITHM_METADATA_H */
+#endif /* GRPC_CORE_LIB_COMPRESSION_ALGORITHM_METADATA_H */
diff --git a/src/core/compression/compression_algorithm.c b/src/core/lib/compression/compression_algorithm.c
index 6f3a8eb28e..f781b45042 100644
--- a/src/core/compression/compression_algorithm.c
+++ b/src/core/lib/compression/compression_algorithm.c
@@ -37,9 +37,9 @@
 #include <grpc/compression.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/compression/algorithm_metadata.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/compression/algorithm_metadata.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 int grpc_compression_algorithm_parse(const char *name, size_t name_length,
                                      grpc_compression_algorithm *algorithm) {
@@ -128,20 +128,57 @@ grpc_mdelem *grpc_compression_encoding_mdelem(
 /* TODO(dgq): Add the ability to specify parameters to the individual
  * compression algorithms */
 grpc_compression_algorithm grpc_compression_algorithm_for_level(
-    grpc_compression_level level) {
+    grpc_compression_level level, uint32_t accepted_encodings) {
   GRPC_API_TRACE("grpc_compression_algorithm_for_level(level=%d)", 1,
                  ((int)level));
+  if (level > GRPC_COMPRESS_LEVEL_HIGH) {
+    gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level);
+    abort();
+  }
+
+  const size_t num_supported =
+      GPR_BITCOUNT(accepted_encodings) - 1; /* discard NONE */
+  if (level == GRPC_COMPRESS_LEVEL_NONE || num_supported == 0) {
+    return GRPC_COMPRESS_NONE;
+  }
+
+  GPR_ASSERT(level > 0);
+
+  /* Establish a "ranking" or compression algorithms in increasing order of
+   * compression.
+   * This is simplistic and we will probably want to introduce other dimensions
+   * in the future (cpu/memory cost, etc). */
+  const grpc_compression_algorithm algos_ranking[] = {GRPC_COMPRESS_GZIP,
+                                                      GRPC_COMPRESS_DEFLATE};
+
+  /* intersect algos_ranking with the supported ones keeping the ranked order */
+  grpc_compression_algorithm
+      sorted_supported_algos[GRPC_COMPRESS_ALGORITHMS_COUNT];
+  size_t algos_supported_idx = 0;
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(algos_ranking); i++) {
+    const grpc_compression_algorithm alg = algos_ranking[i];
+    for (size_t j = 0; j < num_supported; j++) {
+      if (GPR_BITGET(accepted_encodings, alg) == 1) {
+        /* if \a alg in supported */
+        sorted_supported_algos[algos_supported_idx++] = alg;
+        break;
+      }
+    }
+    if (algos_supported_idx == num_supported) break;
+  }
+
   switch (level) {
     case GRPC_COMPRESS_LEVEL_NONE:
-      return GRPC_COMPRESS_NONE;
+      abort(); /* should have been handled already */
     case GRPC_COMPRESS_LEVEL_LOW:
+      return sorted_supported_algos[0];
     case GRPC_COMPRESS_LEVEL_MED:
+      return sorted_supported_algos[num_supported / 2];
     case GRPC_COMPRESS_LEVEL_HIGH:
-      return GRPC_COMPRESS_DEFLATE;
+      return sorted_supported_algos[num_supported - 1];
     default:
-      gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level);
       abort();
-  }
+  };
 }
 
 void grpc_compression_options_init(grpc_compression_options *opts) {
diff --git a/src/core/compression/message_compress.c b/src/core/lib/compression/message_compress.c
index edc21a9eb7..b4b6a2d75e 100644
--- a/src/core/compression/message_compress.c
+++ b/src/core/lib/compression/message_compress.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/compression/message_compress.h"
+#include "src/core/lib/compression/message_compress.h"
 
 #include <string.h>
 
diff --git a/src/core/compression/message_compress.h b/src/core/lib/compression/message_compress.h
index 14652004b8..b71608139e 100644
--- a/src/core/compression/message_compress.h
+++ b/src/core/lib/compression/message_compress.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_COMPRESSION_MESSAGE_COMPRESS_H
-#define GRPC_INTERNAL_CORE_COMPRESSION_MESSAGE_COMPRESS_H
+#ifndef GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
+#define GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
 
 #include <grpc/compression.h>
 #include <grpc/support/slice_buffer.h>
@@ -49,4 +49,4 @@ int grpc_msg_compress(grpc_compression_algorithm algorithm,
 int grpc_msg_decompress(grpc_compression_algorithm algorithm,
                         gpr_slice_buffer* input, gpr_slice_buffer* output);
 
-#endif /* GRPC_INTERNAL_CORE_COMPRESSION_MESSAGE_COMPRESS_H */
+#endif /* GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H */
diff --git a/src/core/debug/trace.c b/src/core/lib/debug/trace.c
index 3b35d81cd8..786dd9324f 100644
--- a/src/core/debug/trace.c
+++ b/src/core/lib/debug/trace.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/debug/trace.h"
+#include "src/core/lib/debug/trace.h"
 
 #include <string.h>
 
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include "src/core/support/env.h"
+#include "src/core/lib/support/env.h"
 
 typedef struct tracer {
   const char *name;
diff --git a/src/core/debug/trace.h b/src/core/lib/debug/trace.h
index dc5875976e..76ea5a7c64 100644
--- a/src/core/debug/trace.h
+++ b/src/core/lib/debug/trace.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_DEBUG_TRACE_H
-#define GRPC_INTERNAL_CORE_DEBUG_TRACE_H
+#ifndef GRPC_CORE_LIB_DEBUG_TRACE_H
+#define GRPC_CORE_LIB_DEBUG_TRACE_H
 
 #include <grpc/support/port_platform.h>
 
@@ -40,4 +40,4 @@ void grpc_register_tracer(const char *name, int *flag);
 void grpc_tracer_init(const char *env_var_name);
 void grpc_tracer_shutdown(void);
 
-#endif /* GRPC_INTERNAL_CORE_DEBUG_TRACE_H */
+#endif /* GRPC_CORE_LIB_DEBUG_TRACE_H */
diff --git a/src/core/httpcli/format_request.c b/src/core/lib/http/format_request.c
index 04f2a2d99a..95b3918646 100644
--- a/src/core/httpcli/format_request.c
+++ b/src/core/lib/http/format_request.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,22 +31,22 @@
  *
  */
 
-#include "src/core/httpcli/format_request.h"
+#include "src/core/lib/http/format_request.h"
 
 #include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/support/string.h"
 
 static void fill_common_header(const grpc_httpcli_request *request,
                                gpr_strvec *buf) {
   size_t i;
-  gpr_strvec_add(buf, gpr_strdup(request->path));
+  gpr_strvec_add(buf, gpr_strdup(request->http.path));
   gpr_strvec_add(buf, gpr_strdup(" HTTP/1.0\r\n"));
   /* just in case some crazy server really expects HTTP/1.1 */
   gpr_strvec_add(buf, gpr_strdup("Host: "));
@@ -56,10 +56,10 @@ static void fill_common_header(const grpc_httpcli_request *request,
   gpr_strvec_add(buf,
                  gpr_strdup("User-Agent: " GRPC_HTTPCLI_USER_AGENT "\r\n"));
   /* user supplied headers */
-  for (i = 0; i < request->hdr_count; i++) {
-    gpr_strvec_add(buf, gpr_strdup(request->hdrs[i].key));
+  for (i = 0; i < request->http.hdr_count; i++) {
+    gpr_strvec_add(buf, gpr_strdup(request->http.hdrs[i].key));
     gpr_strvec_add(buf, gpr_strdup(": "));
-    gpr_strvec_add(buf, gpr_strdup(request->hdrs[i].value));
+    gpr_strvec_add(buf, gpr_strdup(request->http.hdrs[i].value));
     gpr_strvec_add(buf, gpr_strdup("\r\n"));
   }
 }
@@ -94,8 +94,8 @@ gpr_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request,
   fill_common_header(request, &out);
   if (body_bytes) {
     uint8_t has_content_type = 0;
-    for (i = 0; i < request->hdr_count; i++) {
-      if (strcmp(request->hdrs[i].key, "Content-Type") == 0) {
+    for (i = 0; i < request->http.hdr_count; i++) {
+      if (strcmp(request->http.hdrs[i].key, "Content-Type") == 0) {
         has_content_type = 1;
         break;
       }
diff --git a/src/core/httpcli/format_request.h b/src/core/lib/http/format_request.h
index c8dc8f7d4e..2e933d804b 100644
--- a/src/core/httpcli/format_request.h
+++ b/src/core/lib/http/format_request.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,15 +31,15 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_HTTPCLI_FORMAT_REQUEST_H
-#define GRPC_INTERNAL_CORE_HTTPCLI_FORMAT_REQUEST_H
+#ifndef GRPC_CORE_LIB_HTTP_FORMAT_REQUEST_H
+#define GRPC_CORE_LIB_HTTP_FORMAT_REQUEST_H
 
-#include "src/core/httpcli/httpcli.h"
 #include <grpc/support/slice.h>
+#include "src/core/lib/http/httpcli.h"
 
 gpr_slice grpc_httpcli_format_get_request(const grpc_httpcli_request *request);
 gpr_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request,
                                            const char *body_bytes,
                                            size_t body_size);
 
-#endif /* GRPC_INTERNAL_CORE_HTTPCLI_FORMAT_REQUEST_H */
+#endif /* GRPC_CORE_LIB_HTTP_FORMAT_REQUEST_H */
diff --git a/src/core/httpcli/httpcli.c b/src/core/lib/http/httpcli.c
index 1219c444c7..aab28ad8b6 100644
--- a/src/core/httpcli/httpcli.c
+++ b/src/core/lib/http/httpcli.c
@@ -31,8 +31,8 @@
  *
  */
 
-#include "src/core/httpcli/httpcli.h"
-#include "src/core/iomgr/sockaddr.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
 #include <string.h>
 
@@ -40,17 +40,17 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/httpcli/format_request.h"
-#include "src/core/httpcli/parser.h"
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/iomgr/tcp_client.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/http/format_request.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/support/string.h"
 
 typedef struct {
   gpr_slice request_text;
-  grpc_httpcli_parser parser;
+  grpc_http_parser parser;
   grpc_resolved_addresses *addresses;
   size_t next_address;
   grpc_endpoint *ep;
@@ -99,8 +99,9 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
                    int success) {
   grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set,
                                req->pollset);
-  req->on_response(exec_ctx, req->user_data, success ? &req->parser.r : NULL);
-  grpc_httpcli_parser_destroy(&req->parser);
+  req->on_response(exec_ctx, req->user_data,
+                   success ? &req->parser.http.response : NULL);
+  grpc_http_parser_destroy(&req->parser);
   if (req->addresses != NULL) {
     grpc_resolved_addresses_destroy(req->addresses);
   }
@@ -129,7 +130,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
   for (i = 0; i < req->incoming.count; i++) {
     if (GPR_SLICE_LENGTH(req->incoming.slices[i])) {
       req->have_read_byte = 1;
-      if (!grpc_httpcli_parser_parse(&req->parser, req->incoming.slices[i])) {
+      if (!grpc_http_parser_parse(&req->parser, req->incoming.slices[i])) {
         finish(exec_ctx, req, 0);
         return;
       }
@@ -141,7 +142,11 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
   } else if (!req->have_read_byte) {
     next_address(exec_ctx, req);
   } else {
-    finish(exec_ctx, req, grpc_httpcli_parser_eof(&req->parser));
+    int parse_success = grpc_http_parser_eof(&req->parser);
+    if (parse_success && (req->parser.type != GRPC_HTTP_RESPONSE)) {
+      parse_success = 0;
+    }
+    finish(exec_ctx, req, parse_success);
   }
 }
 
@@ -223,7 +228,7 @@ static void internal_request_begin(
   internal_request *req = gpr_malloc(sizeof(internal_request));
   memset(req, 0, sizeof(*req));
   req->request_text = request_text;
-  grpc_httpcli_parser_init(&req->parser);
+  grpc_http_parser_init(&req->parser);
   req->on_response = on_response;
   req->user_data = user_data;
   req->deadline = deadline;
@@ -255,7 +260,7 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
       g_get_override(exec_ctx, request, deadline, on_response, user_data)) {
     return;
   }
-  gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->path);
+  gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->http.path);
   internal_request_begin(exec_ctx, context, pollset, request, deadline,
                          on_response, user_data, name,
                          grpc_httpcli_format_get_request(request));
@@ -274,7 +279,7 @@ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
                       on_response, user_data)) {
     return;
   }
-  gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->path);
+  gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->http.path);
   internal_request_begin(
       exec_ctx, context, pollset, request, deadline, on_response, user_data,
       name, grpc_httpcli_format_post_request(request, body_bytes, body_size));
diff --git a/src/core/httpcli/httpcli.h b/src/core/lib/http/httpcli.h
index c9cd987c79..b8d54a8586 100644
--- a/src/core/httpcli/httpcli.h
+++ b/src/core/lib/http/httpcli.h
@@ -31,27 +31,20 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_H
-#define GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_H
+#ifndef GRPC_CORE_LIB_HTTP_HTTPCLI_H
+#define GRPC_CORE_LIB_HTTP_HTTPCLI_H
 
 #include <stddef.h>
 
 #include <grpc/support/time.h>
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/pollset_set.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 
 /* User agent this library reports */
 #define GRPC_HTTPCLI_USER_AGENT "grpc-httpcli/0.0"
-/* Maximum length of a header string of the form 'Key: Value\r\n' */
-#define GRPC_HTTPCLI_MAX_HEADER_LENGTH 4096
-
-/* A single header to be passed in a request */
-typedef struct grpc_httpcli_header {
-  char *key;
-  char *value;
-} grpc_httpcli_header;
 
 /* Tracks in-progress http requests
    TODO(ctiller): allow caching and capturing multiple requests for the
@@ -77,33 +70,21 @@ typedef struct grpc_httpcli_request {
   char *host;
   /* The host to verify in the SSL handshake (or NULL) */
   char *ssl_host_override;
-  /* The path of the resource to fetch */
-  char *path;
-  /* Additional headers: count and key/values; the following are supplied
-     automatically and MUST NOT be set here:
+  /* The main part of the request
+     The following headers are supplied automatically and MUST NOT be set here:
      Host, Connection, User-Agent */
-  size_t hdr_count;
-  grpc_httpcli_header *hdrs;
+  grpc_http_request http;
   /* handshaker to use ssl for the request */
   const grpc_httpcli_handshaker *handshaker;
 } grpc_httpcli_request;
 
-/* A response */
-typedef struct grpc_httpcli_response {
-  /* HTTP status code */
-  int status;
-  /* Headers: count and key/values */
-  size_t hdr_count;
-  grpc_httpcli_header *hdrs;
-  /* Body: length and contents; contents are NOT null-terminated */
-  size_t body_length;
-  char *body;
-} grpc_httpcli_response;
+/* Expose the parser response type as a httpcli response too */
+typedef struct grpc_http_response grpc_httpcli_response;
 
 /* Callback for grpc_httpcli_get and grpc_httpcli_post. */
 typedef void (*grpc_httpcli_response_cb)(grpc_exec_ctx *exec_ctx,
                                          void *user_data,
-                                         const grpc_httpcli_response *response);
+                                         const grpc_http_response *response);
 
 void grpc_httpcli_context_init(grpc_httpcli_context *context);
 void grpc_httpcli_context_destroy(grpc_httpcli_context *context);
@@ -160,4 +141,4 @@ typedef int (*grpc_httpcli_post_override)(
 void grpc_httpcli_set_override(grpc_httpcli_get_override get,
                                grpc_httpcli_post_override post);
 
-#endif /* GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_H */
+#endif /* GRPC_CORE_LIB_HTTP_HTTPCLI_H */
diff --git a/src/core/httpcli/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c
index 156961a377..6f1630ac1f 100644
--- a/src/core/httpcli/httpcli_security_connector.c
+++ b/src/core/lib/http/httpcli_security_connector.c
@@ -31,16 +31,16 @@
  *
  */
 
-#include "src/core/httpcli/httpcli.h"
+#include "src/core/lib/http/httpcli.h"
 
 #include <string.h>
 
-#include "src/core/security/handshake.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/tsi/ssl_transport_security.h"
+#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/tsi/ssl_transport_security.h"
 
 typedef struct {
   grpc_channel_security_connector base;
diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c
new file mode 100644
index 0000000000..2782ad758e
--- /dev/null
+++ b/src/core/lib/http/parser.c
@@ -0,0 +1,317 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/http/parser.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/useful.h>
+
+extern int grpc_http_trace;
+
+static char *buf2str(void *buffer, size_t length) {
+  char *out = gpr_malloc(length + 1);
+  memcpy(out, buffer, length);
+  out[length] = 0;
+  return out;
+}
+
+static int handle_response_line(grpc_http_parser *parser) {
+  uint8_t *beg = parser->cur_line;
+  uint8_t *cur = beg;
+  uint8_t *end = beg + parser->cur_line_length;
+
+  if (cur == end || *cur++ != 'H') goto error;
+  if (cur == end || *cur++ != 'T') goto error;
+  if (cur == end || *cur++ != 'T') goto error;
+  if (cur == end || *cur++ != 'P') goto error;
+  if (cur == end || *cur++ != '/') goto error;
+  if (cur == end || *cur++ != '1') goto error;
+  if (cur == end || *cur++ != '.') goto error;
+  if (cur == end || *cur < '0' || *cur++ > '1') goto error;
+  if (cur == end || *cur++ != ' ') goto error;
+  if (cur == end || *cur < '1' || *cur++ > '9') goto error;
+  if (cur == end || *cur < '0' || *cur++ > '9') goto error;
+  if (cur == end || *cur < '0' || *cur++ > '9') goto error;
+  parser->http.response.status =
+      (cur[-3] - '0') * 100 + (cur[-2] - '0') * 10 + (cur[-1] - '0');
+  if (cur == end || *cur++ != ' ') goto error;
+
+  /* we don't really care about the status code message */
+
+  return 1;
+
+error:
+  if (grpc_http_trace) gpr_log(GPR_ERROR, "Failed parsing response line");
+  return 0;
+}
+
+static int handle_request_line(grpc_http_parser *parser) {
+  uint8_t *beg = parser->cur_line;
+  uint8_t *cur = beg;
+  uint8_t *end = beg + parser->cur_line_length;
+  uint8_t vers_major = 0;
+  uint8_t vers_minor = 0;
+
+  while (cur != end && *cur++ != ' ')
+    ;
+  if (cur == end) goto error;
+  parser->http.request.method = buf2str(beg, (size_t)(cur - beg - 1));
+
+  beg = cur;
+  while (cur != end && *cur++ != ' ')
+    ;
+  if (cur == end) goto error;
+  parser->http.request.path = buf2str(beg, (size_t)(cur - beg - 1));
+
+  if (cur == end || *cur++ != 'H') goto error;
+  if (cur == end || *cur++ != 'T') goto error;
+  if (cur == end || *cur++ != 'T') goto error;
+  if (cur == end || *cur++ != 'P') goto error;
+  if (cur == end || *cur++ != '/') goto error;
+  vers_major = (uint8_t)(*cur++ - '1' + 1);
+  ++cur;
+  if (cur == end) goto error;
+  vers_minor = (uint8_t)(*cur++ - '1' + 1);
+
+  if (vers_major == 1) {
+    if (vers_minor == 0) {
+      parser->http.request.version = GRPC_HTTP_HTTP10;
+    } else if (vers_minor == 1) {
+      parser->http.request.version = GRPC_HTTP_HTTP11;
+    } else {
+      goto error;
+    }
+  } else if (vers_major == 2) {
+    if (vers_minor == 0) {
+      parser->http.request.version = GRPC_HTTP_HTTP20;
+    } else {
+      goto error;
+    }
+  } else {
+    goto error;
+  }
+
+  return 1;
+
+error:
+  if (grpc_http_trace) gpr_log(GPR_ERROR, "Failed parsing request line");
+  return 0;
+}
+
+static int handle_first_line(grpc_http_parser *parser) {
+  if (parser->cur_line[0] == 'H') {
+    parser->type = GRPC_HTTP_RESPONSE;
+    return handle_response_line(parser);
+  } else {
+    parser->type = GRPC_HTTP_REQUEST;
+    return handle_request_line(parser);
+  }
+}
+
+static int add_header(grpc_http_parser *parser) {
+  uint8_t *beg = parser->cur_line;
+  uint8_t *cur = beg;
+  uint8_t *end = beg + parser->cur_line_length;
+  size_t *hdr_count = NULL;
+  grpc_http_header **hdrs = NULL;
+  grpc_http_header hdr = {NULL, NULL};
+
+  GPR_ASSERT(cur != end);
+
+  if (*cur == ' ' || *cur == '\t') {
+    if (grpc_http_trace)
+      gpr_log(GPR_ERROR, "Continued header lines not supported yet");
+    goto error;
+  }
+
+  while (cur != end && *cur != ':') {
+    cur++;
+  }
+  if (cur == end) {
+    if (grpc_http_trace) gpr_log(GPR_ERROR, "Didn't find ':' in header string");
+    goto error;
+  }
+  GPR_ASSERT(cur >= beg);
+  hdr.key = buf2str(beg, (size_t)(cur - beg));
+  cur++; /* skip : */
+
+  while (cur != end && (*cur == ' ' || *cur == '\t')) {
+    cur++;
+  }
+  GPR_ASSERT(end - cur >= 2);
+  hdr.value = buf2str(cur, (size_t)(end - cur) - 2);
+
+  if (parser->type == GRPC_HTTP_RESPONSE) {
+    hdr_count = &parser->http.response.hdr_count;
+    hdrs = &parser->http.response.hdrs;
+  } else if (parser->type == GRPC_HTTP_REQUEST) {
+    hdr_count = &parser->http.request.hdr_count;
+    hdrs = &parser->http.request.hdrs;
+  } else {
+    return 0;
+  }
+
+  if (*hdr_count == parser->hdr_capacity) {
+    parser->hdr_capacity =
+        GPR_MAX(parser->hdr_capacity + 1, parser->hdr_capacity * 3 / 2);
+    *hdrs = gpr_realloc(*hdrs, parser->hdr_capacity * sizeof(**hdrs));
+  }
+  (*hdrs)[(*hdr_count)++] = hdr;
+  return 1;
+
+error:
+  gpr_free(hdr.key);
+  gpr_free(hdr.value);
+  return 0;
+}
+
+static int finish_line(grpc_http_parser *parser) {
+  switch (parser->state) {
+    case GRPC_HTTP_FIRST_LINE:
+      if (!handle_first_line(parser)) {
+        return 0;
+      }
+      parser->state = GRPC_HTTP_HEADERS;
+      break;
+    case GRPC_HTTP_HEADERS:
+      if (parser->cur_line_length == 2) {
+        parser->state = GRPC_HTTP_BODY;
+        break;
+      }
+      if (!add_header(parser)) {
+        return 0;
+      }
+      break;
+    case GRPC_HTTP_BODY:
+      GPR_UNREACHABLE_CODE(return 0);
+  }
+
+  parser->cur_line_length = 0;
+  return 1;
+}
+
+static int addbyte_body(grpc_http_parser *parser, uint8_t byte) {
+  size_t *body_length = NULL;
+  char **body = NULL;
+
+  if (parser->type == GRPC_HTTP_RESPONSE) {
+    body_length = &parser->http.response.body_length;
+    body = &parser->http.response.body;
+  } else if (parser->type == GRPC_HTTP_REQUEST) {
+    body_length = &parser->http.request.body_length;
+    body = &parser->http.request.body;
+  } else {
+    return 0;
+  }
+
+  if (*body_length == parser->body_capacity) {
+    parser->body_capacity = GPR_MAX(8, parser->body_capacity * 3 / 2);
+    *body = gpr_realloc((void *)*body, parser->body_capacity);
+  }
+  (*body)[*body_length] = (char)byte;
+  (*body_length)++;
+
+  return 1;
+}
+
+static int addbyte(grpc_http_parser *parser, uint8_t byte) {
+  switch (parser->state) {
+    case GRPC_HTTP_FIRST_LINE:
+    case GRPC_HTTP_HEADERS:
+      if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) {
+        if (grpc_http_trace)
+          gpr_log(GPR_ERROR, "HTTP client max line length (%d) exceeded",
+                  GRPC_HTTP_PARSER_MAX_HEADER_LENGTH);
+        return 0;
+      }
+      parser->cur_line[parser->cur_line_length] = byte;
+      parser->cur_line_length++;
+      if (parser->cur_line_length >= 2 &&
+          parser->cur_line[parser->cur_line_length - 2] == '\r' &&
+          parser->cur_line[parser->cur_line_length - 1] == '\n') {
+        return finish_line(parser);
+      } else {
+        return 1;
+      }
+      GPR_UNREACHABLE_CODE(return 0);
+    case GRPC_HTTP_BODY:
+      return addbyte_body(parser, byte);
+  }
+  GPR_UNREACHABLE_CODE(return 0);
+}
+
+void grpc_http_parser_init(grpc_http_parser *parser) {
+  memset(parser, 0, sizeof(*parser));
+  parser->state = GRPC_HTTP_FIRST_LINE;
+  parser->type = GRPC_HTTP_UNKNOWN;
+}
+
+void grpc_http_parser_destroy(grpc_http_parser *parser) {
+  size_t i;
+  if (parser->type == GRPC_HTTP_RESPONSE) {
+    gpr_free(parser->http.response.body);
+    for (i = 0; i < parser->http.response.hdr_count; i++) {
+      gpr_free(parser->http.response.hdrs[i].key);
+      gpr_free(parser->http.response.hdrs[i].value);
+    }
+    gpr_free(parser->http.response.hdrs);
+  } else if (parser->type == GRPC_HTTP_REQUEST) {
+    gpr_free(parser->http.request.body);
+    for (i = 0; i < parser->http.request.hdr_count; i++) {
+      gpr_free(parser->http.request.hdrs[i].key);
+      gpr_free(parser->http.request.hdrs[i].value);
+    }
+    gpr_free(parser->http.request.hdrs);
+    gpr_free(parser->http.request.method);
+    gpr_free(parser->http.request.path);
+  }
+}
+
+int grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice) {
+  size_t i;
+
+  for (i = 0; i < GPR_SLICE_LENGTH(slice); i++) {
+    if (!addbyte(parser, GPR_SLICE_START_PTR(slice)[i])) {
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
+int grpc_http_parser_eof(grpc_http_parser *parser) {
+  return parser->state == GRPC_HTTP_BODY;
+}
diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h
new file mode 100644
index 0000000000..6a72174aa6
--- /dev/null
+++ b/src/core/lib/http/parser.h
@@ -0,0 +1,116 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_HTTP_PARSER_H
+#define GRPC_CORE_LIB_HTTP_PARSER_H
+
+#include <grpc/support/port_platform.h>
+#include <grpc/support/slice.h>
+
+/* Maximum length of a header string of the form 'Key: Value\r\n' */
+#define GRPC_HTTP_PARSER_MAX_HEADER_LENGTH 4096
+
+/* A single header to be passed in a request */
+typedef struct grpc_http_header {
+  char *key;
+  char *value;
+} grpc_http_header;
+
+typedef enum {
+  GRPC_HTTP_FIRST_LINE,
+  GRPC_HTTP_HEADERS,
+  GRPC_HTTP_BODY
+} grpc_http_parser_state;
+
+typedef enum {
+  GRPC_HTTP_HTTP10,
+  GRPC_HTTP_HTTP11,
+  GRPC_HTTP_HTTP20,
+} grpc_http_version;
+
+typedef enum {
+  GRPC_HTTP_RESPONSE,
+  GRPC_HTTP_REQUEST,
+  GRPC_HTTP_UNKNOWN
+} grpc_http_type;
+
+/* A request */
+typedef struct grpc_http_request {
+  /* Method of the request (e.g. GET, POST) */
+  char *method;
+  /* The path of the resource to fetch */
+  char *path;
+  /* HTTP version to use */
+  grpc_http_version version;
+  /* Headers attached to the request */
+  size_t hdr_count;
+  grpc_http_header *hdrs;
+  /* Body: length and contents; contents are NOT null-terminated */
+  size_t body_length;
+  char *body;
+} grpc_http_request;
+
+/* A response */
+typedef struct grpc_http_response {
+  /* HTTP status code */
+  int status;
+  /* Headers: count and key/values */
+  size_t hdr_count;
+  grpc_http_header *hdrs;
+  /* Body: length and contents; contents are NOT null-terminated */
+  size_t body_length;
+  char *body;
+} grpc_http_response;
+
+typedef struct {
+  grpc_http_parser_state state;
+  grpc_http_type type;
+
+  union {
+    grpc_http_response response;
+    grpc_http_request request;
+  } http;
+  size_t body_capacity;
+  size_t hdr_capacity;
+
+  uint8_t cur_line[GRPC_HTTP_PARSER_MAX_HEADER_LENGTH];
+  size_t cur_line_length;
+} grpc_http_parser;
+
+void grpc_http_parser_init(grpc_http_parser *parser);
+void grpc_http_parser_destroy(grpc_http_parser *parser);
+
+int grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice);
+int grpc_http_parser_eof(grpc_http_parser *parser);
+
+#endif /* GRPC_CORE_LIB_HTTP_PARSER_H */
diff --git a/src/core/iomgr/closure.c b/src/core/lib/iomgr/closure.c
index 3a96f7385f..724ebc284a 100644
--- a/src/core/iomgr/closure.c
+++ b/src/core/lib/iomgr/closure.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/iomgr/closure.h"
+#include "src/core/lib/iomgr/closure.h"
 
 #include <grpc/support/alloc.h>
 
diff --git a/src/core/iomgr/closure.h b/src/core/lib/iomgr/closure.h
index ea96c19c71..2597cf1706 100644
--- a/src/core/iomgr/closure.h
+++ b/src/core/lib/iomgr/closure.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H
-#define GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H
+#ifndef GRPC_CORE_LIB_IOMGR_CLOSURE_H
+#define GRPC_CORE_LIB_IOMGR_CLOSURE_H
 
 #include <grpc/support/port_platform.h>
 #include <stdbool.h>
@@ -95,4 +95,4 @@ bool grpc_closure_list_empty(grpc_closure_list list);
 /** return the next pointer for a queued closure list */
 grpc_closure *grpc_closure_next(grpc_closure *closure);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H */
+#endif /* GRPC_CORE_LIB_IOMGR_CLOSURE_H */
diff --git a/src/core/iomgr/endpoint.c b/src/core/lib/iomgr/endpoint.c
index bd64707669..576b5a6e5c 100644
--- a/src/core/iomgr/endpoint.c
+++ b/src/core/lib/iomgr/endpoint.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/endpoint.h"
 
 void grpc_endpoint_read(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep,
                         gpr_slice_buffer* slices, grpc_closure* cb) {
diff --git a/src/core/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h
index cbdc947abb..918e705fbd 100644
--- a/src/core/iomgr/endpoint.h
+++ b/src/core/lib/iomgr/endpoint.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_H
-#define GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_H
+#ifndef GRPC_CORE_LIB_IOMGR_ENDPOINT_H
+#define GRPC_CORE_LIB_IOMGR_ENDPOINT_H
 
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/pollset_set.h"
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 
 /* An endpoint caps a streaming channel between two communicating processes.
    Examples may be: a tcp socket, <stdin+stdout>, or some shared memory. */
@@ -99,4 +99,4 @@ struct grpc_endpoint {
   const grpc_endpoint_vtable *vtable;
 };
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_H */
+#endif /* GRPC_CORE_LIB_IOMGR_ENDPOINT_H */
diff --git a/src/core/iomgr/endpoint_pair.h b/src/core/lib/iomgr/endpoint_pair.h
index 095ec5fcc9..bef8bb3518 100644
--- a/src/core/iomgr/endpoint_pair.h
+++ b/src/core/lib/iomgr/endpoint_pair.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_PAIR_H
-#define GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_PAIR_H
+#ifndef GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H
+#define GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H
 
-#include "src/core/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/endpoint.h"
 
 typedef struct {
   grpc_endpoint *client;
@@ -44,4 +44,4 @@ typedef struct {
 grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
                                                    size_t read_slice_size);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_ENDPOINT_PAIR_H */
+#endif /* GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H */
diff --git a/src/core/iomgr/endpoint_pair_posix.c b/src/core/lib/iomgr/endpoint_pair_posix.c
index 56f6f146fd..e0ce47c773 100644
--- a/src/core/iomgr/endpoint_pair_posix.c
+++ b/src/core/lib/iomgr/endpoint_pair_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,24 +35,25 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/endpoint_pair.h"
-#include "src/core/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/endpoint_pair.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
 
 #include <errno.h>
 #include <fcntl.h>
 #include <string.h>
-#include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/types.h>
 
-#include "src/core/iomgr/tcp_posix.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/support/string.h"
 
 static void create_sockets(int sv[2]) {
   int flags;
-  GPR_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
+  grpc_create_socketpair_if_unix(sv);
   flags = fcntl(sv[0], F_GETFL, 0);
   GPR_ASSERT(fcntl(sv[0], F_SETFL, flags | O_NONBLOCK) == 0);
   flags = fcntl(sv[1], F_GETFL, 0);
diff --git a/src/core/iomgr/endpoint_pair_windows.c b/src/core/lib/iomgr/endpoint_pair_windows.c
index db9d092dca..cba18db81f 100644
--- a/src/core/iomgr/endpoint_pair_windows.c
+++ b/src/core/lib/iomgr/endpoint_pair_windows.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,16 +34,16 @@
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_WINSOCK_SOCKET
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/iomgr/endpoint_pair.h"
+#include "src/core/lib/iomgr/endpoint_pair.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
 
 #include <errno.h>
 #include <fcntl.h>
 #include <string.h>
 
-#include "src/core/iomgr/tcp_windows.h"
-#include "src/core/iomgr/socket_windows.h"
 #include <grpc/support/log.h>
+#include "src/core/lib/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/tcp_windows.h"
 
 static void create_sockets(SOCKET sv[2]) {
   SOCKET svr_sock = INVALID_SOCKET;
diff --git a/src/core/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c
index 1fd79f6eba..1ed6da623a 100644
--- a/src/core/iomgr/exec_ctx.c
+++ b/src/core/lib/iomgr/exec_ctx.c
@@ -31,12 +31,15 @@
  *
  */
 
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 #include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
+#ifndef GRPC_EXECUTION_CONTEXT_SANITIZER
 bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
   bool did_something = 0;
   GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);
@@ -74,3 +77,75 @@ void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(offload_target_or_null == NULL);
   grpc_closure_list_move(list, &exec_ctx->closure_list);
 }
+
+void grpc_exec_ctx_global_init(void) {}
+void grpc_exec_ctx_global_shutdown(void) {}
+#else
+static gpr_mu g_mu;
+static gpr_cv g_cv;
+static int g_threads = 0;
+
+static void run_closure(void *arg) {
+  grpc_closure *closure = arg;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  closure->cb(&exec_ctx, closure->cb_arg, (closure->final_data & 1) != 0);
+  grpc_exec_ctx_finish(&exec_ctx);
+  gpr_mu_lock(&g_mu);
+  if (--g_threads == 0) {
+    gpr_cv_signal(&g_cv);
+  }
+  gpr_mu_unlock(&g_mu);
+}
+
+static void start_closure(grpc_closure *closure) {
+  gpr_thd_id id;
+  gpr_mu_lock(&g_mu);
+  g_threads++;
+  gpr_mu_unlock(&g_mu);
+  gpr_thd_new(&id, run_closure, closure, NULL);
+}
+
+bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { return false; }
+
+void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {}
+
+void grpc_exec_ctx_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                           bool success,
+                           grpc_workqueue *offload_target_or_null) {
+  GPR_ASSERT(offload_target_or_null == NULL);
+  if (closure == NULL) return;
+  closure->final_data = success;
+  start_closure(closure);
+}
+
+void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
+                                grpc_closure_list *list,
+                                grpc_workqueue *offload_target_or_null) {
+  GPR_ASSERT(offload_target_or_null == NULL);
+  if (list == NULL) return;
+  grpc_closure *p = list->head;
+  while (p) {
+    grpc_closure *start = p;
+    p = grpc_closure_next(start);
+    start_closure(start);
+  }
+  grpc_closure_list r = GRPC_CLOSURE_LIST_INIT;
+  *list = r;
+}
+
+void grpc_exec_ctx_global_init(void) {
+  gpr_mu_init(&g_mu);
+  gpr_cv_init(&g_cv);
+}
+
+void grpc_exec_ctx_global_shutdown(void) {
+  gpr_mu_lock(&g_mu);
+  while (g_threads != 0) {
+    gpr_cv_wait(&g_cv, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME));
+  }
+  gpr_mu_unlock(&g_mu);
+
+  gpr_mu_destroy(&g_mu);
+  gpr_cv_destroy(&g_cv);
+}
+#endif
diff --git a/src/core/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h
index 9a9b2e55fa..e62ea2dedf 100644
--- a/src/core/iomgr/exec_ctx.h
+++ b/src/core/lib/iomgr/exec_ctx.h
@@ -31,11 +31,19 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_EXEC_CTX_H
-#define GRPC_INTERNAL_CORE_IOMGR_EXEC_CTX_H
+#ifndef GRPC_CORE_LIB_IOMGR_EXEC_CTX_H
+#define GRPC_CORE_LIB_IOMGR_EXEC_CTX_H
 
-#include "src/core/iomgr/closure.h"
+#include "src/core/lib/iomgr/closure.h"
 
+/* #define GRPC_EXECUTION_CONTEXT_SANITIZER 1 */
+
+/** A workqueue represents a list of work to be executed asynchronously.
+    Forward declared here to avoid a circular dependency with workqueue.h. */
+struct grpc_workqueue;
+typedef struct grpc_workqueue grpc_workqueue;
+
+#ifndef GRPC_EXECUTION_CONTEXT_SANITIZER
 /** Execution context.
  *  A bag of data that collects information along a callstack.
  *  Generally created at public API entry points, and passed down as
@@ -57,13 +65,15 @@ struct grpc_exec_ctx {
   grpc_closure_list closure_list;
 };
 
-/** A workqueue represents a list of work to be executed asynchronously.
-    Forward declared here to avoid a circular dependency with workqueue.h. */
-struct grpc_workqueue;
-typedef struct grpc_workqueue grpc_workqueue;
-
 #define GRPC_EXEC_CTX_INIT \
   { GRPC_CLOSURE_LIST_INIT }
+#else
+struct grpc_exec_ctx {
+  int unused;
+};
+#define GRPC_EXEC_CTX_INIT \
+  { 0 }
+#endif
 
 /** Flush any work that has been enqueued onto this grpc_exec_ctx.
  *  Caller must guarantee that no interfering locks are held.
@@ -82,4 +92,7 @@ void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
                                 grpc_closure_list *list,
                                 grpc_workqueue *offload_target_or_null);
 
-#endif
+void grpc_exec_ctx_global_init(void);
+void grpc_exec_ctx_global_shutdown(void);
+
+#endif /* GRPC_CORE_LIB_IOMGR_EXEC_CTX_H */
diff --git a/src/core/iomgr/executor.c b/src/core/lib/iomgr/executor.c
index f22d8f30ac..42a9db3cbb 100644
--- a/src/core/iomgr/executor.c
+++ b/src/core/lib/iomgr/executor.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/iomgr/executor.h"
+#include "src/core/lib/iomgr/executor.h"
 
 #include <string.h>
 
@@ -39,7 +39,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/thd.h>
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef struct grpc_executor_data {
   int busy;          /**< is the thread currently running? */
diff --git a/src/core/iomgr/executor.h b/src/core/lib/iomgr/executor.h
index aac057ddf5..f1871416a0 100644
--- a/src/core/iomgr/executor.h
+++ b/src/core/lib/iomgr/executor.h
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H
-#define GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H
+#ifndef GRPC_CORE_LIB_IOMGR_EXECUTOR_H
+#define GRPC_CORE_LIB_IOMGR_EXECUTOR_H
 
-#include "src/core/iomgr/closure.h"
+#include "src/core/lib/iomgr/closure.h"
 
 /** Initialize the global executor.
  *
@@ -50,4 +50,4 @@ void grpc_executor_enqueue(grpc_closure *closure, bool success);
 /** Shutdown the executor, running all pending work as part of the call */
 void grpc_executor_shutdown();
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H */
+#endif /* GRPC_CORE_LIB_IOMGR_EXECUTOR_H */
diff --git a/src/core/iomgr/fd_posix.c b/src/core/lib/iomgr/fd_posix.c
index 4ba7c5df94..72c924bdcb 100644
--- a/src/core/iomgr/fd_posix.c
+++ b/src/core/lib/iomgr/fd_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/fd_posix.h"
 
 #include <assert.h>
 #include <sys/socket.h>
@@ -46,7 +46,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
 
 #define CLOSURE_NOT_READY ((grpc_closure *)0)
 #define CLOSURE_READY ((grpc_closure *)1)
@@ -92,7 +92,7 @@ static grpc_fd *alloc_fd(int fd) {
     gpr_mu_init(&r->mu);
   }
 
-  gpr_atm_rel_store(&r->refst, 1);
+  gpr_mu_lock(&r->mu);
   r->shutdown = 0;
   r->read_closure = CLOSURE_NOT_READY;
   r->write_closure = CLOSURE_NOT_READY;
@@ -104,6 +104,9 @@ static grpc_fd *alloc_fd(int fd) {
   r->on_done_closure = NULL;
   r->closed = 0;
   r->released = 0;
+  gpr_atm_rel_store(&r->refst, 1);
+  gpr_mu_unlock(&r->mu);
+
   return r;
 }
 
diff --git a/src/core/iomgr/fd_posix.h b/src/core/lib/iomgr/fd_posix.h
index 17e7de88ff..69d09ef5e3 100644
--- a/src/core/iomgr/fd_posix.h
+++ b/src/core/lib/iomgr/fd_posix.h
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_FD_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_FD_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_FD_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_FD_POSIX_H
 
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/pollset.h"
 #include <grpc/support/atm.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/pollset.h"
 
 typedef struct grpc_fd grpc_fd;
 
@@ -189,4 +189,4 @@ void grpc_fd_unref(grpc_fd *fd);
 void grpc_fd_global_init(void);
 void grpc_fd_global_shutdown(void);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_FD_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_FD_POSIX_H */
diff --git a/src/core/iomgr/iocp_windows.c b/src/core/lib/iomgr/iocp_windows.c
index 807729708e..682a32c0da 100644
--- a/src/core/iomgr/iocp_windows.c
+++ b/src/core/lib/iomgr/iocp_windows.c
@@ -37,15 +37,15 @@
 
 #include <winsock2.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/log_win32.h>
-#include <grpc/support/alloc.h>
 #include <grpc/support/thd.h>
 
-#include "src/core/iomgr/timer.h"
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/timer.h"
 
 static ULONG g_iocp_kick_token;
 static OVERLAPPED g_iocp_custom_overlap;
@@ -71,7 +71,8 @@ static DWORD deadline_to_millis_timeout(gpr_timespec deadline,
       timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN)));
 }
 
-void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) {
+grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx,
+                                     gpr_timespec deadline) {
   BOOL success;
   DWORD bytes = 0;
   DWORD flags = 0;
@@ -84,14 +85,14 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) {
       g_iocp, &bytes, &completion_key, &overlapped,
       deadline_to_millis_timeout(deadline, gpr_now(deadline.clock_type)));
   if (success == 0 && overlapped == NULL) {
-    return;
+    return GRPC_IOCP_WORK_TIMEOUT;
   }
   GPR_ASSERT(completion_key && overlapped);
   if (overlapped == &g_iocp_custom_overlap) {
     gpr_atm_full_fetch_add(&g_custom_events, -1);
     if (completion_key == (ULONG_PTR)&g_iocp_kick_token) {
       /* We were awoken from a kick. */
-      return;
+      return GRPC_IOCP_WORK_KICK;
     }
     gpr_log(GPR_ERROR, "Unknown custom completion key.");
     abort();
@@ -121,6 +122,7 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) {
   }
   gpr_mu_unlock(&socket->state_mu);
   grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
+  return GRPC_IOCP_WORK_WORK;
 }
 
 void grpc_iocp_init(void) {
@@ -140,10 +142,12 @@ void grpc_iocp_kick(void) {
 
 void grpc_iocp_flush(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_iocp_work_status work_status;
 
   do {
-    grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC));
-  } while (grpc_exec_ctx_flush(&exec_ctx));
+    work_status = grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC));
+  } while (work_status == GRPC_IOCP_WORK_KICK ||
+           grpc_exec_ctx_flush(&exec_ctx));
 }
 
 void grpc_iocp_shutdown(void) {
diff --git a/src/core/iomgr/iocp_windows.h b/src/core/lib/iomgr/iocp_windows.h
index 75f3ba8477..856c837fb4 100644
--- a/src/core/iomgr/iocp_windows.h
+++ b/src/core/lib/iomgr/iocp_windows.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,21 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_IOCP_WINDOWS_H
-#define GRPC_INTERNAL_CORE_IOMGR_IOCP_WINDOWS_H
+#ifndef GRPC_CORE_LIB_IOMGR_IOCP_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_IOCP_WINDOWS_H
 
 #include <grpc/support/sync.h>
 
-#include "src/core/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/socket_windows.h"
 
-void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline);
+typedef enum {
+  GRPC_IOCP_WORK_WORK,
+  GRPC_IOCP_WORK_TIMEOUT,
+  GRPC_IOCP_WORK_KICK
+} grpc_iocp_work_status;
+
+grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx,
+                                     gpr_timespec deadline);
 void grpc_iocp_init(void);
 void grpc_iocp_kick(void);
 void grpc_iocp_flush(void);
@@ -53,4 +60,4 @@ void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx,
                                 grpc_winsocket *winsocket,
                                 grpc_closure *closure);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_IOCP_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_IOCP_WINDOWS_H */
diff --git a/src/core/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c
index 9c89c2c08a..bb544c8280 100644
--- a/src/core/iomgr/iomgr.c
+++ b/src/core/lib/iomgr/iomgr.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/iomgr.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -43,10 +43,11 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/timer.h"
-#include "src/core/support/env.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/string.h"
 
 static gpr_mu g_mu;
 static gpr_cv g_rcv;
@@ -57,6 +58,7 @@ void grpc_iomgr_init(void) {
   g_shutdown = 0;
   gpr_mu_init(&g_mu);
   gpr_cv_init(&g_rcv);
+  grpc_exec_ctx_global_init();
   grpc_timer_list_init(gpr_now(GPR_CLOCK_MONOTONIC));
   g_root_object.next = g_root_object.prev = &g_root_object;
   g_root_object.name = "root";
@@ -138,6 +140,7 @@ void grpc_iomgr_shutdown(void) {
 
   grpc_pollset_global_shutdown();
   grpc_iomgr_platform_shutdown();
+  grpc_exec_ctx_global_shutdown();
   gpr_mu_destroy(&g_mu);
   gpr_cv_destroy(&g_rcv);
 }
diff --git a/src/core/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h
index c9ea84c605..babf0a85b7 100644
--- a/src/core/iomgr/iomgr.h
+++ b/src/core/lib/iomgr/iomgr.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_IOMGR_H
-#define GRPC_INTERNAL_CORE_IOMGR_IOMGR_H
+#ifndef GRPC_CORE_LIB_IOMGR_IOMGR_H
+#define GRPC_CORE_LIB_IOMGR_IOMGR_H
 
 /** Initializes the iomgr. */
 void grpc_iomgr_init(void);
@@ -40,4 +40,4 @@ void grpc_iomgr_init(void);
 /** Signals the intention to shutdown the iomgr. */
 void grpc_iomgr_shutdown(void);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_H */
+#endif /* GRPC_CORE_LIB_IOMGR_IOMGR_H */
diff --git a/src/core/iomgr/iomgr_internal.h b/src/core/lib/iomgr/iomgr_internal.h
index ac2c46ebe6..0963630c61 100644
--- a/src/core/iomgr/iomgr_internal.h
+++ b/src/core/lib/iomgr/iomgr_internal.h
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H
-#define GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H
+#ifndef GRPC_CORE_LIB_IOMGR_IOMGR_INTERNAL_H
+#define GRPC_CORE_LIB_IOMGR_IOMGR_INTERNAL_H
 
 #include <stdbool.h>
 
-#include "src/core/iomgr/iomgr.h"
 #include <grpc/support/sync.h>
+#include "src/core/lib/iomgr/iomgr.h"
 
 typedef struct grpc_iomgr_object {
   char *name;
@@ -59,4 +59,4 @@ void grpc_iomgr_platform_shutdown(void);
 
 bool grpc_iomgr_abort_on_leaks(void);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H */
+#endif /* GRPC_CORE_LIB_IOMGR_IOMGR_INTERNAL_H */
diff --git a/src/core/iomgr/iomgr_posix.c b/src/core/lib/iomgr/iomgr_posix.c
index fecb7b9760..e4990f7bce 100644
--- a/src/core/iomgr/iomgr_posix.c
+++ b/src/core/lib/iomgr/iomgr_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,10 +35,10 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/iomgr_posix.h"
-#include "src/core/debug/trace.h"
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/tcp_posix.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/iomgr_posix.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
 
 void grpc_iomgr_platform_init(void) {
   grpc_fd_global_init();
diff --git a/src/core/iomgr/iomgr_posix.h b/src/core/lib/iomgr/iomgr_posix.h
index 068a5c6d7c..6a8996e403 100644
--- a/src/core/iomgr/iomgr_posix.h
+++ b/src/core/lib/iomgr/iomgr_posix.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,9 +31,9 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_IOMGR_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_IOMGR_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_IOMGR_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_IOMGR_POSIX_H
 
-#include "src/core/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_IOMGR_POSIX_H */
diff --git a/src/core/iomgr/iomgr_windows.c b/src/core/lib/iomgr/iomgr_windows.c
index 14775516bb..af7e616394 100644
--- a/src/core/iomgr/iomgr_windows.c
+++ b/src/core/lib/iomgr/iomgr_windows.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,13 +35,13 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_win32.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/iomgr/socket_windows.h"
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/socket_windows.h"
 
 /* Windows' io manager is going to be fully designed using IO completion
    ports. All of what we're doing here is basically make sure that
diff --git a/src/core/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h
index 92a0374ddd..6156124862 100644
--- a/src/core/iomgr/pollset.h
+++ b/src/core/lib/iomgr/pollset.h
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_H
+#ifndef GRPC_CORE_LIB_IOMGR_POLLSET_H
+#define GRPC_CORE_LIB_IOMGR_POLLSET_H
 
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
 
@@ -55,7 +55,7 @@ typedef struct grpc_pollset_worker grpc_pollset_worker;
 size_t grpc_pollset_size(void);
 void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu);
 /* Begin shutting down the pollset, and call closure when done.
- * GRPC_POLLSET_MU(pollset) must be held */
+ * pollset's mutex must be held */
 void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                            grpc_closure *closure);
 /** Reset the pollset to its initial state (perhaps with some cached objects);
@@ -66,16 +66,16 @@ void grpc_pollset_destroy(grpc_pollset *pollset);
 /* Do some work on a pollset.
    May involve invoking asynchronous callbacks, or actually polling file
    descriptors.
-   Requires GRPC_POLLSET_MU(pollset) locked.
-   May unlock GRPC_POLLSET_MU(pollset) during its execution.
+   Requires pollset's mutex locked.
+   May unlock its mutex during its execution.
 
    worker is a (platform-specific) handle that can be used to wake up
    from grpc_pollset_work before any events are received and before the timeout
    has expired. It is both initialized and destroyed by grpc_pollset_work.
    Initialization of worker is guaranteed to occur BEFORE the
-   GRPC_POLLSET_MU(pollset) is released for the first time by
-   grpc_pollset_work, and it is guaranteed that GRPC_POLLSET_MU(pollset) will
-   not be released by grpc_pollset_work AFTER worker has been destroyed.
+   pollset's mutex is released for the first time by grpc_pollset_work
+   and it is guaranteed that it will not be released by grpc_pollset_work
+   AFTER worker has been destroyed.
 
    Tries not to block past deadline.
    May call grpc_closure_list_run on grpc_closure_list, without holding the
@@ -91,4 +91,4 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 void grpc_pollset_kick(grpc_pollset *pollset,
                        grpc_pollset_worker *specific_worker);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */
+#endif /* GRPC_CORE_LIB_IOMGR_POLLSET_H */
diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/lib/iomgr/pollset_multipoller_with_epoll.c
index 2e0f27fab8..fa1b0d2d84 100644
--- a/src/core/iomgr/pollset_multipoller_with_epoll.c
+++ b/src/core/lib/iomgr/pollset_multipoller_with_epoll.c
@@ -44,10 +44,10 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/block_annotate.h"
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/block_annotate.h"
 
 struct epoll_fd_list {
   int *epoll_fds;
diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/lib/iomgr/pollset_multipoller_with_poll_posix.c
index 92d6fb7241..9b33f6dbb2 100644
--- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c
+++ b/src/core/lib/iomgr/pollset_multipoller_with_poll_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
 
 #include <errno.h>
 #include <poll.h>
@@ -46,10 +46,10 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/support/block_annotate.h"
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/support/block_annotate.h"
 
 typedef struct {
   /* all polled fds */
diff --git a/src/core/iomgr/pollset_posix.c b/src/core/lib/iomgr/pollset_posix.c
index e895a77884..259c7bc194 100644
--- a/src/core/iomgr/pollset_posix.c
+++ b/src/core/lib/iomgr/pollset_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -47,11 +47,11 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/socket_utils_posix.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/block_annotate.h"
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/block_annotate.h"
 
 GPR_TLS_DECL(g_current_thread_poller);
 GPR_TLS_DECL(g_current_thread_worker);
diff --git a/src/core/iomgr/pollset_posix.h b/src/core/lib/iomgr/pollset_posix.h
index bbedb66b00..7d8e9fc279 100644
--- a/src/core/iomgr/pollset_posix.h
+++ b/src/core/lib/iomgr/pollset_posix.h
@@ -31,17 +31,17 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_POLLSET_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_POLLSET_POSIX_H
 
 #include <poll.h>
 
 #include <grpc/support/sync.h>
 
-#include "src/core/iomgr/exec_ctx.h"
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 typedef struct grpc_pollset_vtable grpc_pollset_vtable;
 
@@ -150,4 +150,4 @@ typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int);
 extern grpc_poll_function_type grpc_poll_function;
 extern grpc_wakeup_fd grpc_global_wakeup_fd;
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_POLLSET_POSIX_H */
diff --git a/src/core/iomgr/pollset_set.h b/src/core/lib/iomgr/pollset_set.h
index dddcd8313f..fb29d692d7 100644
--- a/src/core/iomgr/pollset_set.h
+++ b/src/core/lib/iomgr/pollset_set.h
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_H
+#ifndef GRPC_CORE_LIB_IOMGR_POLLSET_SET_H
+#define GRPC_CORE_LIB_IOMGR_POLLSET_SET_H
 
-#include "src/core/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset.h"
 
 /* A grpc_pollset_set is a set of pollsets that are interested in an
    action. Adding a pollset to a pollset_set automatically adds any
@@ -58,4 +58,4 @@ void grpc_pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
                                       grpc_pollset_set *bag,
                                       grpc_pollset_set *item);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */
+#endif /* GRPC_CORE_LIB_IOMGR_POLLSET_SET_H */
diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/lib/iomgr/pollset_set_posix.c
index 9dc9aff4a8..d6142f9b6b 100644
--- a/src/core/iomgr/pollset_set_posix.c
+++ b/src/core/lib/iomgr/pollset_set_posix.c
@@ -41,8 +41,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/iomgr/pollset_set_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/pollset_set_posix.h"
 
 struct grpc_pollset_set {
   gpr_mu mu;
diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/lib/iomgr/pollset_set_posix.h
index 7d1aaf4181..4e6b063c6f 100644
--- a/src/core/iomgr/pollset_set_posix.h
+++ b/src/core/lib/iomgr/pollset_set_posix.h
@@ -31,15 +31,15 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_POLLSET_SET_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_POLLSET_SET_POSIX_H
 
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/pollset_set.h"
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 
 void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
                              grpc_pollset_set *pollset_set, grpc_fd *fd);
 void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
                              grpc_pollset_set *pollset_set, grpc_fd *fd);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_POLLSET_SET_POSIX_H */
diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/lib/iomgr/pollset_set_windows.c
index 3b8eca28e6..720fc331ed 100644
--- a/src/core/iomgr/pollset_set_windows.c
+++ b/src/core/lib/iomgr/pollset_set_windows.c
@@ -35,9 +35,9 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/iomgr/pollset_set_windows.h"
+#include "src/core/lib/iomgr/pollset_set_windows.h"
 
-grpc_pollset_set* grpc_pollset_set_create(pollset_set) { return NULL; }
+grpc_pollset_set* grpc_pollset_set_create(void) { return NULL; }
 
 void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {}
 
diff --git a/src/core/iomgr/pollset_set_windows.h b/src/core/lib/iomgr/pollset_set_windows.h
index 9661cd2c39..7c2cea23de 100644
--- a/src/core/iomgr/pollset_set_windows.h
+++ b/src/core/lib/iomgr/pollset_set_windows.h
@@ -31,9 +31,9 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H
+#ifndef GRPC_CORE_LIB_IOMGR_POLLSET_SET_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_POLLSET_SET_WINDOWS_H
 
-#include "src/core/iomgr/pollset_set.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_POLLSET_SET_WINDOWS_H */
diff --git a/src/core/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c
index c7f30f435f..6b339127a8 100644
--- a/src/core/iomgr/pollset_windows.c
+++ b/src/core/lib/iomgr/pollset_windows.c
@@ -38,10 +38,10 @@
 #include <grpc/support/log.h>
 #include <grpc/support/thd.h>
 
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/pollset_windows.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset_windows.h"
 
 gpr_mu grpc_polling_mu;
 static grpc_pollset_worker *g_active_poller;
diff --git a/src/core/iomgr/pollset_windows.h b/src/core/lib/iomgr/pollset_windows.h
index dc0b7a4104..fa9553ffea 100644
--- a/src/core/iomgr/pollset_windows.h
+++ b/src/core/lib/iomgr/pollset_windows.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H
-#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H
+#ifndef GRPC_CORE_LIB_IOMGR_POLLSET_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_POLLSET_WINDOWS_H
 
 #include <grpc/support/sync.h>
 
-#include "src/core/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/socket_windows.h"
 
 /* There isn't really any such thing as a pollset under Windows, due to the
    nature of the IO completion ports. A Windows "pollset" is merely a mutex
@@ -72,4 +72,4 @@ struct grpc_pollset {
   grpc_closure *on_shutdown;
 };
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_POLLSET_WINDOWS_H */
diff --git a/src/core/iomgr/resolve_address.h b/src/core/lib/iomgr/resolve_address.h
index 01eedffa88..f748288685 100644
--- a/src/core/iomgr/resolve_address.h
+++ b/src/core/lib/iomgr/resolve_address.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_RESOLVE_ADDRESS_H
-#define GRPC_INTERNAL_CORE_IOMGR_RESOLVE_ADDRESS_H
+#ifndef GRPC_CORE_LIB_IOMGR_RESOLVE_ADDRESS_H
+#define GRPC_CORE_LIB_IOMGR_RESOLVE_ADDRESS_H
 
 #include <stddef.h>
-#include "src/core/iomgr/exec_ctx.h"
-#include "src/core/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr.h"
 
 #define GRPC_MAX_SOCKADDR_SIZE 128
 
@@ -66,7 +66,7 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addresses);
 
 /* Resolve addr in a blocking fashion. Returns NULL on failure. On success,
    result must be freed with grpc_resolved_addresses_destroy. */
-grpc_resolved_addresses *grpc_blocking_resolve_address(
-    const char *addr, const char *default_port);
+extern grpc_resolved_addresses *(*grpc_blocking_resolve_address)(
+    const char *name, const char *default_port);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_RESOLVE_ADDRESS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_RESOLVE_ADDRESS_H */
diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c
index c51745b918..ebecb39c16 100644
--- a/src/core/iomgr/resolve_address_posix.c
+++ b/src/core/lib/iomgr/resolve_address_posix.c
@@ -34,18 +34,12 @@
 #include <grpc/support/port_platform.h>
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/sockaddr.h"
-#include "src/core/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
-#include <sys/types.h>
-#include <sys/un.h>
 #include <string.h>
+#include <sys/types.h>
 
-#include "src/core/iomgr/executor.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/support/block_annotate.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
@@ -53,6 +47,12 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/string.h"
 
 typedef struct {
   char *name;
@@ -62,7 +62,7 @@ typedef struct {
   void *arg;
 } request;
 
-grpc_resolved_addresses *grpc_blocking_resolve_address(
+static grpc_resolved_addresses *blocking_resolve_address_impl(
     const char *name, const char *default_port) {
   struct addrinfo hints;
   struct addrinfo *result = NULL, *resp;
@@ -71,18 +71,10 @@ grpc_resolved_addresses *grpc_blocking_resolve_address(
   int s;
   size_t i;
   grpc_resolved_addresses *addrs = NULL;
-  struct sockaddr_un *un;
 
   if (name[0] == 'u' && name[1] == 'n' && name[2] == 'i' && name[3] == 'x' &&
       name[4] == ':' && name[5] != 0) {
-    addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
-    addrs->naddrs = 1;
-    addrs->addrs = gpr_malloc(sizeof(grpc_resolved_address));
-    un = (struct sockaddr_un *)addrs->addrs->addr;
-    un->sun_family = AF_UNIX;
-    strcpy(un->sun_path, name + 5);
-    addrs->addrs->len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
-    return addrs;
+    return grpc_resolve_unix_domain_address(name + 5);
   }
 
   /* parse name, splitting it into host and port parts */
@@ -150,6 +142,9 @@ done:
   return addrs;
 }
 
+grpc_resolved_addresses *(*grpc_blocking_resolve_address)(
+    const char *name, const char *default_port) = blocking_resolve_address_impl;
+
 /* Callback to be passed to grpc_executor to asynch-ify
  * grpc_blocking_resolve_address */
 static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) {
diff --git a/src/core/iomgr/resolve_address_windows.c b/src/core/lib/iomgr/resolve_address_windows.c
index 28c8661e73..bde1f1b7f7 100644
--- a/src/core/iomgr/resolve_address_windows.c
+++ b/src/core/lib/iomgr/resolve_address_windows.c
@@ -34,17 +34,12 @@
 #include <grpc/support/port_platform.h>
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/iomgr/sockaddr.h"
-#include "src/core/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
-#include <sys/types.h>
 #include <string.h>
+#include <sys/types.h>
 
-#include "src/core/iomgr/executor.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/support/block_annotate.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
@@ -52,6 +47,11 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/string.h"
 
 typedef struct {
   char *name;
@@ -61,7 +61,7 @@ typedef struct {
   void *arg;
 } request;
 
-grpc_resolved_addresses *grpc_blocking_resolve_address(
+static grpc_resolved_addresses *blocking_resolve_address_impl(
     const char *name, const char *default_port) {
   struct addrinfo hints;
   struct addrinfo *result = NULL, *resp;
@@ -133,6 +133,9 @@ done:
   return addrs;
 }
 
+grpc_resolved_addresses *(*grpc_blocking_resolve_address)(
+    const char *name, const char *default_port) = blocking_resolve_address_impl;
+
 /* Callback to be passed to grpc_executor to asynch-ify
  * grpc_blocking_resolve_address */
 static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) {
diff --git a/src/core/iomgr/sockaddr.h b/src/core/lib/iomgr/sockaddr.h
index e41e1ec6b4..66a930ee6a 100644
--- a/src/core/iomgr/sockaddr.h
+++ b/src/core/lib/iomgr/sockaddr.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,17 +31,17 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_H
-#define GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_H
+#define GRPC_CORE_LIB_IOMGR_SOCKADDR_H
 
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_WIN32
-#include "src/core/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_win32.h"
 #endif
 
 #ifdef GPR_POSIX_SOCKETADDR
-#include "src/core/iomgr/sockaddr_posix.h"
+#include "src/core/lib/iomgr/sockaddr_posix.h"
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_H */
diff --git a/src/core/iomgr/sockaddr_posix.h b/src/core/lib/iomgr/sockaddr_posix.h
index 388abb3306..79a7467c5d 100644
--- a/src/core/iomgr/sockaddr_posix.h
+++ b/src/core/lib/iomgr/sockaddr_posix.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_SOCKADDR_POSIX_H
 
 #include <arpa/inet.h>
+#include <netdb.h>
+#include <netinet/in.h>
 #include <sys/socket.h>
 #include <sys/types.h>
-#include <netinet/in.h>
-#include <netdb.h>
 #include <unistd.h>
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_POSIX_H */
diff --git a/src/core/iomgr/sockaddr_utils.c b/src/core/lib/iomgr/sockaddr_utils.c
index 61006d7a7a..127d95c618 100644
--- a/src/core/iomgr/sockaddr_utils.c
+++ b/src/core/lib/iomgr/sockaddr_utils.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,22 +31,19 @@
  *
  */
 
-#include "src/core/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
 
 #include <errno.h>
 #include <string.h>
 
-#ifdef GPR_POSIX_SOCKET
-#include <sys/un.h>
-#endif
-
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/string.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/support/string.h"
 
 static const uint8_t kV4MappedPrefix[] = {0, 0, 0, 0, 0,    0,
                                           0, 0, 0, 0, 0xff, 0xff};
@@ -191,14 +188,9 @@ char *grpc_sockaddr_to_uri(const struct sockaddr *addr) {
       gpr_asprintf(&result, "ipv6:%s", temp);
       gpr_free(temp);
       return result;
-#ifdef GPR_POSIX_SOCKET
-    case AF_UNIX:
-      gpr_asprintf(&result, "unix:%s", ((struct sockaddr_un *)addr)->sun_path);
-      return result;
-#endif
+    default:
+      return grpc_sockaddr_to_uri_unix_if_possible(addr);
   }
-
-  return NULL;
 }
 
 int grpc_sockaddr_get_port(const struct sockaddr *addr) {
@@ -207,9 +199,10 @@ int grpc_sockaddr_get_port(const struct sockaddr *addr) {
       return ntohs(((struct sockaddr_in *)addr)->sin_port);
     case AF_INET6:
       return ntohs(((struct sockaddr_in6 *)addr)->sin6_port);
-    case AF_UNIX:
-      return 1;
     default:
+      if (grpc_is_unix_socket(addr)) {
+        return 1;
+      }
       gpr_log(GPR_ERROR, "Unknown socket family %d in grpc_sockaddr_get_port",
               addr->sa_family);
       return 0;
diff --git a/src/core/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h
index 6f7a279900..20a3e3bec3 100644
--- a/src/core/iomgr/sockaddr_utils.h
+++ b/src/core/lib/iomgr/sockaddr_utils.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_UTILS_H
-#define GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_UTILS_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H
+#define GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H
 
-#include "src/core/iomgr/sockaddr.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
 /* Returns true if addr is an IPv4-mapped IPv6 address within the
    ::ffff:0.0.0.0/96 range, or false otherwise.
@@ -86,4 +86,4 @@ int grpc_sockaddr_to_string(char **out, const struct sockaddr *addr,
 
 char *grpc_sockaddr_to_uri(const struct sockaddr *addr);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_UTILS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H */
diff --git a/src/core/iomgr/sockaddr_win32.h b/src/core/lib/iomgr/sockaddr_win32.h
index 8e3946a7d8..2dd7111240 100644
--- a/src/core/iomgr/sockaddr_win32.h
+++ b/src/core/lib/iomgr/sockaddr_win32.h
@@ -31,11 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_WIN32_H
-#define GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_WIN32_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H
+#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H
 
 #include <winsock2.h>
 #include <ws2tcpip.h>
+
+// must be included after the above
 #include <mswsock.h>
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_WIN32_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H */
diff --git a/src/core/iomgr/socket_utils_common_posix.c b/src/core/lib/iomgr/socket_utils_common_posix.c
index a9af594700..b433aee7fa 100644
--- a/src/core/iomgr/socket_utils_common_posix.c
+++ b/src/core/lib/iomgr/socket_utils_common_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,26 +35,26 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
 
 #include <arpa/inet.h>
-#include <limits.h>
+#include <errno.h>
 #include <fcntl.h>
+#include <limits.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <stdio.h>
-#include <sys/types.h>
+#include <string.h>
 #include <sys/socket.h>
+#include <sys/types.h>
 #include <unistd.h>
-#include <string.h>
-#include <errno.h>
 
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/support/string.h"
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/support/string.h"
 
 /* set a socket to non blocking mode */
 int grpc_set_socket_nonblocking(int fd, int non_blocking) {
@@ -89,6 +89,28 @@ int grpc_set_socket_no_sigpipe_if_possible(int fd) {
 #endif
 }
 
+int grpc_set_socket_ip_pktinfo_if_possible(int fd) {
+#ifdef GPR_HAVE_IP_PKTINFO
+  int get_local_ip = 1;
+  return 0 == setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &get_local_ip,
+                         sizeof(get_local_ip));
+#else
+  (void)fd;
+  return 1;
+#endif
+}
+
+int grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd) {
+#ifdef GPR_HAVE_IPV6_RECVPKTINFO
+  int get_local_ip = 1;
+  return 0 == setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &get_local_ip,
+                         sizeof(get_local_ip));
+#else
+  (void)fd;
+  return 1;
+#endif
+}
+
 /* set a socket to close on exec */
 int grpc_set_socket_cloexec(int fd, int close_on_exec) {
   int oldflags = fcntl(fd, F_GETFD, 0);
diff --git a/src/core/iomgr/socket_utils_linux.c b/src/core/lib/iomgr/socket_utils_linux.c
index a87625262b..e7dfe892ca 100644
--- a/src/core/iomgr/socket_utils_linux.c
+++ b/src/core/lib/iomgr/socket_utils_linux.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,10 +35,10 @@
 
 #ifdef GPR_LINUX_SOCKETUTILS
 
-#include "src/core/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
 
-#include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/types.h>
 
 int grpc_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen,
                  int nonblock, int cloexec) {
diff --git a/src/core/iomgr/socket_utils_posix.c b/src/core/lib/iomgr/socket_utils_posix.c
index 3c56b46744..b2fa00c5c1 100644
--- a/src/core/iomgr/socket_utils_posix.c
+++ b/src/core/lib/iomgr/socket_utils_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKETUTILS
 
-#include "src/core/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
 
 #include <fcntl.h>
 #include <sys/socket.h>
diff --git a/src/core/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h
index d330d1986e..f73ad6317d 100644
--- a/src/core/iomgr/socket_utils_posix.h
+++ b/src/core/lib/iomgr/socket_utils_posix.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_SOCKET_UTILS_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_SOCKET_UTILS_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKET_UTILS_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_SOCKET_UTILS_POSIX_H
 
-#include <unistd.h>
 #include <sys/socket.h>
+#include <unistd.h>
 
 /* a wrapper for accept or accept4 */
 int grpc_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen,
@@ -68,6 +68,16 @@ int grpc_ipv6_loopback_available(void);
    If SO_NO_SIGPIPE is not available, returns 1. */
 int grpc_set_socket_no_sigpipe_if_possible(int fd);
 
+/* Tries to set IP_PKTINFO if available on this platform.
+   Returns 1 on success, 0 on failure.
+   If IP_PKTINFO is not available, returns 1. */
+int grpc_set_socket_ip_pktinfo_if_possible(int fd);
+
+/* Tries to set IPV6_RECVPKTINFO if available on this platform.
+   Returns 1 on success, 0 on failure.
+   If IPV6_RECVPKTINFO is not available, returns 1. */
+int grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd);
+
 /* An enum to keep track of IPv4/IPv6 socket modes.
 
    Currently, this information is only used when a socket is first created, but
@@ -110,4 +120,4 @@ extern int grpc_forbid_dualstack_sockets_for_testing;
 int grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
                                  int protocol, grpc_dualstack_mode *dsmode);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKET_UTILS_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKET_UTILS_POSIX_H */
diff --git a/src/core/iomgr/socket_windows.c b/src/core/lib/iomgr/socket_windows.c
index fafb7b6622..1023a6d4f8 100644
--- a/src/core/iomgr/socket_windows.c
+++ b/src/core/lib/iomgr/socket_windows.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,8 @@
 #ifdef GPR_WINSOCK_SOCKET
 
 #include <winsock2.h>
+
+// must be included after winsock2.h
 #include <mswsock.h>
 
 #include <grpc/support/alloc.h>
@@ -43,11 +45,11 @@
 #include <grpc/support/log_win32.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/pollset_windows.h"
-#include "src/core/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset_windows.h"
+#include "src/core/lib/iomgr/socket_windows.h"
 
 grpc_winsocket *grpc_winsocket_create(SOCKET socket, const char *name) {
   char *final_name;
diff --git a/src/core/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h
index dfbfabe1f9..74447896c9 100644
--- a/src/core/iomgr/socket_windows.h
+++ b/src/core/lib/iomgr/socket_windows.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,17 +31,17 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_SOCKET_WINDOWS_H
-#define GRPC_INTERNAL_CORE_IOMGR_SOCKET_WINDOWS_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H
 
 #include <grpc/support/port_platform.h>
 #include <winsock2.h>
 
-#include <grpc/support/sync.h>
 #include <grpc/support/atm.h>
+#include <grpc/support/sync.h>
 
-#include "src/core/iomgr/iomgr_internal.h"
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
 
 /* This holds the data for an outstanding read or write on a socket.
    The mutex to protect the concurrent access to that data is the one
@@ -108,4 +108,4 @@ void grpc_winsocket_shutdown(grpc_winsocket *socket);
 /* Destroy a socket. Should only be called if there's no pending operation. */
 void grpc_winsocket_destroy(grpc_winsocket *socket);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKET_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H */
diff --git a/src/core/iomgr/tcp_client.h b/src/core/lib/iomgr/tcp_client.h
index 5e18e71ca2..6bbe26445a 100644
--- a/src/core/iomgr/tcp_client.h
+++ b/src/core/lib/iomgr/tcp_client.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H
-#define GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H
+#ifndef GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H
+#define GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/pollset_set.h"
-#include "src/core/iomgr/sockaddr.h"
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 
 /* Asynchronously connect to an address (specified as (addr, len)), and call
    cb with arg and the completed connection when done (or call cb with arg and
@@ -50,4 +50,4 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_connect,
                              const struct sockaddr *addr, size_t addr_len,
                              gpr_timespec deadline);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H */
diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index 15727856ab..b8ef643298 100644
--- a/src/core/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/tcp_client.h"
+#include "src/core/lib/iomgr/tcp_client.h"
 
 #include <errno.h>
 #include <netinet/in.h>
@@ -47,14 +47,15 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
-#include "src/core/iomgr/iomgr_posix.h"
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/iomgr/pollset_set_posix.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/iomgr/socket_utils_posix.h"
-#include "src/core/iomgr/tcp_posix.h"
-#include "src/core/iomgr/timer.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/iomgr/iomgr_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/pollset_set_posix.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/support/string.h"
 
 extern int grpc_tcp_trace;
 
@@ -77,13 +78,12 @@ static int prepare_socket(const struct sockaddr *addr, int fd) {
   }
 
   if (!grpc_set_socket_nonblocking(fd, 1) || !grpc_set_socket_cloexec(fd, 1) ||
-      (addr->sa_family != AF_UNIX && !grpc_set_socket_low_latency(fd, 1)) ||
+      (!grpc_is_unix_socket(addr) && !grpc_set_socket_low_latency(fd, 1)) ||
       !grpc_set_socket_no_sigpipe_if_possible(fd)) {
     gpr_log(GPR_ERROR, "Unable to configure socket %d: %s", fd,
             strerror(errno));
     goto error;
   }
-
   return 1;
 
 error:
diff --git a/src/core/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c
index 689c6f7b10..86b8d58975 100644
--- a/src/core/iomgr/tcp_client_windows.c
+++ b/src/core/lib/iomgr/tcp_client_windows.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_win32.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -43,13 +43,13 @@
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/timer.h"
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/tcp_client.h"
-#include "src/core/iomgr/tcp_windows.h"
-#include "src/core/iomgr/sockaddr.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/sockaddr.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/iomgr/tcp_windows.h"
+#include "src/core/lib/iomgr/timer.h"
 
 typedef struct {
   grpc_closure *on_done;
diff --git a/src/core/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index f74eb3fe51..1898d96901 100644
--- a/src/core/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/tcp_posix.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -51,11 +51,11 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 
-#include "src/core/debug/trace.h"
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/iomgr/pollset_set_posix.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/pollset_set_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
 
 #ifdef GPR_HAVE_MSG_NOSIGNAL
 #define SENDMSG_FLAGS MSG_NOSIGNAL
@@ -297,7 +297,7 @@ static flush_result tcp_flush(grpc_tcp *tcp) {
     unwind_slice_idx = tcp->outgoing_slice_idx;
     unwind_byte_idx = tcp->outgoing_byte_idx;
     for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count &&
-                           iov_size != MAX_WRITE_IOVEC;
+                       iov_size != MAX_WRITE_IOVEC;
          iov_size++) {
       iov[iov_size].iov_base =
           GPR_SLICE_START_PTR(
@@ -446,7 +446,7 @@ static char *tcp_get_peer(grpc_endpoint *ep) {
 }
 
 static const grpc_endpoint_vtable vtable = {
-    tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set,
+    tcp_read,     tcp_write,   tcp_add_to_pollset, tcp_add_to_pollset_set,
     tcp_shutdown, tcp_destroy, tcp_get_peer};
 
 grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
diff --git a/src/core/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h
index 2a40cdd385..09c4436f1f 100644
--- a/src/core/iomgr/tcp_posix.h
+++ b/src/core/lib/iomgr/tcp_posix.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TCP_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_TCP_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_TCP_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_TCP_POSIX_H
 /*
    Low level TCP "bottom half" implementation, for use by transports built on
    top of a TCP connection.
@@ -44,8 +44,8 @@
    otherwise specified.
 */
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/fd_posix.h"
 
 #define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192
 
@@ -68,4 +68,4 @@ int grpc_tcp_fd(grpc_endpoint *ep);
 void grpc_tcp_destroy_and_release_fd(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                                      int *fd, grpc_closure *done);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TCP_POSIX_H */
diff --git a/src/core/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h
index a39dd3bafc..81edb61997 100644
--- a/src/core/iomgr/tcp_server.h
+++ b/src/core/lib/iomgr/tcp_server.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H
-#define GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H
+#ifndef GRPC_CORE_LIB_IOMGR_TCP_SERVER_H
+#define GRPC_CORE_LIB_IOMGR_TCP_SERVER_H
 
-#include "src/core/iomgr/closure.h"
-#include "src/core/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/endpoint.h"
 
 /* Forward decl of grpc_tcp_server */
 typedef struct grpc_tcp_server grpc_tcp_server;
@@ -100,4 +100,4 @@ void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
    a call (exec_ctx!=NULL) to shutdown_complete. */
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TCP_SERVER_H */
diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index 5e07f8261c..ef1bf9aa94 100644
--- a/src/core/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -40,7 +40,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/tcp_server.h"
+#include "src/core/lib/iomgr/tcp_server.h"
 
 #include <errno.h>
 #include <fcntl.h>
@@ -52,20 +52,20 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <sys/un.h>
 #include <unistd.h>
 
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/iomgr/socket_utils_posix.h"
-#include "src/core/iomgr/tcp_posix.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/support/string.h"
 
 #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100
 
@@ -81,7 +81,6 @@ struct grpc_tcp_listener {
   union {
     uint8_t untyped[GRPC_MAX_SOCKADDR_SIZE];
     struct sockaddr sockaddr;
-    struct sockaddr_un un;
   } addr;
   size_t addr_len;
   int port;
@@ -98,14 +97,6 @@ struct grpc_tcp_listener {
   int is_sibling;
 };
 
-static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) {
-  struct stat st;
-
-  if (stat(un->sun_path, &st) == 0 && (st.st_mode & S_IFMT) == S_IFSOCK) {
-    unlink(un->sun_path);
-  }
-}
-
 /* the overall server */
 struct grpc_tcp_server {
   gpr_refcount refs;
@@ -203,9 +194,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (s->head) {
     grpc_tcp_listener *sp;
     for (sp = s->head; sp; sp = sp->next) {
-      if (sp->addr.sockaddr.sa_family == AF_UNIX) {
-        unlink_if_unix_domain_socket(&sp->addr.un);
-      }
+      grpc_unlink_if_unix_domain_socket(&sp->addr.sockaddr);
       sp->destroyed_closure.cb = destroyed_port;
       sp->destroyed_closure.cb_arg = s;
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
@@ -281,7 +270,7 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
   }
 
   if (!grpc_set_socket_nonblocking(fd, 1) || !grpc_set_socket_cloexec(fd, 1) ||
-      (addr->sa_family != AF_UNIX && (!grpc_set_socket_low_latency(fd, 1) ||
+      (!grpc_is_unix_socket(addr) && (!grpc_set_socket_low_latency(fd, 1) ||
                                       !grpc_set_socket_reuse_addr(fd, 1))) ||
       !grpc_set_socket_no_sigpipe_if_possible(fd)) {
     gpr_log(GPR_ERROR, "Unable to configure socket %d: %s", fd,
@@ -451,9 +440,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
   if (s->tail != NULL) {
     port_index = s->tail->port_index + 1;
   }
-  if (((struct sockaddr *)addr)->sa_family == AF_UNIX) {
-    unlink_if_unix_domain_socket(addr);
-  }
+  grpc_unlink_if_unix_domain_socket((struct sockaddr *)addr);
 
   /* Check if this is a wildcard port, and if so, try to keep the port the same
      as some previously created listener. */
@@ -499,7 +486,6 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
     /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */
     if (port == 0 && sp != NULL) {
       grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port);
-      sp2 = sp;
     }
     addr = (struct sockaddr *)&wild4;
     addr_len = sizeof(wild4);
@@ -508,16 +494,18 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
   fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode);
   if (fd < 0) {
     gpr_log(GPR_ERROR, "Unable to create socket: %s", strerror(errno));
-  }
-  if (dsmode == GRPC_DSMODE_IPV4 &&
-      grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) {
-    addr = (struct sockaddr *)&addr4_copy;
-    addr_len = sizeof(addr4_copy);
-  }
-  sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index);
-  if (sp2 != NULL && sp != NULL) {
-    sp2->sibling = sp;
-    sp->is_sibling = 1;
+  } else {
+    if (dsmode == GRPC_DSMODE_IPV4 &&
+        grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) {
+      addr = (struct sockaddr *)&addr4_copy;
+      addr_len = sizeof(addr4_copy);
+    }
+    sp2 = sp;
+    sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index);
+    if (sp2 != NULL && sp != NULL) {
+      sp2->sibling = sp;
+      sp->is_sibling = 1;
+    }
   }
 
 done:
diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index ce930b8f41..3d6a29b2e2 100644
--- a/src/core/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -37,7 +37,7 @@
 
 #include <io.h>
 
-#include "src/core/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -46,11 +46,11 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/pollset_windows.h"
-#include "src/core/iomgr/socket_windows.h"
-#include "src/core/iomgr/tcp_server.h"
-#include "src/core/iomgr/tcp_windows.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/pollset_windows.h"
+#include "src/core/lib/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/iomgr/tcp_windows.h"
 
 #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100
 
@@ -240,8 +240,7 @@ static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx,
   sp->shutting_down = 0;
   gpr_mu_lock(&sp->server->mu);
   GPR_ASSERT(sp->server->active_ports > 0);
-  if (0 == --sp->server->active_ports &&
-      sp->server->shutdown_complete != NULL) {
+  if (0 == --sp->server->active_ports) {
     notify = 1;
   }
   gpr_mu_unlock(&sp->server->mu);
diff --git a/src/core/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c
index 038e4158c8..c1ce725f2c 100644
--- a/src/core/iomgr/tcp_windows.c
+++ b/src/core/lib/iomgr/tcp_windows.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_win32.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -44,12 +44,12 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/timer.h"
-#include "src/core/iomgr/iocp_windows.h"
-#include "src/core/iomgr/sockaddr.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/iomgr/socket_windows.h"
-#include "src/core/iomgr/tcp_client.h"
+#include "src/core/lib/iomgr/iocp_windows.h"
+#include "src/core/lib/iomgr/sockaddr.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/iomgr/timer.h"
 
 static int set_non_block(SOCKET sock) {
   int status;
@@ -146,8 +146,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, bool success) {
   grpc_winsocket_callback_info *info = &socket->read_info;
 
   if (success) {
-    if (socket->read_info.wsa_error != 0 && !tcp->shutting_down) {
-      if (socket->read_info.wsa_error != WSAECONNRESET) {
+    if (info->wsa_error != 0 && !tcp->shutting_down) {
+      if (info->wsa_error != WSAECONNRESET) {
         char *utf8_message = gpr_format_message(info->wsa_error);
         gpr_log(GPR_ERROR, "ReadFile overlapped error: %s", utf8_message);
         gpr_free(utf8_message);
@@ -306,7 +306,7 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
       ok = true;
       GPR_ASSERT(bytes_sent == tcp->write_slices->length);
     } else {
-      if (socket->read_info.wsa_error != WSAECONNRESET) {
+      if (info->wsa_error != WSAECONNRESET) {
         char *utf8_message = gpr_format_message(info->wsa_error);
         gpr_log(GPR_ERROR, "WSASend error: %s", utf8_message);
         gpr_free(utf8_message);
@@ -382,9 +382,9 @@ static char *win_get_peer(grpc_endpoint *ep) {
   return gpr_strdup(tcp->peer_string);
 }
 
-static grpc_endpoint_vtable vtable = {win_read, win_write, win_add_to_pollset,
-                                      win_add_to_pollset_set, win_shutdown,
-                                      win_destroy, win_get_peer};
+static grpc_endpoint_vtable vtable = {
+    win_read,     win_write,   win_add_to_pollset, win_add_to_pollset_set,
+    win_shutdown, win_destroy, win_get_peer};
 
 grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp));
diff --git a/src/core/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h
index deb3e48293..7a9ebd85eb 100644
--- a/src/core/iomgr/tcp_windows.h
+++ b/src/core/lib/iomgr/tcp_windows.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TCP_WINDOWS_H
-#define GRPC_INTERNAL_CORE_IOMGR_TCP_WINDOWS_H
+#ifndef GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H
 /*
    Low level TCP "bottom half" implementation, for use by transports built on
    top of a TCP connection.
@@ -44,8 +44,8 @@
    otherwise specified.
 */
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/socket_windows.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/socket_windows.h"
 
 /* Create a tcp endpoint given a winsock handle.
  * Takes ownership of the handle.
@@ -54,4 +54,4 @@ grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string);
 
 int grpc_tcp_prepare_socket(SOCKET sock);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H */
diff --git a/src/core/iomgr/time_averaged_stats.c b/src/core/lib/iomgr/time_averaged_stats.c
index e075db4373..f24d68087e 100644
--- a/src/core/iomgr/time_averaged_stats.c
+++ b/src/core/lib/iomgr/time_averaged_stats.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/iomgr/time_averaged_stats.h"
+#include "src/core/lib/iomgr/time_averaged_stats.h"
 
 void grpc_time_averaged_stats_init(grpc_time_averaged_stats* stats,
                                    double init_avg, double regress_weight,
diff --git a/src/core/iomgr/time_averaged_stats.h b/src/core/lib/iomgr/time_averaged_stats.h
index 4e9e3956c2..4a662e17ec 100644
--- a/src/core/iomgr/time_averaged_stats.h
+++ b/src/core/lib/iomgr/time_averaged_stats.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TIME_AVERAGED_STATS_H
-#define GRPC_INTERNAL_CORE_IOMGR_TIME_AVERAGED_STATS_H
+#ifndef GRPC_CORE_LIB_IOMGR_TIME_AVERAGED_STATS_H
+#define GRPC_CORE_LIB_IOMGR_TIME_AVERAGED_STATS_H
 
 /* This tracks a time-decaying weighted average.  It works by collecting
    batches of samples and then mixing their average into a time-decaying
@@ -85,4 +85,4 @@ void grpc_time_averaged_stats_add_sample(grpc_time_averaged_stats* stats,
    value. */
 double grpc_time_averaged_stats_update_average(grpc_time_averaged_stats* stats);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TIME_AVERAGED_STATS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TIME_AVERAGED_STATS_H */
diff --git a/src/core/iomgr/timer.c b/src/core/lib/iomgr/timer.c
index 8379fffad0..4748f9b270 100644
--- a/src/core/iomgr/timer.c
+++ b/src/core/lib/iomgr/timer.c
@@ -31,13 +31,13 @@
  *
  */
 
-#include "src/core/iomgr/timer.h"
+#include "src/core/lib/iomgr/timer.h"
 
-#include "src/core/iomgr/timer_heap.h"
-#include "src/core/iomgr/time_averaged_stats.h"
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/iomgr/time_averaged_stats.h"
+#include "src/core/lib/iomgr/timer_heap.h"
 
 #define INVALID_HEAP_INDEX 0xffffffffu
 
@@ -330,6 +330,18 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now,
 
     gpr_mu_unlock(&g_mu);
     gpr_mu_unlock(&g_checker_mu);
+  } else if (next != NULL) {
+    /* TODO(ctiller): this forces calling code to do an short poll, and
+       then retry the timer check (because this time through the timer list was
+       contended).
+
+       We could reduce the cost here dramatically by keeping a count of how many
+       currently active pollers got through the uncontended case above
+       successfully, and waking up other pollers IFF that count drops to zero.
+
+       Once that count is in place, this entire else branch could disappear. */
+    *next = gpr_time_min(
+        *next, gpr_time_add(now, gpr_time_from_millis(1, GPR_TIMESPAN)));
   }
 
   return (int)n;
diff --git a/src/core/iomgr/timer.h b/src/core/lib/iomgr/timer.h
index 9ad1e92f42..54f301c5ed 100644
--- a/src/core/iomgr/timer.h
+++ b/src/core/lib/iomgr/timer.h
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TIMER_H
-#define GRPC_INTERNAL_CORE_IOMGR_TIMER_H
+#ifndef GRPC_CORE_LIB_IOMGR_TIMER_H
+#define GRPC_CORE_LIB_IOMGR_TIMER_H
 
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/iomgr/exec_ctx.h"
 #include <grpc/support/port_platform.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr.h"
 
 typedef struct grpc_timer {
   gpr_timespec deadline;
@@ -96,7 +96,6 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer);
    *next is never guaranteed to be updated on any given execution; however,
    with high probability at least one thread in the system will see an update
    at any time slice. */
-
 bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
                       gpr_timespec *next);
 void grpc_timer_list_init(gpr_timespec now);
@@ -106,4 +105,4 @@ void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx);
 
 void grpc_kick_poller(void);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TIMER_H */
diff --git a/src/core/iomgr/timer_heap.c b/src/core/lib/iomgr/timer_heap.c
index 9d8be5c1fc..d43b6ccf75 100644
--- a/src/core/iomgr/timer_heap.c
+++ b/src/core/lib/iomgr/timer_heap.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/iomgr/timer_heap.h"
+#include "src/core/lib/iomgr/timer_heap.h"
 
 #include <string.h>
 
@@ -46,7 +46,7 @@
 static void adjust_upwards(grpc_timer **first, uint32_t i, grpc_timer *t) {
   while (i > 0) {
     uint32_t parent = (uint32_t)(((int)i - 1) / 2);
-    if (gpr_time_cmp(first[parent]->deadline, t->deadline) >= 0) break;
+    if (gpr_time_cmp(first[parent]->deadline, t->deadline) <= 0) break;
     first[i] = first[parent];
     first[i]->heap_index = i;
     i = parent;
@@ -62,16 +62,14 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length,
                              grpc_timer *t) {
   for (;;) {
     uint32_t left_child = 1u + 2u * i;
-    uint32_t right_child;
-    uint32_t next_i;
     if (left_child >= length) break;
-    right_child = left_child + 1;
-    next_i = right_child < length &&
-                     gpr_time_cmp(first[left_child]->deadline,
-                                  first[right_child]->deadline) < 0
-                 ? right_child
-                 : left_child;
-    if (gpr_time_cmp(t->deadline, first[next_i]->deadline) >= 0) break;
+    uint32_t right_child = left_child + 1;
+    uint32_t next_i = right_child < length &&
+                              gpr_time_cmp(first[left_child]->deadline,
+                                           first[right_child]->deadline) > 0
+                          ? right_child
+                          : left_child;
+    if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break;
     first[i] = first[next_i];
     first[i]->heap_index = i;
     i = next_i;
@@ -95,7 +93,7 @@ static void maybe_shrink(grpc_timer_heap *heap) {
 static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) {
   uint32_t i = timer->heap_index;
   uint32_t parent = (uint32_t)(((int)i - 1) / 2);
-  if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) < 0) {
+  if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) > 0) {
     adjust_upwards(heap->timers, i, timer);
   } else {
     adjust_downwards(heap->timers, i, heap->timer_count, timer);
diff --git a/src/core/iomgr/timer_heap.h b/src/core/lib/iomgr/timer_heap.h
index 2d220f1677..d5112cf0de 100644
--- a/src/core/iomgr/timer_heap.h
+++ b/src/core/lib/iomgr/timer_heap.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_TIMER_HEAP_H
-#define GRPC_INTERNAL_CORE_IOMGR_TIMER_HEAP_H
+#ifndef GRPC_CORE_LIB_IOMGR_TIMER_HEAP_H
+#define GRPC_CORE_LIB_IOMGR_TIMER_HEAP_H
 
-#include "src/core/iomgr/timer.h"
+#include "src/core/lib/iomgr/timer.h"
 
 typedef struct {
   grpc_timer **timers;
@@ -54,4 +54,4 @@ void grpc_timer_heap_pop(grpc_timer_heap *heap);
 
 int grpc_timer_heap_is_empty(grpc_timer_heap *heap);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_HEAP_H */
+#endif /* GRPC_CORE_LIB_IOMGR_TIMER_HEAP_H */
diff --git a/src/core/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c
index ef548cfe4d..9068109c3a 100644
--- a/src/core/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@@ -41,7 +41,7 @@
 #ifdef GRPC_NEED_UDP
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/udp_server.h"
+#include "src/core/lib/iomgr/udp_server.h"
 
 #include <errno.h>
 #include <fcntl.h>
@@ -52,20 +52,21 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <sys/un.h>
 #include <unistd.h>
 
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/pollset_posix.h"
-#include "src/core/iomgr/resolve_address.h"
-#include "src/core/iomgr/sockaddr_utils.h"
-#include "src/core/iomgr/socket_utils_posix.h"
-#include "src/core/support/string.h"
+#include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/sync.h>
 #include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/support/string.h"
 
 #define INIT_PORT_CAP 2
 
@@ -77,7 +78,6 @@ typedef struct {
   union {
     uint8_t untyped[GRPC_MAX_SOCKADDR_SIZE];
     struct sockaddr sockaddr;
-    struct sockaddr_un un;
   } addr;
   size_t addr_len;
   grpc_closure read_closure;
@@ -85,14 +85,6 @@ typedef struct {
   grpc_udp_server_read_cb read_cb;
 } server_port;
 
-static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) {
-  struct stat st;
-
-  if (stat(un->sun_path, &st) == 0 && (st.st_mode & S_IFMT) == S_IFSOCK) {
-    unlink(un->sun_path);
-  }
-}
-
 /* the overall server */
 struct grpc_udp_server {
   gpr_mu mu;
@@ -176,9 +168,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
   if (s->nports) {
     for (i = 0; i < s->nports; i++) {
       server_port *sp = &s->ports[i];
-      if (sp->addr.sockaddr.sa_family == AF_UNIX) {
-        unlink_if_unix_domain_socket(&sp->addr.un);
-      }
+      grpc_unlink_if_unix_domain_socket(&sp->addr.sockaddr);
       sp->destroyed_closure.cb = destroyed_port;
       sp->destroyed_closure.cb_arg = s;
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
@@ -218,8 +208,6 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
                           size_t addr_len) {
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
-  int get_local_ip;
-  int rc;
 
   if (fd < 0) {
     goto error;
@@ -230,14 +218,9 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
             strerror(errno));
   }
 
-  get_local_ip = 1;
-  rc = setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &get_local_ip,
-                  sizeof(get_local_ip));
-  if (rc == 0 && addr->sa_family == AF_INET6) {
-#if !defined(__APPLE__)
-    rc = setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &get_local_ip,
-                    sizeof(get_local_ip));
-#endif
+  if (grpc_set_socket_ip_pktinfo_if_possible(fd) &&
+      addr->sa_family == AF_INET6) {
+    grpc_set_socket_ipv6_recvpktinfo_if_possible(fd);
   }
 
   GPR_ASSERT(addr_len < ~(socklen_t)0);
@@ -336,9 +319,7 @@ int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr,
   socklen_t sockname_len;
   int port;
 
-  if (((struct sockaddr *)addr)->sa_family == AF_UNIX) {
-    unlink_if_unix_domain_socket(addr);
-  }
+  grpc_unlink_if_unix_domain_socket((struct sockaddr *)addr);
 
   /* Check if this is a wildcard port, and if so, try to keep the port the same
      as some previously created listener. */
diff --git a/src/core/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h
index a9d0489edf..316845ad66 100644
--- a/src/core/iomgr/udp_server.h
+++ b/src/core/lib/iomgr/udp_server.h
@@ -31,21 +31,22 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_UDP_SERVER_H
-#define GRPC_INTERNAL_CORE_IOMGR_UDP_SERVER_H
+#ifndef GRPC_CORE_LIB_IOMGR_UDP_SERVER_H
+#define GRPC_CORE_LIB_IOMGR_UDP_SERVER_H
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/fd_posix.h"
 
-/* Forward decl of grpc_server */
-typedef struct grpc_server grpc_server;
+/* Forward decl of struct grpc_server */
+/* This is not typedef'ed to avoid a typedef-redefinition error */
+struct grpc_server;
 
 /* Forward decl of grpc_udp_server */
 typedef struct grpc_udp_server grpc_udp_server;
 
 /* Called when data is available to read from the socket. */
 typedef void (*grpc_udp_server_read_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd,
-                                        grpc_server *server);
+                                        struct grpc_server *server);
 
 /* Create a server, initially not bound to any ports */
 grpc_udp_server *grpc_udp_server_create(void);
@@ -53,7 +54,7 @@ grpc_udp_server *grpc_udp_server_create(void);
 /* Start listening to bound ports */
 void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *udp_server,
                            grpc_pollset **pollsets, size_t pollset_count,
-                           grpc_server *server);
+                           struct grpc_server *server);
 
 int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index);
 
@@ -73,4 +74,4 @@ int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr,
 void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *server,
                              grpc_closure *on_done);
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_UDP_SERVER_H */
+#endif /* GRPC_CORE_LIB_IOMGR_UDP_SERVER_H */
diff --git a/src/core/lib/iomgr/unix_sockets_posix.c b/src/core/lib/iomgr/unix_sockets_posix.c
new file mode 100644
index 0000000000..42e44989e0
--- /dev/null
+++ b/src/core/lib/iomgr/unix_sockets_posix.c
@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
+
+#ifdef GPR_HAVE_UNIX_SOCKET
+
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+
+#include <grpc/support/alloc.h>
+
+void grpc_create_socketpair_if_unix(int sv[2]) {
+  GPR_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
+}
+
+grpc_resolved_addresses *grpc_resolve_unix_domain_address(const char *name) {
+  struct sockaddr_un *un;
+
+  grpc_resolved_addresses *addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
+  addrs->naddrs = 1;
+  addrs->addrs = gpr_malloc(sizeof(grpc_resolved_address));
+  un = (struct sockaddr_un *)addrs->addrs->addr;
+  un->sun_family = AF_UNIX;
+  strcpy(un->sun_path, name);
+  addrs->addrs->len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
+  return addrs;
+}
+
+int grpc_is_unix_socket(const struct sockaddr *addr) {
+  return addr->sa_family == AF_UNIX;
+}
+
+void grpc_unlink_if_unix_domain_socket(const struct sockaddr *addr) {
+  if (addr->sa_family != AF_UNIX) {
+    return;
+  }
+  struct sockaddr_un *un = (struct sockaddr_un *)addr;
+  struct stat st;
+
+  if (stat(un->sun_path, &st) == 0 && (st.st_mode & S_IFMT) == S_IFSOCK) {
+    unlink(un->sun_path);
+  }
+}
+
+int grpc_parse_unix(grpc_uri *uri, struct sockaddr_storage *addr, size_t *len) {
+  struct sockaddr_un *un = (struct sockaddr_un *)addr;
+
+  un->sun_family = AF_UNIX;
+  strcpy(un->sun_path, uri->path);
+  *len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
+
+  return 1;
+}
+
+char *grpc_unix_get_default_authority(grpc_resolver_factory *factory,
+                                      grpc_uri *uri) {
+  return gpr_strdup("localhost");
+}
+
+char *grpc_sockaddr_to_uri_unix_if_possible(const struct sockaddr *addr) {
+  if (addr->sa_family != AF_UNIX) {
+    return NULL;
+  }
+
+  char *result;
+  gpr_asprintf(&result, "unix:%s", ((struct sockaddr_un *)addr)->sun_path);
+  return result;
+}
+
+#endif
diff --git a/src/core/lib/iomgr/unix_sockets_posix.h b/src/core/lib/iomgr/unix_sockets_posix.h
new file mode 100644
index 0000000000..752cab85a5
--- /dev/null
+++ b/src/core/lib/iomgr/unix_sockets_posix.h
@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_UNIX_SOCKETS_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_UNIX_SOCKETS_POSIX_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/support/string_util.h>
+
+#include "src/core/lib/client_config/resolver_factory.h"
+#include "src/core/lib/client_config/uri_parser.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/sockaddr.h"
+
+void grpc_create_socketpair_if_unix(int sv[2]);
+
+grpc_resolved_addresses *grpc_resolve_unix_domain_address(const char *name);
+
+int grpc_is_unix_socket(const struct sockaddr *addr);
+
+void grpc_unlink_if_unix_domain_socket(const struct sockaddr *addr);
+
+int grpc_parse_unix(grpc_uri *uri, struct sockaddr_storage *addr, size_t *len);
+
+char *grpc_unix_get_default_authority(grpc_resolver_factory *factory,
+                                      grpc_uri *uri);
+
+char *grpc_sockaddr_to_uri_unix_if_possible(const struct sockaddr *addr);
+
+#endif /* GRPC_CORE_LIB_IOMGR_UNIX_SOCKETS_POSIX_H */
diff --git a/src/core/channel/connected_channel.h b/src/core/lib/iomgr/unix_sockets_posix_noop.c
index 95c1834bfa..06f6ee05e7 100644
--- a/src/core/channel/connected_channel.h
+++ b/src/core/lib/iomgr/unix_sockets_posix_noop.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,21 +31,31 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_CONNECTED_CHANNEL_H
-#define GRPC_INTERNAL_CORE_CHANNEL_CONNECTED_CHANNEL_H
+#include "src/core/lib/iomgr/unix_sockets_posix.h"
 
-#include "src/core/channel/channel_stack.h"
+#ifndef GPR_HAVE_UNIX_SOCKET
 
-/* A channel filter representing a channel that is on a connected transport.
-   This filter performs actual sending and receiving of messages. */
+void grpc_create_socketpair_if_unix(int sv[2]) {}
 
-extern const grpc_channel_filter grpc_connected_channel_filter;
+grpc_resolved_addresses *grpc_resolve_unix_domain_address(const char *name) {
+  return NULL;
+}
 
-/* Post construction fixup: set the transport in the connected channel.
-   Must be called before any call stack using this filter is used. */
-void grpc_connected_channel_bind_transport(grpc_channel_stack* channel_stack,
-                                           grpc_transport* transport);
+int grpc_is_unix_socket(const struct sockaddr *addr) { return false; }
 
-grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem);
+void grpc_unlink_if_unix_domain_socket(const struct sockaddr *addr) {}
 
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_CONNECTED_CHANNEL_H */
+int grpc_parse_unix(grpc_uri *uri, struct sockaddr_storage *addr, size_t *len) {
+  return 0;
+}
+
+char *grpc_unix_get_default_authority(grpc_resolver_factory *factory,
+                                      grpc_uri *uri) {
+  return NULL;
+}
+
+char *grpc_sockaddr_to_uri_unix_if_possible(const struct sockaddr *addr) {
+  return NULL;
+}
+
+#endif
diff --git a/src/core/iomgr/wakeup_fd_eventfd.c b/src/core/lib/iomgr/wakeup_fd_eventfd.c
index f67379e4fc..41ded0ca4d 100644
--- a/src/core/iomgr/wakeup_fd_eventfd.c
+++ b/src/core/lib/iomgr/wakeup_fd_eventfd.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,8 +41,8 @@
 
 #include <grpc/support/log.h>
 
-#include "src/core/iomgr/wakeup_fd_posix.h"
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/profiling/timers.h"
 
 static void eventfd_create(grpc_wakeup_fd* fd_info) {
   int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
diff --git a/src/core/iomgr/wakeup_fd_nospecial.c b/src/core/lib/iomgr/wakeup_fd_nospecial.c
index 78d763c103..39defa65c6 100644
--- a/src/core/iomgr/wakeup_fd_nospecial.c
+++ b/src/core/lib/iomgr/wakeup_fd_nospecial.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -40,8 +40,8 @@
 
 #ifdef GPR_POSIX_NO_SPECIAL_WAKEUP_FD
 
-#include "src/core/iomgr/wakeup_fd_posix.h"
 #include <stddef.h>
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 static int check_availability_invalid(void) { return 0; }
 
diff --git a/src/core/iomgr/wakeup_fd_pipe.c b/src/core/lib/iomgr/wakeup_fd_pipe.c
index 80de181d9d..820919e4dd 100644
--- a/src/core/iomgr/wakeup_fd_pipe.c
+++ b/src/core/lib/iomgr/wakeup_fd_pipe.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,19 +35,24 @@
 
 #ifdef GPR_POSIX_WAKEUP_FD
 
-#include "src/core/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
 
-#include "src/core/iomgr/socket_utils_posix.h"
 #include <grpc/support/log.h>
 
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+
 static void pipe_init(grpc_wakeup_fd* fd_info) {
   int pipefd[2];
   /* TODO(klempner): Make this nonfatal */
-  GPR_ASSERT(0 == pipe(pipefd));
+  int r = pipe(pipefd);
+  if (0 != r) {
+    gpr_log(GPR_ERROR, "pipe creation failed (%d): %s", errno, strerror(errno));
+    abort();
+  }
   GPR_ASSERT(grpc_set_socket_nonblocking(pipefd[0], 1));
   GPR_ASSERT(grpc_set_socket_nonblocking(pipefd[1], 1));
   fd_info->read_fd = pipefd[0];
diff --git a/src/core/iomgr/wakeup_fd_pipe.h b/src/core/lib/iomgr/wakeup_fd_pipe.h
index 01a13a97c0..bbdb1fc448 100644
--- a/src/core/iomgr/wakeup_fd_pipe.h
+++ b/src/core/lib/iomgr/wakeup_fd_pipe.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_WAKEUP_FD_PIPE_H
-#define GRPC_INTERNAL_CORE_IOMGR_WAKEUP_FD_PIPE_H
+#ifndef GRPC_CORE_LIB_IOMGR_WAKEUP_FD_PIPE_H
+#define GRPC_CORE_LIB_IOMGR_WAKEUP_FD_PIPE_H
 
-#include "src/core/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 extern grpc_wakeup_fd_vtable grpc_pipe_wakeup_fd_vtable;
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_WAKEUP_FD_PIPE_H */
+#endif /* GRPC_CORE_LIB_IOMGR_WAKEUP_FD_PIPE_H */
diff --git a/src/core/iomgr/wakeup_fd_posix.c b/src/core/lib/iomgr/wakeup_fd_posix.c
index f40be081b0..c4d174fb34 100644
--- a/src/core/iomgr/wakeup_fd_posix.c
+++ b/src/core/lib/iomgr/wakeup_fd_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,9 +35,9 @@
 
 #ifdef GPR_POSIX_WAKEUP_FD
 
-#include "src/core/iomgr/wakeup_fd_posix.h"
-#include "src/core/iomgr/wakeup_fd_pipe.h"
 #include <stddef.h>
+#include "src/core/lib/iomgr/wakeup_fd_pipe.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 static const grpc_wakeup_fd_vtable *wakeup_fd_vtable = NULL;
 int grpc_allow_specialized_wakeup_fd = 1;
diff --git a/src/core/iomgr/wakeup_fd_posix.h b/src/core/lib/iomgr/wakeup_fd_posix.h
index ffd60d1d4e..20988d5fd3 100644
--- a/src/core/iomgr/wakeup_fd_posix.h
+++ b/src/core/lib/iomgr/wakeup_fd_posix.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -59,8 +59,8 @@
  * 2. If the polling thread was awakened by a wakeup_fd event, call
  *    grpc_wakeup_fd_consume_wakeup() on it.
  */
-#ifndef GRPC_INTERNAL_CORE_IOMGR_WAKEUP_FD_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_WAKEUP_FD_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_WAKEUP_FD_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_WAKEUP_FD_POSIX_H
 
 void grpc_wakeup_fd_global_init(void);
 void grpc_wakeup_fd_global_destroy(void);
@@ -98,4 +98,4 @@ void grpc_wakeup_fd_destroy(grpc_wakeup_fd* fd_info);
  * wakeup_fd_nospecial.c if no such implementation exists. */
 extern const grpc_wakeup_fd_vtable grpc_specialized_wakeup_fd_vtable;
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_WAKEUP_FD_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_WAKEUP_FD_POSIX_H */
diff --git a/src/core/iomgr/workqueue.h b/src/core/lib/iomgr/workqueue.h
index 36dd133468..9c420c57de 100644
--- a/src/core/iomgr/workqueue.h
+++ b/src/core/lib/iomgr/workqueue.h
@@ -31,20 +31,20 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_H
-#define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_H
+#ifndef GRPC_CORE_LIB_IOMGR_WORKQUEUE_H
+#define GRPC_CORE_LIB_IOMGR_WORKQUEUE_H
 
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/closure.h"
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/pollset.h"
 
 #ifdef GPR_POSIX_SOCKET
-#include "src/core/iomgr/workqueue_posix.h"
+#include "src/core/lib/iomgr/workqueue_posix.h"
 #endif
 
 #ifdef GPR_WIN32
-#include "src/core/iomgr/workqueue_windows.h"
+#include "src/core/lib/iomgr/workqueue_windows.h"
 #endif
 
 /* grpc_workqueue is forward declared in exec_ctx.h */
@@ -80,4 +80,4 @@ void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx,
 void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure,
                          int success);
 
-#endif
+#endif /* GRPC_CORE_LIB_IOMGR_WORKQUEUE_H */
diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/lib/iomgr/workqueue_posix.c
index c096dbfb30..76830ef12d 100644
--- a/src/core/iomgr/workqueue_posix.c
+++ b/src/core/lib/iomgr/workqueue_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_SOCKET
 
-#include "src/core/iomgr/workqueue.h"
+#include "src/core/lib/iomgr/workqueue.h"
 
 #include <stdio.h>
 
@@ -43,8 +43,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/iomgr/fd_posix.h"
-#include "src/core/iomgr/pollset_posix.h"
+#include "src/core/lib/iomgr/fd_posix.h"
+#include "src/core/lib/iomgr/pollset_posix.h"
 
 static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success);
 
@@ -107,7 +107,7 @@ void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
   if (grpc_closure_list_empty(workqueue->closure_list)) {
     grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
   }
-  grpc_closure_list_move(&exec_ctx->closure_list, &workqueue->closure_list);
+  grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
   gpr_mu_unlock(&workqueue->mu);
 }
 
@@ -123,7 +123,7 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     gpr_free(workqueue);
   } else {
     gpr_mu_lock(&workqueue->mu);
-    grpc_closure_list_move(&workqueue->closure_list, &exec_ctx->closure_list);
+    grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
     grpc_wakeup_fd_consume_wakeup(&workqueue->wakeup_fd);
     gpr_mu_unlock(&workqueue->mu);
     grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd,
diff --git a/src/core/iomgr/workqueue_posix.h b/src/core/lib/iomgr/workqueue_posix.h
index 68f195ee0d..956de8fb27 100644
--- a/src/core/iomgr/workqueue_posix.h
+++ b/src/core/lib/iomgr/workqueue_posix.h
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H
-#define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H
+#ifndef GRPC_CORE_LIB_IOMGR_WORKQUEUE_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_WORKQUEUE_POSIX_H
 
-#include "src/core/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 struct grpc_fd;
 
@@ -50,4 +50,4 @@ struct grpc_workqueue {
   grpc_closure read_closure;
 };
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H */
+#endif /* GRPC_CORE_LIB_IOMGR_WORKQUEUE_POSIX_H */
diff --git a/src/core/iomgr/workqueue_windows.c b/src/core/lib/iomgr/workqueue_windows.c
index f9ca57557b..6697f93498 100644
--- a/src/core/iomgr/workqueue_windows.c
+++ b/src/core/lib/iomgr/workqueue_windows.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,6 +35,6 @@
 
 #ifdef GPR_WIN32
 
-#include "src/core/iomgr/workqueue.h"
+#include "src/core/lib/iomgr/workqueue.h"
 
 #endif /* GPR_WIN32 */
diff --git a/src/core/iomgr/workqueue_windows.h b/src/core/lib/iomgr/workqueue_windows.h
index 941f195f51..8e6980b6d9 100644
--- a/src/core/iomgr/workqueue_windows.h
+++ b/src/core/lib/iomgr/workqueue_windows.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_WINDOWS_H
-#define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_WINDOWS_H
+#ifndef GRPC_CORE_LIB_IOMGR_WORKQUEUE_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_WORKQUEUE_WINDOWS_H
 
-#endif /* GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_IOMGR_WORKQUEUE_WINDOWS_H */
diff --git a/src/core/json/json.c b/src/core/lib/json/json.c
index 96e11eebb1..9793045d91 100644
--- a/src/core/json/json.c
+++ b/src/core/lib/json/json.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,7 +35,7 @@
 
 #include <grpc/support/alloc.h>
 
-#include "src/core/json/json.h"
+#include "src/core/lib/json/json.h"
 
 grpc_json *grpc_json_create(grpc_json_type type) {
   grpc_json *json = gpr_malloc(sizeof(*json));
diff --git a/src/core/json/json.h b/src/core/lib/json/json.h
index c4df2998c3..41d87dd5ce 100644
--- a/src/core/json/json.h
+++ b/src/core/lib/json/json.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_JSON_JSON_H
-#define GRPC_INTERNAL_CORE_JSON_JSON_H
+#ifndef GRPC_CORE_LIB_JSON_JSON_H
+#define GRPC_CORE_LIB_JSON_JSON_H
 
 #include <stdlib.h>
 
-#include "src/core/json/json_common.h"
+#include "src/core/lib/json/json_common.h"
 
 /* A tree-like structure to hold json values. The key and value pointers
  * are not owned by it.
@@ -85,4 +85,4 @@ char *grpc_json_dump_to_string(grpc_json *json, int indent);
 grpc_json *grpc_json_create(grpc_json_type type);
 void grpc_json_destroy(grpc_json *json);
 
-#endif /* GRPC_INTERNAL_CORE_JSON_JSON_H */
+#endif /* GRPC_CORE_LIB_JSON_JSON_H */
diff --git a/src/core/json/json_common.h b/src/core/lib/json/json_common.h
index 481695b38b..ce980040f8 100644
--- a/src/core/json/json_common.h
+++ b/src/core/lib/json/json_common.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_JSON_JSON_COMMON_H
-#define GRPC_INTERNAL_CORE_JSON_JSON_COMMON_H
+#ifndef GRPC_CORE_LIB_JSON_JSON_COMMON_H
+#define GRPC_CORE_LIB_JSON_JSON_COMMON_H
 
 /* The various json types. */
 typedef enum {
@@ -46,4 +46,4 @@ typedef enum {
   GRPC_JSON_TOP_LEVEL
 } grpc_json_type;
 
-#endif /* GRPC_INTERNAL_CORE_JSON_JSON_COMMON_H */
+#endif /* GRPC_CORE_LIB_JSON_JSON_COMMON_H */
diff --git a/src/core/json/json_reader.c b/src/core/lib/json/json_reader.c
index 30da6f28f3..4cff13dff1 100644
--- a/src/core/json/json_reader.c
+++ b/src/core/lib/json/json_reader.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,7 +37,7 @@
 
 #include <grpc/support/log.h>
 
-#include "src/core/json/json_reader.h"
+#include "src/core/lib/json/json_reader.h"
 
 static void json_reader_string_clear(grpc_json_reader *reader) {
   reader->vtable->string_clear(reader->userdata);
@@ -280,13 +280,14 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
             break;
 
           case GRPC_JSON_STATE_OBJECT_KEY_STRING:
-            GPR_ASSERT(reader->unicode_high_surrogate == 0);
+            if (reader->unicode_high_surrogate != 0)
+              return GRPC_JSON_PARSE_ERROR;
             if (c == '"') {
               reader->state = GRPC_JSON_STATE_OBJECT_KEY_END;
               json_reader_set_key(reader);
               json_reader_string_clear(reader);
             } else {
-              if (c <= 0x001f) return GRPC_JSON_PARSE_ERROR;
+              if (c < 32) return GRPC_JSON_PARSE_ERROR;
               json_reader_string_add_char(reader, c);
             }
             break;
@@ -362,6 +363,8 @@ grpc_json_reader_status grpc_json_reader_run(grpc_json_reader *reader) {
                 reader->in_object = 0;
                 reader->in_array = 1;
                 break;
+              default:
+                return GRPC_JSON_PARSE_ERROR;
             }
             break;
 
diff --git a/src/core/json/json_reader.h b/src/core/lib/json/json_reader.h
index 90b9f1f9fe..37a838889d 100644
--- a/src/core/json/json_reader.h
+++ b/src/core/lib/json/json_reader.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_JSON_JSON_READER_H
-#define GRPC_INTERNAL_CORE_JSON_JSON_READER_H
+#ifndef GRPC_CORE_LIB_JSON_JSON_READER_H
+#define GRPC_CORE_LIB_JSON_JSON_READER_H
 
 #include <grpc/support/port_platform.h>
-#include "src/core/json/json_common.h"
+#include "src/core/lib/json/json_common.h"
 
 typedef enum {
   GRPC_JSON_STATE_OBJECT_KEY_BEGIN,
@@ -157,4 +157,4 @@ void grpc_json_reader_init(grpc_json_reader *reader,
  */
 int grpc_json_reader_is_complete(grpc_json_reader *reader);
 
-#endif /* GRPC_INTERNAL_CORE_JSON_JSON_READER_H */
+#endif /* GRPC_CORE_LIB_JSON_JSON_READER_H */
diff --git a/src/core/json/json_string.c b/src/core/lib/json/json_string.c
index 2bc0b513d5..8e6f1253dc 100644
--- a/src/core/json/json_string.c
+++ b/src/core/lib/json/json_string.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,15 +31,15 @@
  *
  */
 
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/json/json.h"
-#include "src/core/json/json_reader.h"
-#include "src/core/json/json_writer.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/json/json_reader.h"
+#include "src/core/lib/json/json_writer.h"
 
 /* The json reader will construct a bunch of grpc_json objects and
  * link them all up together in a tree-like structure that will represent
diff --git a/src/core/json/json_writer.c b/src/core/lib/json/json_writer.c
index 326ec2d431..d614a72fc4 100644
--- a/src/core/json/json_writer.c
+++ b/src/core/lib/json/json_writer.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/json/json_writer.h"
+#include "src/core/lib/json/json_writer.h"
 
 static void json_writer_output_char(grpc_json_writer *writer, char c) {
   writer->vtable->output_char(writer->userdata, c);
diff --git a/src/core/json/json_writer.h b/src/core/lib/json/json_writer.h
index 9ef04aab01..f90e79cd74 100644
--- a/src/core/json/json_writer.h
+++ b/src/core/lib/json/json_writer.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -43,12 +43,12 @@
  * a valid UTF-8 string overall.
  */
 
-#ifndef GRPC_INTERNAL_CORE_JSON_JSON_WRITER_H
-#define GRPC_INTERNAL_CORE_JSON_JSON_WRITER_H
+#ifndef GRPC_CORE_LIB_JSON_JSON_WRITER_H
+#define GRPC_CORE_LIB_JSON_JSON_WRITER_H
 
 #include <stdlib.h>
 
-#include "src/core/json/json_common.h"
+#include "src/core/lib/json/json_common.h"
 
 typedef struct grpc_json_writer_vtable {
   /* Adds a character to the output stream. */
@@ -94,4 +94,4 @@ void grpc_json_writer_value_raw_with_len(grpc_json_writer *writer,
 void grpc_json_writer_value_string(grpc_json_writer *writer,
                                    const char *string);
 
-#endif /* GRPC_INTERNAL_CORE_JSON_JSON_WRITER_H */
+#endif /* GRPC_CORE_LIB_JSON_JSON_WRITER_H */
diff --git a/src/core/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c
index df32472d1c..15a9584981 100644
--- a/src/core/profiling/basic_timers.c
+++ b/src/core/lib/profiling/basic_timers.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,13 +35,13 @@
 
 #ifdef GRPC_BASIC_PROFILER
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/time.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/thd.h>
+#include <grpc/support/time.h>
 #include <stdio.h>
 
 typedef enum { BEGIN = '{', END = '}', MARK = '.' } marker_type;
diff --git a/src/core/profiling/stap_probes.d b/src/core/lib/profiling/stap_probes.d
index 153de91752..153de91752 100644
--- a/src/core/profiling/stap_probes.d
+++ b/src/core/lib/profiling/stap_probes.d
diff --git a/src/core/profiling/stap_timers.c b/src/core/lib/profiling/stap_timers.c
index efcd1af4a1..f55c1a569a 100644
--- a/src/core/profiling/stap_timers.c
+++ b/src/core/lib/profiling/stap_timers.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,11 +35,11 @@
 
 #ifdef GRPC_STAP_PROFILER
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
 #include <sys/sdt.h>
 /* Generated from src/core/profiling/stap_probes.d */
-#include "src/core/profiling/stap_probes.h"
+#include "src/core/lib/profiling/stap_probes.h"
 
 /* Latency profiler API implementation. */
 void gpr_timer_add_mark(int tag, const char *tagstr, void *id, const char *file,
diff --git a/src/core/profiling/timers.h b/src/core/lib/profiling/timers.h
index 6a188dc566..1303593ffb 100644
--- a/src/core/profiling/timers.h
+++ b/src/core/lib/profiling/timers.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_PROFILING_TIMERS_H
-#define GRPC_CORE_PROFILING_TIMERS_H
+#ifndef GRPC_CORE_LIB_PROFILING_TIMERS_H
+#define GRPC_CORE_LIB_PROFILING_TIMERS_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -116,4 +116,4 @@ class ProfileScope {
 #endif
 #endif
 
-#endif /* GRPC_CORE_PROFILING_TIMERS_H */
+#endif /* GRPC_CORE_LIB_PROFILING_TIMERS_H */
diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/lib/proto/grpc/lb/v0/load_balancer.pb.c
index 59aae30cff..8f82141f96 100644
--- a/src/core/proto/grpc/lb/v0/load_balancer.pb.c
+++ b/src/core/lib/proto/grpc/lb/v0/load_balancer.pb.c
@@ -33,7 +33,7 @@
 /* Automatically generated nanopb constant definitions */
 /* Generated by nanopb-0.3.5-dev */
 
-#include "src/core/proto/grpc/lb/v0/load_balancer.pb.h"
+#include "src/core/lib/proto/grpc/lb/v0/load_balancer.pb.h"
 
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.
diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/lib/proto/grpc/lb/v0/load_balancer.pb.h
index 3599f881bb..3599f881bb 100644
--- a/src/core/proto/grpc/lb/v0/load_balancer.pb.h
+++ b/src/core/lib/proto/grpc/lb/v0/load_balancer.pb.h
diff --git a/src/core/security/auth_filters.h b/src/core/lib/security/auth_filters.h
index c179b54bec..162b60e2c8 100644
--- a/src/core/security/auth_filters.h
+++ b/src/core/lib/security/auth_filters.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_AUTH_FILTERS_H
-#define GRPC_INTERNAL_CORE_SECURITY_AUTH_FILTERS_H
+#ifndef GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
+#define GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 
 extern const grpc_channel_filter grpc_client_auth_filter;
 extern const grpc_channel_filter grpc_server_auth_filter;
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_AUTH_FILTERS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H */
diff --git a/src/core/security/b64.c b/src/core/lib/security/b64.c
index c40b528e2f..1d3879534c 100644
--- a/src/core/security/b64.c
+++ b/src/core/lib/security/b64.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/b64.h"
+#include "src/core/lib/security/b64.h"
 
 #include <stdint.h>
 #include <string.h>
diff --git a/src/core/security/b64.h b/src/core/lib/security/b64.h
index 3e3b521120..0bf372a1e7 100644
--- a/src/core/security/b64.h
+++ b/src/core/lib/security/b64.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_BASE64_H
-#define GRPC_INTERNAL_CORE_SECURITY_BASE64_H
+#ifndef GRPC_CORE_LIB_SECURITY_B64_H
+#define GRPC_CORE_LIB_SECURITY_B64_H
 
 #include <grpc/support/slice.h>
 
@@ -49,4 +49,4 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe);
 gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
                                       int url_safe);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_BASE64_H */
+#endif /* GRPC_CORE_LIB_SECURITY_B64_H */
diff --git a/src/core/security/client_auth_filter.c b/src/core/lib/security/client_auth_filter.c
index 332d4259d2..af6073e560 100644
--- a/src/core/security/client_auth_filter.c
+++ b/src/core/lib/security/client_auth_filter.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/auth_filters.h"
+#include "src/core/lib/security/auth_filters.h"
 
 #include <string.h>
 
@@ -39,13 +39,13 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/security/credentials.h"
-#include "src/core/security/security_connector.h"
-#include "src/core/security/security_context.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/call.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 #define MAX_CREDENTIALS_METADATA_COUNT 4
 
@@ -172,7 +172,7 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
     calld->creds = grpc_composite_call_credentials_create(channel_call_creds,
                                                           ctx->creds, NULL);
     if (calld->creds == NULL) {
-      bubble_up_error(exec_ctx, elem, GRPC_STATUS_INVALID_ARGUMENT,
+      bubble_up_error(exec_ctx, elem, GRPC_STATUS_INTERNAL,
                       "Incompatible credentials set on channel and call.");
       return;
     }
@@ -201,7 +201,7 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
     char *error_msg;
     gpr_asprintf(&error_msg, "Invalid host %s set in :authority metadata.",
                  grpc_mdstr_as_c_string(calld->host));
-    bubble_up_error(exec_ctx, elem, GRPC_STATUS_INVALID_ARGUMENT, error_msg);
+    bubble_up_error(exec_ctx, elem, GRPC_STATUS_INTERNAL, error_msg);
     gpr_free(error_msg);
   }
 }
@@ -331,6 +331,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 
 const grpc_channel_filter grpc_client_auth_filter = {
     auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem, set_pollset, destroy_call_elem, sizeof(channel_data),
-    init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer,
-    "client-auth"};
+    init_call_elem,          set_pollset,          destroy_call_elem,
+    sizeof(channel_data),    init_channel_elem,    destroy_channel_elem,
+    grpc_call_next_get_peer, "client-auth"};
diff --git a/src/core/security/credentials.c b/src/core/lib/security/credentials.c
index b4fa616fa7..99a07e5c13 100644
--- a/src/core/security/credentials.c
+++ b/src/core/lib/security/credentials.c
@@ -31,18 +31,19 @@
  *
  */
 
-#include "src/core/security/credentials.h"
+#include "src/core/lib/security/credentials.h"
 
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/http_client_filter.h"
-#include "src/core/httpcli/httpcli.h"
-#include "src/core/iomgr/executor.h"
-#include "src/core/json/json.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -539,7 +540,7 @@ static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
 
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const grpc_httpcli_response *response, grpc_credentials_md_store **token_md,
+    const grpc_http_response *response, grpc_credentials_md_store **token_md,
     gpr_timespec *token_lifetime) {
   char *null_terminated_body = NULL;
   char *new_access_token = NULL;
@@ -629,7 +630,7 @@ end:
 
 static void on_oauth2_token_fetcher_http_response(
     grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_httpcli_response *response) {
+    const grpc_http_response *response) {
   grpc_credentials_metadata_request *r =
       (grpc_credentials_metadata_request *)user_data;
   grpc_oauth2_token_fetcher_credentials *c =
@@ -706,13 +707,13 @@ static void compute_engine_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
     grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
     grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_httpcli_header header = {"Metadata-Flavor", "Google"};
+  grpc_http_header header = {"Metadata-Flavor", "Google"};
   grpc_httpcli_request request;
   memset(&request, 0, sizeof(grpc_httpcli_request));
   request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
-  request.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
-  request.hdr_count = 1;
-  request.hdrs = &header;
+  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
   grpc_httpcli_get(exec_ctx, httpcli_context, pollset, &request, deadline,
                    response_cb, metadata_req);
 }
@@ -747,8 +748,8 @@ static void refresh_token_fetch_oauth2(
     grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
   grpc_google_refresh_token_credentials *c =
       (grpc_google_refresh_token_credentials *)metadata_req->creds;
-  grpc_httpcli_header header = {"Content-Type",
-                                "application/x-www-form-urlencoded"};
+  grpc_http_header header = {"Content-Type",
+                             "application/x-www-form-urlencoded"};
   grpc_httpcli_request request;
   char *body = NULL;
   gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
@@ -756,9 +757,9 @@ static void refresh_token_fetch_oauth2(
                c->refresh_token.refresh_token);
   memset(&request, 0, sizeof(grpc_httpcli_request));
   request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
-  request.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
-  request.hdr_count = 1;
-  request.hdrs = &header;
+  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
   request.handshaker = &grpc_httpcli_ssl;
   grpc_httpcli_post(exec_ctx, httpcli_context, pollset, &request, body,
                     strlen(body), deadline, response_cb, metadata_req);
diff --git a/src/core/security/credentials.h b/src/core/lib/security/credentials.h
index 0de4cd9468..7168b98942 100644
--- a/src/core/security/credentials.h
+++ b/src/core/lib/security/credentials.h
@@ -31,19 +31,20 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_CREDENTIALS_H
-#define GRPC_INTERNAL_CORE_SECURITY_CREDENTIALS_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
 
-#include "src/core/transport/metadata_batch.h"
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
 #include <grpc/support/sync.h>
+#include "src/core/lib/transport/metadata_batch.h"
 
-#include "src/core/httpcli/httpcli.h"
-#include "src/core/security/json_token.h"
-#include "src/core/security/security_connector.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/security_connector.h"
 
-struct grpc_httpcli_response;
+struct grpc_http_response;
 
 /* --- Constants. --- */
 
@@ -207,7 +208,7 @@ grpc_call_credentials *grpc_credentials_contains_type(
 /* Exposed for testing only. */
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const struct grpc_httpcli_response *response,
+    const struct grpc_http_response *response,
     grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
 
 void grpc_flush_cached_google_default_credentials(void);
@@ -373,4 +374,4 @@ typedef struct {
   grpc_credentials_md_store *plugin_md;
 } grpc_plugin_credentials;
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_CREDENTIALS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_H */
diff --git a/src/core/security/credentials_metadata.c b/src/core/lib/security/credentials_metadata.c
index b8a132f1ea..c3bfcb11b5 100644
--- a/src/core/security/credentials_metadata.c
+++ b/src/core/lib/security/credentials_metadata.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/credentials.h"
+#include "src/core/lib/security/credentials.h"
 
 #include <grpc/support/alloc.h>
 
diff --git a/src/core/security/credentials_posix.c b/src/core/lib/security/credentials_posix.c
index 0c92bd4a96..b758cd0a1a 100644
--- a/src/core/security/credentials_posix.c
+++ b/src/core/lib/security/credentials_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,14 +35,14 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/security/credentials.h"
+#include "src/core/lib/security/credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/env.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/string.h"
 
 char *grpc_get_well_known_google_credentials_file_path_impl(void) {
   char *result = NULL;
diff --git a/src/core/security/credentials_win32.c b/src/core/lib/security/credentials_win32.c
index 8ee9f706a1..a225ab0d7d 100644
--- a/src/core/security/credentials_win32.c
+++ b/src/core/lib/security/credentials_win32.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,14 +35,14 @@
 
 #ifdef GPR_WIN32
 
-#include "src/core/security/credentials.h"
+#include "src/core/lib/security/credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/env.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/string.h"
 
 char *grpc_get_well_known_google_credentials_file_path_impl(void) {
   char *result = NULL;
diff --git a/src/core/security/google_default_credentials.c b/src/core/lib/security/google_default_credentials.c
index 1f4f3e4aa5..5c342288cc 100644
--- a/src/core/security/google_default_credentials.c
+++ b/src/core/lib/security/google_default_credentials.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/credentials.h"
+#include "src/core/lib/security/credentials.h"
 
 #include <string.h>
 
@@ -39,10 +39,11 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/httpcli/httpcli.h"
-#include "src/core/support/env.h"
-#include "src/core/support/load_file.h"
-#include "src/core/surface/api_trace.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/load_file.h"
+#include "src/core/lib/surface/api_trace.h"
 
 /* -- Constants. -- */
 
@@ -66,14 +67,14 @@ typedef struct {
 
 static void on_compute_engine_detection_http_response(
     grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_httpcli_response *response) {
+    const grpc_http_response *response) {
   compute_engine_detector *detector = (compute_engine_detector *)user_data;
   if (response != NULL && response->status == 200 && response->hdr_count > 0) {
     /* Internet providers can return a generic response to all requests, so
        it is necessary to check that metadata header is present also. */
     size_t i;
     for (i = 0; i < response->hdr_count; i++) {
-      grpc_httpcli_header *header = &response->hdrs[i];
+      grpc_http_header *header = &response->hdrs[i];
       if (strcmp(header->key, "Metadata-Flavor") == 0 &&
           strcmp(header->value, "Google") == 0) {
         detector->success = 1;
@@ -109,7 +110,7 @@ static int is_stack_running_on_compute_engine(void) {
 
   memset(&request, 0, sizeof(grpc_httpcli_request));
   request.host = GRPC_COMPUTE_ENGINE_DETECTION_HOST;
-  request.path = "/";
+  request.http.path = "/";
 
   grpc_httpcli_context_init(&context);
 
diff --git a/src/core/security/handshake.c b/src/core/lib/security/handshake.c
index b5bb6667a7..adb6d7fe4e 100644
--- a/src/core/security/handshake.c
+++ b/src/core/lib/security/handshake.c
@@ -31,16 +31,16 @@
  *
  */
 
-#include "src/core/security/handshake.h"
+#include "src/core/lib/security/handshake.h"
 
 #include <stdbool.h>
 #include <string.h>
 
-#include "src/core/security/security_context.h"
-#include "src/core/security/secure_endpoint.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
+#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/security_context.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
diff --git a/src/core/security/handshake.h b/src/core/lib/security/handshake.h
index db8b374921..b5d7bb3282 100644
--- a/src/core/security/handshake.h
+++ b/src/core/lib/security/handshake.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_HANDSHAKE_H
-#define GRPC_INTERNAL_CORE_SECURITY_HANDSHAKE_H
+#ifndef GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
+#define GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
 
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/security/security_connector.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/security/security_connector.h"
 
 /* Calls the callback upon completion. Takes owership of handshaker. */
 void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
@@ -48,4 +48,4 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_HANDSHAKE_H */
+#endif /* GRPC_CORE_LIB_SECURITY_HANDSHAKE_H */
diff --git a/src/core/security/json_token.c b/src/core/lib/security/json_token.c
index 372e5bfc5a..97054286d9 100644
--- a/src/core/security/json_token.c
+++ b/src/core/lib/security/json_token.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/json_token.h"
+#include "src/core/lib/security/json_token.h"
 
 #include <string.h>
 
@@ -39,8 +39,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/security/b64.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/security/b64.h"
+#include "src/core/lib/support/string.h"
 
 #include <openssl/bio.h>
 #include <openssl/evp.h>
diff --git a/src/core/security/json_token.h b/src/core/lib/security/json_token.h
index 7e06864ff3..376fb03875 100644
--- a/src/core/security/json_token.h
+++ b/src/core/lib/security/json_token.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_JSON_TOKEN_H
-#define GRPC_INTERNAL_CORE_SECURITY_JSON_TOKEN_H
+#ifndef GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
+#define GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
 
 #include <grpc/support/slice.h>
 #include <openssl/rsa.h>
 
-#include "src/core/json/json.h"
+#include "src/core/lib/json/json.h"
 
 /* --- Constants. --- */
 
@@ -115,4 +115,4 @@ grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
 /* Destructs the object. */
 void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_JSON_TOKEN_H */
+#endif /* GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H */
diff --git a/src/core/security/jwt_verifier.c b/src/core/lib/security/jwt_verifier.c
index 928c6c148d..460b92f9a0 100644
--- a/src/core/security/jwt_verifier.c
+++ b/src/core/lib/security/jwt_verifier.c
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/security/jwt_verifier.h"
+#include "src/core/lib/security/jwt_verifier.h"
 
 #include <limits.h>
 #include <string.h>
 
-#include "src/core/httpcli/httpcli.h"
-#include "src/core/security/b64.h"
-#include "src/core/tsi/ssl_types.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/security/b64.h"
+#include "src/core/lib/tsi/ssl_types.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -635,11 +635,11 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
   jwks_uri += 8;
   req.handshaker = &grpc_httpcli_ssl;
   req.host = gpr_strdup(jwks_uri);
-  req.path = strchr(jwks_uri, '/');
-  if (req.path == NULL) {
-    req.path = "";
+  req.http.path = strchr(jwks_uri, '/');
+  if (req.http.path == NULL) {
+    req.http.path = "";
   } else {
-    *(req.host + (req.path - jwks_uri)) = '\0';
+    *(req.host + (req.http.path - jwks_uri)) = '\0';
   }
   grpc_httpcli_get(
       exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req,
@@ -725,20 +725,20 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
     req.host = gpr_strdup(mapping->key_url_prefix);
     path_prefix = strchr(req.host, '/');
     if (path_prefix == NULL) {
-      gpr_asprintf(&req.path, "/%s", iss);
+      gpr_asprintf(&req.http.path, "/%s", iss);
     } else {
       *(path_prefix++) = '\0';
-      gpr_asprintf(&req.path, "/%s/%s", path_prefix, iss);
+      gpr_asprintf(&req.http.path, "/%s/%s", path_prefix, iss);
     }
     http_cb = on_keys_retrieved;
   } else {
     req.host = gpr_strdup(strstr(iss, "https://") == iss ? iss + 8 : iss);
     path_prefix = strchr(req.host, '/');
     if (path_prefix == NULL) {
-      req.path = gpr_strdup(GRPC_OPENID_CONFIG_URL_SUFFIX);
+      req.http.path = gpr_strdup(GRPC_OPENID_CONFIG_URL_SUFFIX);
     } else {
       *(path_prefix++) = 0;
-      gpr_asprintf(&req.path, "/%s%s", path_prefix,
+      gpr_asprintf(&req.http.path, "/%s%s", path_prefix,
                    GRPC_OPENID_CONFIG_URL_SUFFIX);
     }
     http_cb = on_openid_config_retrieved;
@@ -749,7 +749,7 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
       http_cb, ctx);
   gpr_free(req.host);
-  gpr_free(req.path);
+  gpr_free(req.http.path);
   return;
 
 error:
diff --git a/src/core/security/jwt_verifier.h b/src/core/lib/security/jwt_verifier.h
index 25613f03a0..28a9eff048 100644
--- a/src/core/security/jwt_verifier.h
+++ b/src/core/lib/security/jwt_verifier.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_JWT_VERIFIER_H
-#define GRPC_INTERNAL_CORE_SECURITY_JWT_VERIFIER_H
+#ifndef GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
+#define GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
 
-#include "src/core/iomgr/pollset.h"
-#include "src/core/json/json.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/json/json.h"
 
 #include <grpc/support/slice.h>
 #include <grpc/support/time.h>
@@ -133,4 +133,4 @@ grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer);
 grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
                                                const char *audience);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_JWT_VERIFIER_H */
+#endif /* GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H */
diff --git a/src/core/security/secure_endpoint.c b/src/core/lib/security/secure_endpoint.c
index d11c43be20..e233b081ef 100644
--- a/src/core/security/secure_endpoint.c
+++ b/src/core/lib/security/secure_endpoint.c
@@ -31,15 +31,15 @@
  *
  */
 
-#include "src/core/security/secure_endpoint.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/security/secure_endpoint.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/slice_buffer.h>
 #include <grpc/support/slice.h>
+#include <grpc/support/slice_buffer.h>
 #include <grpc/support/sync.h>
-#include "src/core/tsi/transport_security_interface.h"
-#include "src/core/debug/trace.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
 
 #define STAGING_BUFFER_SIZE 8192
 
@@ -354,8 +354,9 @@ static char *endpoint_get_peer(grpc_endpoint *secure_ep) {
 }
 
 static const grpc_endpoint_vtable vtable = {
-    endpoint_read, endpoint_write, endpoint_add_to_pollset,
-    endpoint_add_to_pollset_set, endpoint_shutdown, endpoint_destroy,
+    endpoint_read,           endpoint_write,
+    endpoint_add_to_pollset, endpoint_add_to_pollset_set,
+    endpoint_shutdown,       endpoint_destroy,
     endpoint_get_peer};
 
 grpc_endpoint *grpc_secure_endpoint_create(
diff --git a/src/core/security/secure_endpoint.h b/src/core/lib/security/secure_endpoint.h
index c563bdd9c5..57bd160a52 100644
--- a/src/core/security/secure_endpoint.h
+++ b/src/core/lib/security/secure_endpoint.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_SECURE_ENDPOINT_H
-#define GRPC_INTERNAL_CORE_SECURITY_SECURE_ENDPOINT_H
+#ifndef GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
+#define GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
 
-#include "src/core/iomgr/endpoint.h"
 #include <grpc/support/slice.h>
+#include "src/core/lib/iomgr/endpoint.h"
 
 struct tsi_frame_protector;
 
@@ -46,4 +46,4 @@ grpc_endpoint *grpc_secure_endpoint_create(
     struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
     gpr_slice *leftover_slices, size_t leftover_nslices);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_SECURE_ENDPOINT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H */
diff --git a/src/core/security/security_connector.c b/src/core/lib/security/security_connector.c
index 33c62a20c2..48b23a9dcf 100644
--- a/src/core/security/security_connector.c
+++ b/src/core/lib/security/security_connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/security_connector.h"
+#include "src/core/lib/security/security_connector.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -42,16 +42,16 @@
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/security/credentials.h"
-#include "src/core/security/handshake.h"
-#include "src/core/security/secure_endpoint.h"
-#include "src/core/security/security_context.h"
-#include "src/core/support/env.h"
-#include "src/core/support/load_file.h"
-#include "src/core/support/string.h"
-#include "src/core/transport/chttp2/alpn.h"
-#include "src/core/tsi/fake_transport_security.h"
-#include "src/core/tsi/ssl_transport_security.h"
+#include "src/core/ext/transport/chttp2/transport/alpn.h"
+#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/load_file.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/tsi/fake_transport_security.h"
+#include "src/core/lib/tsi/ssl_transport_security.h"
 
 /* -- Constants. -- */
 
@@ -492,6 +492,9 @@ grpc_auth_context *tsi_ssl_peer_to_auth_context(const tsi_peer *peer) {
       peer_identity_property_name = GRPC_X509_SAN_PROPERTY_NAME;
       grpc_auth_context_add_property(ctx, GRPC_X509_SAN_PROPERTY_NAME,
                                      prop->value.data, prop->value.length);
+    } else if (strcmp(prop->name, TSI_X509_PEM_CERT_PROPERTY) == 0) {
+      grpc_auth_context_add_property(ctx, GRPC_X509_PEM_CERT_PROPERTY_NAME,
+                                     prop->value.data, prop->value.length);
     }
   }
   if (peer_identity_property_name != NULL) {
@@ -554,9 +557,9 @@ static void ssl_server_check_peer(grpc_exec_ctx *exec_ctx,
   grpc_auth_context_unref(auth_context);
 }
 
-static void add_shalow_auth_property_to_peer(tsi_peer *peer,
-                                             const grpc_auth_property *prop,
-                                             const char *tsi_prop_name) {
+static void add_shallow_auth_property_to_peer(tsi_peer *peer,
+                                              const grpc_auth_property *prop,
+                                              const char *tsi_prop_name) {
   tsi_peer_property *tsi_prop = &peer->properties[peer->property_count++];
   tsi_prop->name = (char *)tsi_prop_name;
   tsi_prop->value.data = prop->value;
@@ -579,11 +582,14 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     it = grpc_auth_context_property_iterator(auth_context);
     while ((prop = grpc_auth_property_iterator_next(&it)) != NULL) {
       if (strcmp(prop->name, GRPC_X509_SAN_PROPERTY_NAME) == 0) {
-        add_shalow_auth_property_to_peer(
+        add_shallow_auth_property_to_peer(
             &peer, prop, TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY);
       } else if (strcmp(prop->name, GRPC_X509_CN_PROPERTY_NAME) == 0) {
-        add_shalow_auth_property_to_peer(
+        add_shallow_auth_property_to_peer(
             &peer, prop, TSI_X509_SUBJECT_COMMON_NAME_PEER_PROPERTY);
+      } else if (strcmp(prop->name, GRPC_X509_PEM_CERT_PROPERTY_NAME) == 0) {
+        add_shallow_auth_property_to_peer(&peer, prop,
+                                          TSI_X509_PEM_CERT_PROPERTY);
       }
     }
   }
diff --git a/src/core/security/security_connector.h b/src/core/lib/security/security_connector.h
index 1e35d3f9b7..d50091c628 100644
--- a/src/core/security/security_connector.h
+++ b/src/core/lib/security/security_connector.h
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONNECTOR_H
-#define GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONNECTOR_H
+#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
+#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
-#include "src/core/iomgr/endpoint.h"
-#include "src/core/iomgr/tcp_server.h"
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
 
 /* --- status enum. --- */
 
@@ -263,4 +263,4 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     const grpc_auth_context *auth_context);
 void tsi_shallow_peer_destruct(tsi_peer *peer);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONNECTOR_H */
+#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H */
diff --git a/src/core/security/security_context.c b/src/core/lib/security/security_context.c
index a71b3bc915..0e66373bd8 100644
--- a/src/core/security/security_context.c
+++ b/src/core/lib/security/security_context.c
@@ -33,10 +33,10 @@
 
 #include <string.h>
 
-#include "src/core/security/security_context.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
 
 #include <grpc/grpc_security.h>
 #include <grpc/support/alloc.h>
diff --git a/src/core/security/security_context.h b/src/core/lib/security/security_context.h
index 794258edbc..e9e4e503bc 100644
--- a/src/core/security/security_context.h
+++ b/src/core/lib/security/security_context.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONTEXT_H
-#define GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONTEXT_H
+#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
+#define GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
 
-#include "src/core/iomgr/pollset.h"
-#include "src/core/security/credentials.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/security/credentials.h"
 
 /* --- grpc_auth_context ---
 
@@ -111,4 +111,4 @@ grpc_auth_context *grpc_auth_context_from_arg(const grpc_arg *arg);
 grpc_auth_context *grpc_find_auth_context_in_args(
     const grpc_channel_args *args);
 
-#endif /* GRPC_INTERNAL_CORE_SECURITY_SECURITY_CONTEXT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H */
diff --git a/src/core/security/server_auth_filter.c b/src/core/lib/security/server_auth_filter.c
index 3d8e5e8d35..158cde0e2c 100644
--- a/src/core/security/server_auth_filter.c
+++ b/src/core/lib/security/server_auth_filter.c
@@ -33,9 +33,9 @@
 
 #include <string.h>
 
-#include "src/core/security/auth_filters.h"
-#include "src/core/security/credentials.h"
-#include "src/core/security/security_context.h"
+#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/security_context.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -259,6 +259,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 
 const grpc_channel_filter grpc_server_auth_filter = {
     auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem, set_pollset, destroy_call_elem, sizeof(channel_data),
-    init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer,
-    "server-auth"};
+    init_call_elem,          set_pollset,          destroy_call_elem,
+    sizeof(channel_data),    init_channel_elem,    destroy_channel_elem,
+    grpc_call_next_get_peer, "server-auth"};
diff --git a/src/core/statistics/census_init.c b/src/core/lib/statistics/census_init.c
index b6a962f228..bbecd62764 100644
--- a/src/core/statistics/census_init.c
+++ b/src/core/lib/statistics/census_init.c
@@ -31,11 +31,11 @@
  *
  */
 
-#include "src/core/statistics/census_interface.h"
+#include "src/core/lib/statistics/census_interface.h"
 
 #include <grpc/support/log.h>
-#include "src/core/statistics/census_rpc_stats.h"
-#include "src/core/statistics/census_tracing.h"
+#include "src/core/lib/statistics/census_rpc_stats.h"
+#include "src/core/lib/statistics/census_tracing.h"
 
 void census_init(void) {
   census_tracing_init();
diff --git a/src/core/statistics/census_interface.h b/src/core/lib/statistics/census_interface.h
index c43acbd317..b3b3439072 100644
--- a/src/core/statistics/census_interface.h
+++ b/src/core/lib/statistics/census_interface.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_STATISTICS_CENSUS_INTERFACE_H
-#define GRPC_INTERNAL_CORE_STATISTICS_CENSUS_INTERFACE_H
+#ifndef GRPC_CORE_LIB_STATISTICS_CENSUS_INTERFACE_H
+#define GRPC_CORE_LIB_STATISTICS_CENSUS_INTERFACE_H
 
 #include <grpc/support/port_platform.h>
 
@@ -73,4 +73,4 @@ census_op_id census_tracing_start_op(void);
 /* Ends tracing. Calling this function will invalidate the input op_id. */
 void census_tracing_end_op(census_op_id op_id);
 
-#endif /* GRPC_INTERNAL_CORE_STATISTICS_CENSUS_INTERFACE_H */
+#endif /* GRPC_CORE_LIB_STATISTICS_CENSUS_INTERFACE_H */
diff --git a/src/core/statistics/census_log.c b/src/core/lib/statistics/census_log.c
index 257ba586a3..1fb942a78a 100644
--- a/src/core/statistics/census_log.c
+++ b/src/core/lib/statistics/census_log.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -89,8 +89,7 @@
    include the name of the structure, which will be passed as the first
    argument. E.g. cl_block_initialize() will initialize a cl_block.
 */
-#include "src/core/statistics/census_log.h"
-#include <string.h>
+#include "src/core/lib/statistics/census_log.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/cpu.h>
@@ -98,6 +97,7 @@
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
+#include <string.h>
 
 /* End of platform specific code */
 
diff --git a/src/core/statistics/census_log.h b/src/core/lib/statistics/census_log.h
index 356437c346..c3fbd555ba 100644
--- a/src/core/statistics/census_log.h
+++ b/src/core/lib/statistics/census_log.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_STATISTICS_CENSUS_LOG_H
-#define GRPC_INTERNAL_CORE_STATISTICS_CENSUS_LOG_H
+#ifndef GRPC_CORE_LIB_STATISTICS_CENSUS_LOG_H
+#define GRPC_CORE_LIB_STATISTICS_CENSUS_LOG_H
 
 #include <stddef.h>
 
@@ -88,4 +88,4 @@ size_t census_log_remaining_space(void);
    out-of-space. */
 int census_log_out_of_space_count(void);
 
-#endif /* GRPC_INTERNAL_CORE_STATISTICS_CENSUS_LOG_H */
+#endif /* GRPC_CORE_LIB_STATISTICS_CENSUS_LOG_H */
diff --git a/src/core/statistics/census_rpc_stats.c b/src/core/lib/statistics/census_rpc_stats.c
index 524a60793a..2182561668 100644
--- a/src/core/statistics/census_rpc_stats.c
+++ b/src/core/lib/statistics/census_rpc_stats.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,16 +33,16 @@
 
 #include <string.h>
 
-#include "src/core/statistics/census_interface.h"
-#include "src/core/statistics/census_rpc_stats.h"
-#include "src/core/statistics/hash_table.h"
-#include "src/core/statistics/census_tracing.h"
-#include "src/core/statistics/window_stats.h"
-#include "src/core/support/murmur_hash.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
+#include "src/core/lib/statistics/census_interface.h"
+#include "src/core/lib/statistics/census_rpc_stats.h"
+#include "src/core/lib/statistics/census_tracing.h"
+#include "src/core/lib/statistics/hash_table.h"
+#include "src/core/lib/statistics/window_stats.h"
+#include "src/core/lib/support/murmur_hash.h"
+#include "src/core/lib/support/string.h"
 
 #define NUM_INTERVALS 3
 #define MINUTE_INTERVAL 0
@@ -85,8 +85,8 @@ static void delete_key(void *key) { gpr_free(key); }
 
 static const census_ht_option ht_opt = {
     CENSUS_HT_POINTER /* key type */, 1999 /* n_of_buckets */,
-    simple_hash /* hash function */, cmp_str_keys /* key comparator */,
-    delete_stats /* data deleter */, delete_key /* key deleter */
+    simple_hash /* hash function */,  cmp_str_keys /* key comparator */,
+    delete_stats /* data deleter */,  delete_key /* key deleter */
 };
 
 static void init_rpc_stats(void *stats) {
diff --git a/src/core/statistics/census_rpc_stats.h b/src/core/lib/statistics/census_rpc_stats.h
index 8bdbc49490..00bb48205e 100644
--- a/src/core/statistics/census_rpc_stats.h
+++ b/src/core/lib/statistics/census_rpc_stats.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_STATISTICS_CENSUS_RPC_STATS_H
-#define GRPC_INTERNAL_CORE_STATISTICS_CENSUS_RPC_STATS_H
+#ifndef GRPC_CORE_LIB_STATISTICS_CENSUS_RPC_STATS_H
+#define GRPC_CORE_LIB_STATISTICS_CENSUS_RPC_STATS_H
 
-#include "src/core/statistics/census_interface.h"
 #include <grpc/support/port_platform.h>
+#include "src/core/lib/statistics/census_interface.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -98,4 +98,4 @@ void census_stats_store_shutdown(void);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_STATISTICS_CENSUS_RPC_STATS_H */
+#endif /* GRPC_CORE_LIB_STATISTICS_CENSUS_RPC_STATS_H */
diff --git a/src/core/statistics/census_tracing.c b/src/core/lib/statistics/census_tracing.c
index dc0f8a26f5..b58ae733fc 100644
--- a/src/core/statistics/census_tracing.c
+++ b/src/core/lib/statistics/census_tracing.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,18 +31,18 @@
  *
  */
 
-#include "src/core/statistics/census_interface.h"
-#include "src/core/statistics/census_tracing.h"
+#include "src/core/lib/statistics/census_tracing.h"
+#include "src/core/lib/statistics/census_interface.h"
 
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/statistics/hash_table.h"
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
+#include "src/core/lib/statistics/hash_table.h"
+#include "src/core/lib/support/string.h"
 
 void census_trace_obj_destroy(census_trace_obj *obj) {
   census_trace_annotation *p = obj->annotations;
@@ -60,8 +60,11 @@ static void delete_trace_obj(void *obj) {
 }
 
 static const census_ht_option ht_opt = {
-    CENSUS_HT_UINT64 /* key type */, 571 /* n_of_buckets */, NULL /* hash */,
-    NULL /* compare_keys */, delete_trace_obj /* delete data */,
+    CENSUS_HT_UINT64 /* key type */,
+    571 /* n_of_buckets */,
+    NULL /* hash */,
+    NULL /* compare_keys */,
+    delete_trace_obj /* delete data */,
     NULL /* delete key */
 };
 
diff --git a/src/core/statistics/census_tracing.h b/src/core/lib/statistics/census_tracing.h
index bb3f2556d2..a101abf3cb 100644
--- a/src/core/statistics/census_tracing.h
+++ b/src/core/lib/statistics/census_tracing.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_STATISTICS_CENSUS_TRACING_H
-#define GRPC_INTERNAL_CORE_STATISTICS_CENSUS_TRACING_H
+#ifndef GRPC_CORE_LIB_STATISTICS_CENSUS_TRACING_H
+#define GRPC_CORE_LIB_STATISTICS_CENSUS_TRACING_H
 
 #include <grpc/support/time.h>
-#include "src/core/statistics/census_rpc_stats.h"
+#include "src/core/lib/statistics/census_rpc_stats.h"
 
 /* WARNING: The data structures and APIs provided by this file are for GRPC
    library's internal use ONLY. They might be changed in backward-incompatible
@@ -93,4 +93,4 @@ census_trace_obj **census_get_active_ops(int *num_active_ops);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_STATISTICS_CENSUS_TRACING_H */
+#endif /* GRPC_CORE_LIB_STATISTICS_CENSUS_TRACING_H */
diff --git a/src/core/statistics/hash_table.c b/src/core/lib/statistics/hash_table.c
index 0cadcd4740..18b7442a0c 100644
--- a/src/core/statistics/hash_table.c
+++ b/src/core/lib/statistics/hash_table.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#include "src/core/statistics/hash_table.h"
+#include "src/core/lib/statistics/hash_table.h"
 
-#include <stdio.h>
 #include <stddef.h>
+#include <stdio.h>
 
-#include <grpc/support/log.h>
 #include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #define CENSUS_HT_NUM_BUCKETS 1999
diff --git a/src/core/statistics/hash_table.h b/src/core/lib/statistics/hash_table.h
index d9860a909f..8f74ec82aa 100644
--- a/src/core/statistics/hash_table.h
+++ b/src/core/lib/statistics/hash_table.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_STATISTICS_HASH_TABLE_H
-#define GRPC_INTERNAL_CORE_STATISTICS_HASH_TABLE_H
+#ifndef GRPC_CORE_LIB_STATISTICS_HASH_TABLE_H
+#define GRPC_CORE_LIB_STATISTICS_HASH_TABLE_H
 
 #include <stddef.h>
 
@@ -128,4 +128,4 @@ typedef void (*census_ht_itr_cb)(census_ht_key key, const void *val_ptr,
    should not invalidate data entries. */
 uint64_t census_ht_for_all(const census_ht *ht, census_ht_itr_cb);
 
-#endif /* GRPC_INTERNAL_CORE_STATISTICS_HASH_TABLE_H */
+#endif /* GRPC_CORE_LIB_STATISTICS_HASH_TABLE_H */
diff --git a/src/core/statistics/window_stats.c b/src/core/lib/statistics/window_stats.c
index 3f2940853a..53427a24bc 100644
--- a/src/core/statistics/window_stats.c
+++ b/src/core/lib/statistics/window_stats.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/statistics/window_stats.h"
-#include <math.h>
-#include <stddef.h>
-#include <string.h>
+#include "src/core/lib/statistics/window_stats.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include <math.h>
+#include <stddef.h>
+#include <string.h>
 
 /* typedefs make typing long names easier. Use cws (for census_window_stats) */
 typedef census_window_stats_stat_info cws_stat_info;
diff --git a/src/core/statistics/window_stats.h b/src/core/lib/statistics/window_stats.h
index f4732e96a0..8dec50d620 100644
--- a/src/core/statistics/window_stats.h
+++ b/src/core/lib/statistics/window_stats.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_STATISTICS_WINDOW_STATS_H
-#define GRPC_INTERNAL_CORE_STATISTICS_WINDOW_STATS_H
+#ifndef GRPC_CORE_LIB_STATISTICS_WINDOW_STATS_H
+#define GRPC_CORE_LIB_STATISTICS_WINDOW_STATS_H
 
 #include <grpc/support/time.h>
 
@@ -170,4 +170,4 @@ void census_window_stats_get_sums(const struct census_window_stats *wstats,
    assertion failure). This function is thread-compatible. */
 void census_window_stats_destroy(struct census_window_stats *wstats);
 
-#endif /* GRPC_INTERNAL_CORE_STATISTICS_WINDOW_STATS_H */
+#endif /* GRPC_CORE_LIB_STATISTICS_WINDOW_STATS_H */
diff --git a/src/core/support/alloc.c b/src/core/lib/support/alloc.c
index 0a064b2c18..27fa6a95ed 100644
--- a/src/core/support/alloc.c
+++ b/src/core/lib/support/alloc.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,10 +33,10 @@
 
 #include <grpc/support/alloc.h>
 
-#include <stdlib.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
-#include "src/core/profiling/timers.h"
+#include <stdlib.h>
+#include "src/core/lib/profiling/timers.h"
 
 static gpr_allocation_functions g_alloc_functions = {malloc, realloc, free};
 
@@ -87,4 +87,4 @@ void *gpr_malloc_aligned(size_t size, size_t alignment_log) {
   return (void *)ret;
 }
 
-void gpr_free_aligned(void *ptr) { free(((void **)ptr)[-1]); }
+void gpr_free_aligned(void *ptr) { gpr_free(((void **)ptr)[-1]); }
diff --git a/src/core/support/avl.c b/src/core/lib/support/avl.c
index f378b3ee17..f378b3ee17 100644
--- a/src/core/support/avl.c
+++ b/src/core/lib/support/avl.c
diff --git a/src/core/lib/support/backoff.c b/src/core/lib/support/backoff.c
new file mode 100644
index 0000000000..e89ef47220
--- /dev/null
+++ b/src/core/lib/support/backoff.c
@@ -0,0 +1,76 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/support/backoff.h"
+
+#include <grpc/support/useful.h>
+
+void gpr_backoff_init(gpr_backoff *backoff, double multiplier, double jitter,
+                      int64_t min_timeout_millis, int64_t max_timeout_millis) {
+  backoff->multiplier = multiplier;
+  backoff->jitter = jitter;
+  backoff->min_timeout_millis = min_timeout_millis;
+  backoff->max_timeout_millis = max_timeout_millis;
+  backoff->rng_state = (uint32_t)gpr_now(GPR_CLOCK_REALTIME).tv_nsec;
+}
+
+gpr_timespec gpr_backoff_begin(gpr_backoff *backoff, gpr_timespec now) {
+  backoff->current_timeout_millis = backoff->min_timeout_millis;
+  return gpr_time_add(
+      now, gpr_time_from_millis(backoff->current_timeout_millis, GPR_TIMESPAN));
+}
+
+/* Generate a random number between 0 and 1. */
+static double generate_uniform_random_number(uint32_t *rng_state) {
+  *rng_state = (1103515245 * *rng_state + 12345) % ((uint32_t)1 << 31);
+  return *rng_state / (double)((uint32_t)1 << 31);
+}
+
+gpr_timespec gpr_backoff_step(gpr_backoff *backoff, gpr_timespec now) {
+  double new_timeout_millis =
+      backoff->multiplier * (double)backoff->current_timeout_millis;
+  double jitter_range = backoff->jitter * new_timeout_millis;
+  double jitter =
+      (2 * generate_uniform_random_number(&backoff->rng_state) - 1) *
+      jitter_range;
+  backoff->current_timeout_millis =
+      GPR_CLAMP((int64_t)(new_timeout_millis + jitter),
+                backoff->min_timeout_millis, backoff->max_timeout_millis);
+  return gpr_time_add(
+      now, gpr_time_from_millis(backoff->current_timeout_millis, GPR_TIMESPAN));
+}
+
+void gpr_backoff_reset(gpr_backoff *backoff) {
+  // forces step() to return a timeout of min_timeout_millis
+  backoff->current_timeout_millis = 0;
+}
diff --git a/src/core/channel/client_uchannel.h b/src/core/lib/support/backoff.h
index 92a831493c..6d40c15546 100644
--- a/src/core/channel/client_uchannel.h
+++ b/src/core/lib/support/backoff.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,30 +31,38 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_CHANNEL_CLIENT_MICROCHANNEL_H
-#define GRPC_INTERNAL_CORE_CHANNEL_CLIENT_MICROCHANNEL_H
+#ifndef GRPC_CORE_LIB_SUPPORT_BACKOFF_H
+#define GRPC_CORE_LIB_SUPPORT_BACKOFF_H
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/client_config/resolver.h"
+#include <grpc/support/time.h>
 
-#define GRPC_MICROCHANNEL_SUBCHANNEL_ARG "grpc.microchannel_subchannel_key"
+typedef struct {
+  /// const: multiplier between retry attempts
+  double multiplier;
+  /// const: amount to randomize backoffs
+  double jitter;
+  /// const: minimum time between retries in milliseconds
+  int64_t min_timeout_millis;
+  /// const: maximum time between retries in milliseconds
+  int64_t max_timeout_millis;
 
-/* A client microchannel (aka uchannel) is a channel wrapping a subchannel, for
- * the purposes of lightweight RPC communications from within the core.*/
+  /// random number generator
+  uint32_t rng_state;
 
-extern const grpc_channel_filter grpc_client_uchannel_filter;
+  /// current retry timeout in milliseconds
+  int64_t current_timeout_millis;
+} gpr_backoff;
 
-grpc_connectivity_state grpc_client_uchannel_check_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect);
+/// Initialize backoff machinery - does not need to be destroyed
+void gpr_backoff_init(gpr_backoff *backoff, double multiplier, double jitter,
+                      int64_t min_timeout_millis, int64_t max_timeout_millis);
 
-void grpc_client_uchannel_watch_connectivity_state(
-    grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset,
-    grpc_connectivity_state *state, grpc_closure *on_complete);
+/// Begin retry loop: returns a timespec for the NEXT retry
+gpr_timespec gpr_backoff_begin(gpr_backoff *backoff, gpr_timespec now);
+/// Step a retry loop: returns a timespec for the NEXT retry
+gpr_timespec gpr_backoff_step(gpr_backoff *backoff, gpr_timespec now);
+/// Reset the backoff, so the next gpr_backoff_step will be a gpr_backoff_begin
+/// instead
+void gpr_backoff_reset(gpr_backoff *backoff);
 
-grpc_channel *grpc_client_uchannel_create(grpc_subchannel *subchannel,
-                                          grpc_channel_args *args);
-
-void grpc_client_uchannel_set_connected_subchannel(
-    grpc_channel *uchannel, grpc_connected_subchannel *connected_subchannel);
-
-#endif /* GRPC_INTERNAL_CORE_CHANNEL_CLIENT_MICROCHANNEL_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_BACKOFF_H */
diff --git a/src/core/support/block_annotate.h b/src/core/lib/support/block_annotate.h
index 3cd8eee272..bd3071655e 100644
--- a/src/core/support/block_annotate.h
+++ b/src/core/lib/support/block_annotate.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H
-#define GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H
+#ifndef GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H
+#define GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H
 
 /* These annotations identify the beginning and end of regions where
    the code may block for reasons other than synchronization functions.
@@ -45,4 +45,4 @@
   do {                                      \
   } while (0)
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H */
diff --git a/src/core/support/cmdline.c b/src/core/lib/support/cmdline.c
index b517f30b2d..35c4990b22 100644
--- a/src/core/support/cmdline.c
+++ b/src/core/lib/support/cmdline.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,10 +37,10 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/support/string.h"
 
 typedef enum { ARGTYPE_INT, ARGTYPE_BOOL, ARGTYPE_STRING } argtype;
 
diff --git a/src/core/support/cpu_iphone.c b/src/core/lib/support/cpu_iphone.c
index 82b49b47bc..e83191bada 100644
--- a/src/core/support/cpu_iphone.c
+++ b/src/core/lib/support/cpu_iphone.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/cpu_linux.c b/src/core/lib/support/cpu_linux.c
index 7af6a8f009..5597df2d03 100644
--- a/src/core/support/cpu_linux.c
+++ b/src/core/lib/support/cpu_linux.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,10 +39,10 @@
 
 #ifdef GPR_CPU_LINUX
 
-#include <sched.h>
 #include <errno.h>
-#include <unistd.h>
+#include <sched.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
diff --git a/src/core/support/cpu_posix.c b/src/core/lib/support/cpu_posix.c
index 8f01c284ca..e508ddd8ca 100644
--- a/src/core/support/cpu_posix.c
+++ b/src/core/lib/support/cpu_posix.c
@@ -36,8 +36,8 @@
 #ifdef GPR_CPU_POSIX
 
 #include <errno.h>
-#include <unistd.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
diff --git a/src/core/support/cpu_windows.c b/src/core/lib/support/cpu_windows.c
index ce32eb0a9d..0f84a9e5ea 100644
--- a/src/core/support/cpu_windows.c
+++ b/src/core/lib/support/cpu_windows.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/env.h b/src/core/lib/support/env.h
index 24172d8673..ddc4ee3c6d 100644
--- a/src/core/support/env.h
+++ b/src/core/lib/support/env.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_ENV_H
-#define GRPC_INTERNAL_CORE_SUPPORT_ENV_H
+#ifndef GRPC_CORE_LIB_SUPPORT_ENV_H
+#define GRPC_CORE_LIB_SUPPORT_ENV_H
 
 #include <stdio.h>
 
@@ -57,4 +57,4 @@ void gpr_setenv(const char *name, const char *value);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_ENV_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_ENV_H */
diff --git a/src/core/support/env_linux.c b/src/core/lib/support/env_linux.c
index fe51f846b7..a86133e6c3 100644
--- a/src/core/support/env_linux.c
+++ b/src/core/lib/support/env_linux.c
@@ -40,7 +40,7 @@
 
 #ifdef GPR_LINUX_ENV
 
-#include "src/core/support/env.h"
+#include "src/core/lib/support/env.h"
 
 #include <dlfcn.h>
 #include <features.h>
@@ -51,7 +51,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 char *gpr_getenv(const char *name) {
 #if defined(GPR_BACKWARDS_COMPATIBILITY_MODE)
diff --git a/src/core/support/env_posix.c b/src/core/lib/support/env_posix.c
index 1dd2af56bc..1b57b094a9 100644
--- a/src/core/support/env_posix.c
+++ b/src/core/lib/support/env_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,14 +35,14 @@
 
 #ifdef GPR_POSIX_ENV
 
-#include "src/core/support/env.h"
+#include "src/core/lib/support/env.h"
 
 #include <stdlib.h>
 
 #include <grpc/support/log.h>
 
-#include "src/core/support/string.h"
 #include <grpc/support/string_util.h>
+#include "src/core/lib/support/string.h"
 
 char *gpr_getenv(const char *name) {
   char *result = getenv(name);
diff --git a/src/core/support/env_win32.c b/src/core/lib/support/env_win32.c
index 10258283ba..566feee49e 100644
--- a/src/core/support/env_win32.c
+++ b/src/core/lib/support/env_win32.c
@@ -35,8 +35,8 @@
 
 #ifdef GPR_WIN32
 
-#include "src/core/support/env.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/string.h"
 
 #ifdef __MINGW32__
 errno_t getenv_s(size_t *size_needed, char *buffer, size_t size,
diff --git a/src/core/support/histogram.c b/src/core/lib/support/histogram.c
index 20ed2b14b1..62227be1a6 100644
--- a/src/core/support/histogram.c
+++ b/src/core/lib/support/histogram.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -38,8 +38,8 @@
 #include <string.h>
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/port_platform.h>
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 #include <grpc/support/useful.h>
 
 /* Histograms are stored with exponentially increasing bucket sizes.
diff --git a/src/core/support/host_port.c b/src/core/lib/support/host_port.c
index 23f65b1581..e03f6241ff 100644
--- a/src/core/support/host_port.c
+++ b/src/core/lib/support/host_port.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,10 +35,10 @@
 
 #include <string.h>
 
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/support/string.h"
 
 int gpr_join_host_port(char **out, const char *host, int port) {
   if (host[0] != '[' && strchr(host, ':') != NULL) {
diff --git a/src/core/support/load_file.c b/src/core/lib/support/load_file.c
index 650bd62ccb..0cecd5edd5 100644
--- a/src/core/support/load_file.c
+++ b/src/core/lib/support/load_file.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/support/load_file.h"
+#include "src/core/lib/support/load_file.h"
 
 #include <errno.h>
 #include <string.h>
@@ -40,8 +40,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/block_annotate.h"
-#include "src/core/support/string.h"
+#include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/string.h"
 
 gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
                         int *success) {
diff --git a/src/core/support/load_file.h b/src/core/lib/support/load_file.h
index 746319a50d..fe030c967e 100644
--- a/src/core/support/load_file.h
+++ b/src/core/lib/support/load_file.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_LOAD_FILE_H
-#define GRPC_INTERNAL_CORE_SUPPORT_LOAD_FILE_H
+#ifndef GRPC_CORE_LIB_SUPPORT_LOAD_FILE_H
+#define GRPC_CORE_LIB_SUPPORT_LOAD_FILE_H
 
 #include <stdio.h>
 
@@ -52,4 +52,4 @@ gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_LOAD_FILE_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_LOAD_FILE_H */
diff --git a/src/core/support/log.c b/src/core/lib/support/log.c
index 04156a5b1f..cd6a0726cf 100644
--- a/src/core/support/log.c
+++ b/src/core/lib/support/log.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/log_android.c b/src/core/lib/support/log_android.c
index 5d0c7d820d..640c9d7099 100644
--- a/src/core/support/log_android.c
+++ b/src/core/lib/support/log_android.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,12 +35,12 @@
 
 #ifdef GPR_ANDROID
 
+#include <android/log.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <string.h>
-#include <android/log.h>
 
 static android_LogPriority severity_to_log_priority(gpr_log_severity severity) {
   switch (severity) {
diff --git a/src/core/support/log_linux.c b/src/core/lib/support/log_linux.c
index d66b7a3cc0..e60512c526 100644
--- a/src/core/support/log_linux.c
+++ b/src/core/lib/support/log_linux.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -47,12 +47,12 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
-#include <stdio.h>
+#include <linux/unistd.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <string.h>
-#include <time.h>
-#include <linux/unistd.h>
 #include <sys/syscall.h>
+#include <time.h>
 #include <unistd.h>
 
 static long gettid(void) { return syscall(__NR_gettid); }
diff --git a/src/core/support/log_posix.c b/src/core/lib/support/log_posix.c
index 3ff171f99c..7429dd0a2c 100644
--- a/src/core/support/log_posix.c
+++ b/src/core/lib/support/log_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -38,12 +38,12 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
-#include <stdio.h>
+#include <pthread.h>
 #include <stdarg.h>
-#include <string.h>
 #include <stdio.h>
+#include <stdio.h>
+#include <string.h>
 #include <time.h>
-#include <pthread.h>
 
 static intptr_t gettid(void) { return (intptr_t)pthread_self(); }
 
diff --git a/src/core/support/log_win32.c b/src/core/lib/support/log_win32.c
index e18e667fe5..cec99440a5 100644
--- a/src/core/support/log_win32.c
+++ b/src/core/lib/support/log_win32.c
@@ -35,17 +35,17 @@
 
 #ifdef GPR_WIN32
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/log_win32.h>
 #include <grpc/support/log.h>
-#include <grpc/support/time.h>
+#include <grpc/support/log_win32.h>
 #include <grpc/support/string_util.h>
+#include <grpc/support/time.h>
 
-#include "src/core/support/string.h"
-#include "src/core/support/string_win32.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/support/string_win32.h"
 
 void gpr_log(const char *file, int line, gpr_log_severity severity,
              const char *format, ...) {
diff --git a/src/core/support/murmur_hash.c b/src/core/lib/support/murmur_hash.c
index a5261c0cc0..97832f1510 100644
--- a/src/core/support/murmur_hash.c
+++ b/src/core/lib/support/murmur_hash.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/support/murmur_hash.h"
+#include "src/core/lib/support/murmur_hash.h"
 
 #define ROTL32(x, r) ((x) << (r)) | ((x) >> (32 - (r)))
 
diff --git a/src/core/support/murmur_hash.h b/src/core/lib/support/murmur_hash.h
index 0bf04f731a..e54cdf2592 100644
--- a/src/core/support/murmur_hash.h
+++ b/src/core/lib/support/murmur_hash.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_MURMUR_HASH_H
-#define GRPC_INTERNAL_CORE_SUPPORT_MURMUR_HASH_H
+#ifndef GRPC_CORE_LIB_SUPPORT_MURMUR_HASH_H
+#define GRPC_CORE_LIB_SUPPORT_MURMUR_HASH_H
 
 #include <grpc/support/port_platform.h>
 
@@ -41,4 +41,4 @@
 /* compute the hash of key (length len) */
 uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed);
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_MURMUR_HASH_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_MURMUR_HASH_H */
diff --git a/src/core/support/slice.c b/src/core/lib/support/slice.c
index b9a7c77bda..cf3953ce4e 100644
--- a/src/core/support/slice.c
+++ b/src/core/lib/support/slice.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/slice_buffer.c b/src/core/lib/support/slice_buffer.c
index 66f111d767..563e659dd7 100644
--- a/src/core/support/slice_buffer.c
+++ b/src/core/lib/support/slice_buffer.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/stack_lockfree.c b/src/core/lib/support/stack_lockfree.c
index 9daecd2e18..de80486132 100644
--- a/src/core/support/stack_lockfree.c
+++ b/src/core/lib/support/stack_lockfree.c
@@ -31,15 +31,15 @@
  *
  */
 
-#include "src/core/support/stack_lockfree.h"
+#include "src/core/lib/support/stack_lockfree.h"
 
 #include <stdlib.h>
 #include <string.h>
 
-#include <grpc/support/port_platform.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 /* The lockfree node structure is a single architecture-level
    word that allows for an atomic CAS to set it up. */
@@ -64,10 +64,10 @@ typedef union lockfree_node {
   struct lockfree_node_contents contents;
 } lockfree_node;
 
-#define ENTRY_ALIGNMENT_BITS 3 /* make sure that entries aligned to 8-bytes */
-#define INVALID_ENTRY_INDEX                        \
-  ((1 << 16) - 1) /* reserve this entry as invalid \
-                    */
+/* make sure that entries aligned to 8-bytes */
+#define ENTRY_ALIGNMENT_BITS 3
+/* reserve this entry as invalid */
+#define INVALID_ENTRY_INDEX ((1 << 16) - 1)
 
 struct gpr_stack_lockfree {
   lockfree_node *entries;
diff --git a/src/core/support/stack_lockfree.h b/src/core/lib/support/stack_lockfree.h
index ca58dd007a..a030a01d1f 100644
--- a/src/core/support/stack_lockfree.h
+++ b/src/core/lib/support/stack_lockfree.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_STACK_LOCKFREE_H
-#define GRPC_INTERNAL_CORE_SUPPORT_STACK_LOCKFREE_H
+#ifndef GRPC_CORE_LIB_SUPPORT_STACK_LOCKFREE_H
+#define GRPC_CORE_LIB_SUPPORT_STACK_LOCKFREE_H
 
 #include <stddef.h>
 
@@ -50,4 +50,4 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *, int entry);
 /* Returns -1 on empty or the actual entry number */
 int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack);
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_STACK_LOCKFREE_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_STACK_LOCKFREE_H */
diff --git a/src/core/support/string.c b/src/core/lib/support/string.c
index 1f541de40f..365d861de3 100644
--- a/src/core/support/string.c
+++ b/src/core/lib/support/string.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 #include <ctype.h>
 #include <stddef.h>
diff --git a/src/core/support/string.h b/src/core/lib/support/string.h
index e6755de106..68c02878e0 100644
--- a/src/core/support/string.h
+++ b/src/core/lib/support/string.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_STRING_H
-#define GRPC_INTERNAL_CORE_SUPPORT_STRING_H
+#ifndef GRPC_CORE_LIB_SUPPORT_STRING_H
+#define GRPC_CORE_LIB_SUPPORT_STRING_H
 
 #include <stddef.h>
 
 #include <grpc/support/port_platform.h>
-#include <grpc/support/slice_buffer.h>
 #include <grpc/support/slice.h>
+#include <grpc/support/slice_buffer.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -118,4 +118,4 @@ char *gpr_strvec_flatten(gpr_strvec *strs, size_t *total_length);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_STRING_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_STRING_H */
diff --git a/src/core/support/string_posix.c b/src/core/lib/support/string_posix.c
index 25c333db4e..a73b3106a5 100644
--- a/src/core/support/string_posix.c
+++ b/src/core/lib/support/string_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,8 +35,8 @@
 
 #ifdef GPR_POSIX_STRING
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <string.h>
 
 #include <grpc/support/alloc.h>
diff --git a/src/core/support/string_win32.c b/src/core/lib/support/string_win32.c
index 3b1f702cf1..16b7e37f2a 100644
--- a/src/core/support/string_win32.c
+++ b/src/core/lib/support/string_win32.c
@@ -37,13 +37,13 @@
 
 #ifdef GPR_WIN32
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <string.h>
 
 #include <grpc/support/alloc.h>
 
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 int gpr_asprintf(char **strp, const char *format, ...) {
   va_list args;
diff --git a/src/core/support/string_win32.h b/src/core/lib/support/string_win32.h
index e3043656fb..f47d567715 100644
--- a/src/core/support/string_win32.h
+++ b/src/core/lib/support/string_win32.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_STRING_WIN32_H
-#define GRPC_INTERNAL_CORE_SUPPORT_STRING_WIN32_H
+#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H
+#define GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H
 
 #include <grpc/support/port_platform.h>
 
@@ -44,4 +44,4 @@ LPSTR gpr_tchar_to_char(LPCTSTR input);
 
 #endif /* GPR_WIN32 */
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_STRING_WIN32_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H */
diff --git a/src/core/support/subprocess_posix.c b/src/core/lib/support/subprocess_posix.c
index 171054e4da..662e7dd999 100644
--- a/src/core/support/subprocess_posix.c
+++ b/src/core/lib/support/subprocess_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,15 +37,15 @@
 
 #include <grpc/support/subprocess.h>
 
-#include <unistd.h>
 #include <assert.h>
 #include <errno.h>
-#include <stdio.h>
-#include <string.h>
 #include <signal.h>
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <unistd.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/support/subprocess_windows.c b/src/core/lib/support/subprocess_windows.c
index 2b25ef063a..264306f1bd 100644
--- a/src/core/support/subprocess_windows.c
+++ b/src/core/lib/support/subprocess_windows.c
@@ -35,15 +35,15 @@
 
 #ifdef GPR_WINDOWS_SUBPROCESS
 
-#include <windows.h>
 #include <string.h>
 #include <tchar.h>
+#include <windows.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/subprocess.h>
-#include "src/core/support/string.h"
-#include "src/core/support/string_win32.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/support/string_win32.h"
 
 struct gpr_subprocess {
   PROCESS_INFORMATION pi;
diff --git a/src/core/support/sync.c b/src/core/lib/support/sync.c
index 69e3e39c5c..800cf20287 100644
--- a/src/core/support/sync.c
+++ b/src/core/lib/support/sync.c
@@ -33,9 +33,9 @@
 
 /* Generic implementation of synchronization primitives. */
 
+#include <grpc/support/atm.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
-#include <grpc/support/atm.h>
 
 /* Number of mutexes to allocate for events, to avoid lock contention.
    Should be a prime. */
diff --git a/src/core/support/sync_posix.c b/src/core/lib/support/sync_posix.c
index d3c483f1b5..a5e59db8c7 100644
--- a/src/core/support/sync_posix.c
+++ b/src/core/lib/support/sync_posix.c
@@ -36,11 +36,11 @@
 #ifdef GPR_POSIX_SYNC
 
 #include <errno.h>
-#include <time.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
-#include "src/core/profiling/timers.h"
+#include <time.h>
+#include "src/core/lib/profiling/timers.h"
 
 void gpr_mu_init(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_init(mu, NULL) == 0); }
 
diff --git a/src/core/support/sync_win32.c b/src/core/lib/support/sync_win32.c
index 41998ebcb6..41998ebcb6 100644
--- a/src/core/support/sync_win32.c
+++ b/src/core/lib/support/sync_win32.c
diff --git a/src/core/support/thd.c b/src/core/lib/support/thd.c
index 41daeb5d0e..d59aace38d 100644
--- a/src/core/support/thd.c
+++ b/src/core/lib/support/thd.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/thd_internal.h b/src/core/lib/support/thd_internal.h
index 1508c4691f..f269a3249e 100644
--- a/src/core/support/thd_internal.h
+++ b/src/core/lib/support/thd_internal.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,9 +31,9 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_THD_INTERNAL_H
-#define GRPC_INTERNAL_CORE_SUPPORT_THD_INTERNAL_H
+#ifndef GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H
+#define GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H
 
 /* Internal interfaces between modules within the gpr support library.  */
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_THD_INTERNAL_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H */
diff --git a/src/core/support/thd_posix.c b/src/core/lib/support/thd_posix.c
index 653a1c88c1..4d874d3656 100644
--- a/src/core/support/thd_posix.c
+++ b/src/core/lib/support/thd_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,13 +37,13 @@
 
 #ifdef GPR_POSIX_SYNC
 
-#include <pthread.h>
-#include <stdlib.h>
-#include <string.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
 
 struct thd_arg {
   void (*body)(void *arg); /* body of a thread */
diff --git a/src/core/support/thd_win32.c b/src/core/lib/support/thd_win32.c
index a9db180c1b..630eb7f625 100644
--- a/src/core/support/thd_win32.c
+++ b/src/core/lib/support/thd_win32.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,10 +37,10 @@
 
 #ifdef GPR_WIN32
 
-#include <string.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/thd.h>
+#include <string.h>
 
 #if defined(_MSC_VER)
 #define thread_local __declspec(thread)
diff --git a/src/core/support/time.c b/src/core/lib/support/time.c
index 423d12ffc0..0e2c8fcf1a 100644
--- a/src/core/support/time.c
+++ b/src/core/lib/support/time.c
@@ -33,11 +33,11 @@
 
 /* Generic implementation of time calls. */
 
+#include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <limits.h>
 #include <stdio.h>
 #include <string.h>
-#include <grpc/support/log.h>
 
 int gpr_time_cmp(gpr_timespec a, gpr_timespec b) {
   int cmp = (a.tv_sec > b.tv_sec) - (a.tv_sec < b.tv_sec);
diff --git a/src/core/support/time_posix.c b/src/core/lib/support/time_posix.c
index 36d75e7da2..fcfab2f2fa 100644
--- a/src/core/support/time_posix.c
+++ b/src/core/lib/support/time_posix.c
@@ -32,7 +32,7 @@
  */
 
 #include <grpc/support/port_platform.h>
-#include <src/core/support/time_precise.h>
+#include "src/core/lib/support/time_precise.h"
 
 #ifdef GPR_POSIX_TIME
 
@@ -44,7 +44,7 @@
 #endif
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
-#include "src/core/support/block_annotate.h"
+#include "src/core/lib/support/block_annotate.h"
 
 static struct timespec timespec_from_gpr(gpr_timespec gts) {
   struct timespec rv;
@@ -98,9 +98,9 @@ gpr_timespec gpr_now(gpr_clock_type clock_type) {
 #else
 /* For some reason Apple's OSes haven't implemented clock_gettime. */
 
-#include <sys/time.h>
 #include <mach/mach.h>
 #include <mach/mach_time.h>
+#include <sys/time.h>
 
 static double g_time_scale;
 static uint64_t g_time_start;
diff --git a/src/core/support/time_precise.c b/src/core/lib/support/time_precise.c
index a2cf74bc84..31ac47e0f8 100644
--- a/src/core/support/time_precise.c
+++ b/src/core/lib/support/time_precise.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/time_precise.h b/src/core/lib/support/time_precise.h
index 80c5000123..e1faee1f9f 100644
--- a/src/core/support/time_precise.h
+++ b/src/core/lib/support/time_precise.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_CORE_SUPPORT_TIME_PRECISE_H_
-#define GRPC_CORE_SUPPORT_TIME_PRECISE_H_
+#ifndef GRPC_CORE_LIB_SUPPORT_TIME_PRECISE_H
+#define GRPC_CORE_LIB_SUPPORT_TIME_PRECISE_H
 
 #include <grpc/support/time.h>
 
 void gpr_precise_clock_init(void);
 void gpr_precise_clock_now(gpr_timespec *clk);
 
-#endif /* GRPC_CORE_SUPPORT_TIME_PRECISE_ */
+#endif /* GRPC_CORE_LIB_SUPPORT_TIME_PRECISE_H */
diff --git a/src/core/support/time_win32.c b/src/core/lib/support/time_win32.c
index 8af957e6f4..a6ac003fb8 100644
--- a/src/core/support/time_win32.c
+++ b/src/core/lib/support/time_win32.c
@@ -39,12 +39,12 @@
 
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
-#include <src/core/support/time_precise.h>
-#include <sys/timeb.h>
-#include <process.h>
 #include <limits.h>
+#include <process.h>
+#include <sys/timeb.h>
 
-#include "src/core/support/block_annotate.h"
+#include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/time_precise.h"
 
 static LARGE_INTEGER g_start_time;
 static double g_time_scale;
diff --git a/src/core/support/tls_pthread.c b/src/core/lib/support/tls_pthread.c
index 9683a6e547..bdc7ed14ae 100644
--- a/src/core/support/tls_pthread.c
+++ b/src/core/lib/support/tls_pthread.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/support/tmpfile.h b/src/core/lib/support/tmpfile.h
index cecc71e242..4fec2076e3 100644
--- a/src/core/support/tmpfile.h
+++ b/src/core/lib/support/tmpfile.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_TMPFILE_H
-#define GRPC_INTERNAL_CORE_SUPPORT_TMPFILE_H
+#ifndef GRPC_CORE_LIB_SUPPORT_TMPFILE_H
+#define GRPC_CORE_LIB_SUPPORT_TMPFILE_H
 
 #include <stdio.h>
 
@@ -52,4 +52,4 @@ FILE *gpr_tmpfile(const char *prefix, char **tmp_filename);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_TMPFILE_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_TMPFILE_H */
diff --git a/src/core/support/tmpfile_posix.c b/src/core/lib/support/tmpfile_posix.c
index b16eeacf9d..743f45e1bc 100644
--- a/src/core/support/tmpfile_posix.c
+++ b/src/core/lib/support/tmpfile_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/support/tmpfile.h"
+#include "src/core/lib/support/tmpfile.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -46,7 +46,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename) {
   FILE *result = NULL;
diff --git a/src/core/support/tmpfile_win32.c b/src/core/lib/support/tmpfile_win32.c
index 3000f0029f..05d92b6036 100644
--- a/src/core/support/tmpfile_win32.c
+++ b/src/core/lib/support/tmpfile_win32.c
@@ -44,8 +44,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/string_win32.h"
-#include "src/core/support/tmpfile.h"
+#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/tmpfile.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
   FILE *result = NULL;
diff --git a/src/core/support/wrap_memcpy.c b/src/core/lib/support/wrap_memcpy.c
index 15c289f7b8..15c289f7b8 100644
--- a/src/core/support/wrap_memcpy.c
+++ b/src/core/lib/support/wrap_memcpy.c
diff --git a/src/core/surface/alarm.c b/src/core/lib/surface/alarm.c
index 8169ede065..368683378e 100644
--- a/src/core/surface/alarm.c
+++ b/src/core/lib/surface/alarm.c
@@ -31,10 +31,10 @@
  *
  */
 
-#include "src/core/iomgr/timer.h"
-#include "src/core/surface/completion_queue.h"
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/surface/completion_queue.h"
 
 struct grpc_alarm {
   grpc_timer alarm;
diff --git a/src/core/surface/api_trace.c b/src/core/lib/surface/api_trace.c
index 9f0b900d46..3702c024db 100644
--- a/src/core/surface/api_trace.c
+++ b/src/core/lib/surface/api_trace.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,6 @@
  *
  */
 
-#include "src/core/surface/api_trace.h"
+#include "src/core/lib/surface/api_trace.h"
 
 int grpc_api_trace = 0;
diff --git a/src/core/surface/api_trace.h b/src/core/lib/surface/api_trace.h
index 82bbf3b62b..b50011c9e5 100644
--- a/src/core/surface/api_trace.h
+++ b/src/core/lib/surface/api_trace.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_API_TRACE_H
-#define GRPC_INTERNAL_CORE_SURFACE_API_TRACE_H
+#ifndef GRPC_CORE_LIB_SURFACE_API_TRACE_H
+#define GRPC_CORE_LIB_SURFACE_API_TRACE_H
 
-#include "src/core/debug/trace.h"
 #include <grpc/support/log.h>
+#include "src/core/lib/debug/trace.h"
 
 extern int grpc_api_trace;
 
@@ -62,4 +62,4 @@ extern int grpc_api_trace;
     gpr_log(GPR_INFO, fmt GRPC_API_TRACE_UNWRAP##nargs args); \
   }
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_API_TRACE_H */
+#endif /* GRPC_CORE_LIB_SURFACE_API_TRACE_H */
diff --git a/src/core/surface/byte_buffer.c b/src/core/lib/surface/byte_buffer.c
index fb39c4531d..03071ef92c 100644
--- a/src/core/surface/byte_buffer.c
+++ b/src/core/lib/surface/byte_buffer.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/surface/byte_buffer_reader.c b/src/core/lib/surface/byte_buffer_reader.c
index 4679854227..7248f5fe71 100644
--- a/src/core/surface/byte_buffer_reader.c
+++ b/src/core/lib/surface/byte_buffer_reader.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,17 +31,17 @@
  *
  */
 
-#include <string.h>
 #include <grpc/byte_buffer_reader.h>
+#include <string.h>
 
+#include <grpc/byte_buffer.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
-#include <grpc/byte_buffer.h>
 
-#include "src/core/compression/message_compress.h"
+#include "src/core/lib/compression/message_compress.h"
 
 static int is_compressed(grpc_byte_buffer *buffer) {
   switch (buffer->type) {
diff --git a/src/core/surface/call.c b/src/core/lib/surface/call.c
index 1b117aa6b8..77ad410b50 100644
--- a/src/core/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -43,16 +43,16 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/compression/algorithm_metadata.h"
-#include "src/core/iomgr/timer.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
-#include "src/core/surface/channel.h"
-#include "src/core/surface/completion_queue.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/compression/algorithm_metadata.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 /** The maximum number of concurrent batches possible.
     Based upon the maximum number of individually queueable ops in the batch
@@ -174,6 +174,9 @@ struct grpc_call {
   /* Received call statuses from various sources */
   received_status status[STATUS_SOURCE_COUNT];
 
+  /* Call stats: only valid after trailing metadata received */
+  grpc_transport_stream_stats stats;
+
   /* Compression algorithm for the call */
   grpc_compression_algorithm compression_algorithm;
   /* Supported encodings (compression algorithms), a bitset */
@@ -909,7 +912,7 @@ static void set_cancelled_value(grpc_status_code status, void *dest) {
   *(int *)dest = (status != GRPC_STATUS_OK);
 }
 
-static int are_write_flags_valid(uint32_t flags) {
+static bool are_write_flags_valid(uint32_t flags) {
   /* check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set */
   const uint32_t allowed_write_positions =
       (GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
@@ -917,6 +920,15 @@ static int are_write_flags_valid(uint32_t flags) {
   return !(flags & invalid_positions);
 }
 
+static bool are_initial_metadata_flags_valid(uint32_t flags, bool is_client) {
+  /* check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set */
+  uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
+  if (!is_client) {
+    invalid_positions |= GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST;
+  }
+  return !(flags & invalid_positions);
+}
+
 static batch_control *allocate_batch_control(grpc_call *call) {
   size_t i;
   for (i = 0; i < MAX_CONCURRENT_BATCHES; i++) {
@@ -1196,7 +1208,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
     switch (op->op) {
       case GRPC_OP_SEND_INITIAL_METADATA:
         /* Flag validation: currently allow no flags */
-        if (op->flags != 0) {
+        if (!are_initial_metadata_flags_valid(op->flags, call->is_client)) {
           error = GRPC_CALL_ERROR_INVALID_FLAGS;
           goto done_with_error;
         }
@@ -1220,6 +1232,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         call->metadata_batch[0][0].deadline = call->send_deadline;
         stream_op.send_initial_metadata =
             &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */];
+        stream_op.send_idempotent_request =
+            (op->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) != 0;
         break;
       case GRPC_OP_SEND_MESSAGE:
         if (!are_write_flags_valid(op->flags)) {
@@ -1371,6 +1385,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         bctl->recv_final_op = 1;
         stream_op.recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
+        stream_op.collect_stats = &call->stats;
         break;
       case GRPC_OP_RECV_CLOSE_ON_SERVER:
         /* Flag validation: currently allow no flags */
@@ -1392,6 +1407,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         bctl->recv_final_op = 1;
         stream_op.recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
+        stream_op.collect_stats = &call->stats;
         break;
     }
   }
@@ -1481,3 +1497,11 @@ void *grpc_call_context_get(grpc_call *call, grpc_context_index elem) {
 }
 
 uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; }
+
+grpc_compression_algorithm grpc_call_compression_for_level(
+    grpc_call *call, grpc_compression_level level) {
+  gpr_mu_lock(&call->mu);
+  const uint32_t accepted_encodings = call->encodings_accepted_by_peer;
+  gpr_mu_unlock(&call->mu);
+  return grpc_compression_algorithm_for_level(level, accepted_encodings);
+}
diff --git a/src/core/surface/call.h b/src/core/lib/surface/call.h
index 0bbffb98ae..e2e75865be 100644
--- a/src/core/surface/call.h
+++ b/src/core/lib/surface/call.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,16 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_CALL_H
-#define GRPC_INTERNAL_CORE_SURFACE_CALL_H
+#ifndef GRPC_CORE_LIB_SURFACE_CALL_H
+#define GRPC_CORE_LIB_SURFACE_CALL_H
+
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/context.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/surface_trace.h"
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/channel/context.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/surface_trace.h"
 #include <grpc/grpc.h>
+#include <grpc/impl/codegen/compression_types.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -102,8 +104,13 @@ void *grpc_call_context_get(grpc_call *call, grpc_context_index elem);
 
 uint8_t grpc_call_is_client(grpc_call *call);
 
+/* Return an appropriate compression algorithm for the requested compression \a
+ * level in the context of \a call. */
+grpc_compression_algorithm grpc_call_compression_for_level(
+    grpc_call *call, grpc_compression_level level);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_CALL_H */
+#endif /* GRPC_CORE_LIB_SURFACE_CALL_H */
diff --git a/src/core/surface/call_details.c b/src/core/lib/surface/call_details.c
index 60f0029819..08f606d84a 100644
--- a/src/core/surface/call_details.c
+++ b/src/core/lib/surface/call_details.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,7 +36,7 @@
 
 #include <string.h>
 
-#include "src/core/surface/api_trace.h"
+#include "src/core/lib/surface/api_trace.h"
 
 void grpc_call_details_init(grpc_call_details* cd) {
   GRPC_API_TRACE("grpc_call_details_init(cd=%p)", 1, (cd));
diff --git a/src/core/surface/call_log_batch.c b/src/core/lib/surface/call_log_batch.c
index 46756f418b..bc5a2ffb65 100644
--- a/src/core/surface/call_log_batch.c
+++ b/src/core/lib/surface/call_log_batch.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#include "src/core/surface/call.h"
+#include "src/core/lib/surface/call.h"
 
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/support/string.h"
 
 static void add_metadata(gpr_strvec *b, const grpc_metadata *md, size_t count) {
   size_t i;
diff --git a/src/core/surface/call_test_only.h b/src/core/lib/surface/call_test_only.h
index b57c95c64a..400214189e 100644
--- a/src/core/surface/call_test_only.h
+++ b/src/core/lib/surface/call_test_only.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_CALL_TEST_ONLY_H
-#define GRPC_INTERNAL_CORE_SURFACE_CALL_TEST_ONLY_H
+#ifndef GRPC_CORE_LIB_SURFACE_CALL_TEST_ONLY_H
+#define GRPC_CORE_LIB_SURFACE_CALL_TEST_ONLY_H
 
 #include <grpc/grpc.h>
 
@@ -61,4 +61,4 @@ uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_CALL_TEST_ONLY_H */
+#endif /* GRPC_CORE_LIB_SURFACE_CALL_TEST_ONLY_H */
diff --git a/src/core/surface/channel.c b/src/core/lib/surface/channel.c
index 12d8ebceb9..d815daa70c 100644
--- a/src/core/surface/channel.c
+++ b/src/core/lib/surface/channel.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/surface/channel.h"
+#include "src/core/lib/surface/channel.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -40,13 +40,14 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/client_config/resolver_registry.h"
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
-#include "src/core/surface/init.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/client_config/resolver_registry.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/surface/init.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 /** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS.
  *  Avoids needing to take a metadata context lock for sending status
@@ -82,24 +83,25 @@ struct grpc_channel {
 
 static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg, bool success);
 
-grpc_channel *grpc_channel_create_from_filters(
-    grpc_exec_ctx *exec_ctx, const char *target,
-    const grpc_channel_filter **filters, size_t num_filters,
-    const grpc_channel_args *args, int is_client) {
-  size_t i;
-  size_t size =
-      sizeof(grpc_channel) + grpc_channel_stack_size(filters, num_filters);
-  grpc_channel *channel = gpr_malloc(size);
+grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
+                                  const grpc_channel_args *args,
+                                  grpc_channel_stack_type channel_stack_type,
+                                  grpc_transport *optional_transport) {
+  bool is_client = grpc_channel_stack_type_is_client(channel_stack_type);
+
+  grpc_channel *channel = grpc_channel_init_create_stack(
+      exec_ctx, channel_stack_type, sizeof(grpc_channel), args, 1,
+      destroy_channel, NULL, optional_transport);
+
   memset(channel, 0, sizeof(*channel));
   channel->target = gpr_strdup(target);
-  GPR_ASSERT(grpc_is_initialized() && "call grpc_init()");
   channel->is_client = is_client;
   gpr_mu_init(&channel->registered_call_mu);
   channel->registered_calls = NULL;
 
   channel->max_message_length = DEFAULT_MAX_MESSAGE_LENGTH;
   if (args) {
-    for (i = 0; i < args->num_args; i++) {
+    for (size_t i = 0; i < args->num_args; i++) {
       if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_MESSAGE_LENGTH)) {
         if (args->args[i].type != GRPC_ARG_INTEGER) {
           gpr_log(GPR_ERROR, "%s ignored: it must be an integer",
@@ -152,11 +154,6 @@ grpc_channel *grpc_channel_create_from_filters(
     gpr_free(default_authority);
   }
 
-  grpc_channel_stack_init(exec_ctx, 1, destroy_channel, channel, filters,
-                          num_filters, args,
-                          is_client ? "CLIENT_CHANNEL" : "SERVER_CHANNEL",
-                          CHANNEL_STACK_FROM_CHANNEL(channel));
-
   return channel;
 }
 
diff --git a/src/core/surface/channel.h b/src/core/lib/surface/channel.h
index 00240c637f..09de0fccc9 100644
--- a/src/core/surface/channel.h
+++ b/src/core/lib/surface/channel.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,17 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_CHANNEL_H
-#define GRPC_INTERNAL_CORE_SURFACE_CHANNEL_H
+#ifndef GRPC_CORE_LIB_SURFACE_CHANNEL_H
+#define GRPC_CORE_LIB_SURFACE_CHANNEL_H
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/client_config/subchannel_factory.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/client_config/subchannel_factory.h"
+#include "src/core/lib/surface/channel_stack_type.h"
 
-grpc_channel *grpc_channel_create_from_filters(
-    grpc_exec_ctx *exec_ctx, const char *target,
-    const grpc_channel_filter **filters, size_t count,
-    const grpc_channel_args *args, int is_client);
+grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
+                                  const grpc_channel_args *args,
+                                  grpc_channel_stack_type channel_stack_type,
+                                  grpc_transport *optional_transport);
 
 /** Get a (borrowed) pointer to this channels underlying channel stack */
 grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel);
@@ -71,4 +72,4 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
   grpc_channel_internal_unref(exec_ctx, channel)
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_CHANNEL_H */
+#endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_H */
diff --git a/src/core/surface/channel_connectivity.c b/src/core/lib/surface/channel_connectivity.c
index 2dd4fce26b..2f5d763e70 100644
--- a/src/core/surface/channel_connectivity.c
+++ b/src/core/lib/surface/channel_connectivity.c
@@ -31,16 +31,15 @@
  *
  */
 
-#include "src/core/surface/channel.h"
+#include "src/core/lib/surface/channel.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/channel/client_channel.h"
-#include "src/core/channel/client_uchannel.h"
-#include "src/core/iomgr/timer.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/completion_queue.h"
+#include "src/core/lib/channel/client_channel.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/completion_queue.h"
 
 grpc_connectivity_state grpc_channel_check_connectivity_state(
     grpc_channel *channel, int try_to_connect) {
@@ -58,12 +57,6 @@ grpc_connectivity_state grpc_channel_check_connectivity_state(
     grpc_exec_ctx_finish(&exec_ctx);
     return state;
   }
-  if (client_channel_elem->filter == &grpc_client_uchannel_filter) {
-    state = grpc_client_uchannel_check_connectivity_state(
-        &exec_ctx, client_channel_elem, try_to_connect);
-    grpc_exec_ctx_finish(&exec_ctx);
-    return state;
-  }
   gpr_log(GPR_ERROR,
           "grpc_channel_check_connectivity_state called on something that is "
           "not a (u)client channel, but '%s'",
@@ -98,9 +91,6 @@ static void delete_state_watcher(grpc_exec_ctx *exec_ctx, state_watcher *w) {
   if (client_channel_elem->filter == &grpc_client_channel_filter) {
     GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, w->channel,
                                 "watch_channel_connectivity");
-  } else if (client_channel_elem->filter == &grpc_client_uchannel_filter) {
-    GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, w->channel,
-                                "watch_uchannel_connectivity");
   } else {
     abort();
   }
@@ -209,11 +199,8 @@ void grpc_channel_watch_connectivity_state(
     grpc_client_channel_watch_connectivity_state(&exec_ctx, client_channel_elem,
                                                  grpc_cq_pollset(cq), &w->state,
                                                  &w->on_complete);
-  } else if (client_channel_elem->filter == &grpc_client_uchannel_filter) {
-    GRPC_CHANNEL_INTERNAL_REF(channel, "watch_uchannel_connectivity");
-    grpc_client_uchannel_watch_connectivity_state(
-        &exec_ctx, client_channel_elem, grpc_cq_pollset(cq), &w->state,
-        &w->on_complete);
+  } else {
+    abort();
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
diff --git a/src/core/lib/surface/channel_init.c b/src/core/lib/surface/channel_init.c
new file mode 100644
index 0000000000..fc69f61f77
--- /dev/null
+++ b/src/core/lib/surface/channel_init.c
@@ -0,0 +1,146 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/surface/channel_init.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/useful.h>
+
+typedef struct stage_slot {
+  grpc_channel_init_stage fn;
+  void *arg;
+  int priority;
+  size_t insertion_order;
+} stage_slot;
+
+typedef struct stage_slots {
+  stage_slot *slots;
+  size_t num_slots;
+  size_t cap_slots;
+} stage_slots;
+
+static stage_slots g_slots[GRPC_NUM_CHANNEL_STACK_TYPES];
+static bool g_finalized;
+
+void grpc_channel_init_init(void) {
+  for (int i = 0; i < GRPC_NUM_CHANNEL_STACK_TYPES; i++) {
+    g_slots[i].slots = NULL;
+    g_slots[i].num_slots = 0;
+    g_slots[i].cap_slots = 0;
+  }
+  g_finalized = false;
+}
+
+void grpc_channel_init_register_stage(grpc_channel_stack_type type,
+                                      int priority,
+                                      grpc_channel_init_stage stage,
+                                      void *stage_arg) {
+  GPR_ASSERT(!g_finalized);
+  if (g_slots[type].cap_slots == g_slots[type].num_slots) {
+    g_slots[type].cap_slots = GPR_MAX(8, 3 * g_slots[type].cap_slots / 2);
+    g_slots[type].slots =
+        gpr_realloc(g_slots[type].slots,
+                    g_slots[type].cap_slots * sizeof(*g_slots[type].slots));
+  }
+  stage_slot *s = &g_slots[type].slots[g_slots[type].num_slots++];
+  s->insertion_order = g_slots[type].num_slots;
+  s->priority = priority;
+  s->fn = stage;
+  s->arg = stage_arg;
+}
+
+static int compare_slots(const void *a, const void *b) {
+  const stage_slot *sa = a;
+  const stage_slot *sb = b;
+
+  int c = GPR_ICMP(sa->priority, sb->priority);
+  if (c != 0) return c;
+  return GPR_ICMP(sa->insertion_order, sb->insertion_order);
+}
+
+void grpc_channel_init_finalize(void) {
+  GPR_ASSERT(!g_finalized);
+  for (int i = 0; i < GRPC_NUM_CHANNEL_STACK_TYPES; i++) {
+    qsort(g_slots[i].slots, g_slots[i].num_slots, sizeof(*g_slots[i].slots),
+          compare_slots);
+  }
+  g_finalized = true;
+}
+
+void grpc_channel_init_shutdown(void) {
+  for (int i = 0; i < GRPC_NUM_CHANNEL_STACK_TYPES; i++) {
+    gpr_free(g_slots[i].slots);
+    g_slots[i].slots = (void *)(uintptr_t)0xdeadbeef;
+  }
+}
+
+static const char *name_for_type(grpc_channel_stack_type type) {
+  switch (type) {
+    case GRPC_CLIENT_CHANNEL:
+      return "CLIENT_CHANNEL";
+    case GRPC_CLIENT_SUBCHANNEL:
+      return "CLIENT_SUBCHANNEL";
+    case GRPC_SERVER_CHANNEL:
+      return "SERVER_CHANNEL";
+    case GRPC_CLIENT_LAME_CHANNEL:
+      return "CLIENT_LAME_CHANNEL";
+    case GRPC_CLIENT_DIRECT_CHANNEL:
+      return "CLIENT_DIRECT_CHANNEL";
+    case GRPC_NUM_CHANNEL_STACK_TYPES:
+      break;
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
+void *grpc_channel_init_create_stack(
+    grpc_exec_ctx *exec_ctx, grpc_channel_stack_type type, size_t prefix_bytes,
+    const grpc_channel_args *args, int initial_refs, grpc_iomgr_cb_func destroy,
+    void *destroy_arg, grpc_transport *transport) {
+  GPR_ASSERT(g_finalized);
+
+  grpc_channel_stack_builder *builder = grpc_channel_stack_builder_create();
+  grpc_channel_stack_builder_set_name(builder, name_for_type(type));
+  grpc_channel_stack_builder_set_channel_arguments(builder, args);
+  grpc_channel_stack_builder_set_transport(builder, transport);
+
+  for (size_t i = 0; i < g_slots[type].num_slots; i++) {
+    const stage_slot *slot = &g_slots[type].slots[i];
+    if (!slot->fn(builder, slot->arg)) {
+      grpc_channel_stack_builder_destroy(builder);
+      return NULL;
+    }
+  }
+
+  return grpc_channel_stack_builder_finish(exec_ctx, builder, prefix_bytes,
+                                           initial_refs, destroy, destroy_arg);
+}
diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h
new file mode 100644
index 0000000000..a4d8271ca6
--- /dev/null
+++ b/src/core/lib/surface/channel_init.h
@@ -0,0 +1,86 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H
+#define GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H
+
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_stack_type.h"
+#include "src/core/lib/transport/transport.h"
+
+/// This module provides a way for plugins (and the grpc core library itself)
+/// to register mutators for channel stacks.
+/// It also provides a universal entry path to run those mutators to build
+/// a channel stack for various subsystems.
+
+/// One stage of mutation: call functions against \a builder to influence the
+/// finally constructed channel stack
+typedef bool (*grpc_channel_init_stage)(grpc_channel_stack_builder *builder,
+                                        void *arg);
+
+/// Global initialization of the system
+void grpc_channel_init_init(void);
+
+/// Register one stage of mutators.
+/// Stages are run in priority order (lowest to highest), and then in
+/// registration order (in the case of a tie).
+/// Stages are registered against one of the pre-determined channel stack
+/// types.
+void grpc_channel_init_register_stage(grpc_channel_stack_type type,
+                                      int priority,
+                                      grpc_channel_init_stage stage_fn,
+                                      void *stage_arg);
+
+/// Finalize registration. No more calls to grpc_channel_init_register_stage are
+/// allowed.
+void grpc_channel_init_finalize(void);
+/// Shutdown the channel init system
+void grpc_channel_init_shutdown(void);
+
+/// Construct a channel stack of some sort: see channel_stack.h for details
+/// \a type is the type of channel stack to create
+/// \a prefix_bytes is the number of bytes before the channel stack to allocate
+/// \a args are configuration arguments for the channel stack
+/// \a initial_refs is the initial refcount to give the channel stack
+/// \a destroy and \a destroy_arg specify how to destroy the channel stack
+///    if destroy_arg is NULL, the returned value from this function will be
+///    substituted
+/// \a optional_transport is either NULL or a constructed transport object
+/// Returns a pointer to the base of the memory allocated (the actual channel
+/// stack object will be prefix_bytes past that pointer)
+void *grpc_channel_init_create_stack(
+    grpc_exec_ctx *exec_ctx, grpc_channel_stack_type type, size_t prefix_bytes,
+    const grpc_channel_args *args, int initial_refs, grpc_iomgr_cb_func destroy,
+    void *destroy_arg, grpc_transport *optional_transport);
+
+#endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H */
diff --git a/src/core/surface/channel_ping.c b/src/core/lib/surface/channel_ping.c
index 983f1c8a66..dd862cdadd 100644
--- a/src/core/surface/channel_ping.c
+++ b/src/core/lib/surface/channel_ping.c
@@ -31,15 +31,15 @@
  *
  */
 
-#include "src/core/surface/channel.h"
+#include "src/core/lib/surface/channel.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/completion_queue.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/completion_queue.h"
 
 typedef struct {
   grpc_closure closure;
diff --git a/src/core/lib/surface/channel_stack_type.c b/src/core/lib/surface/channel_stack_type.c
new file mode 100644
index 0000000000..c35d603ca3
--- /dev/null
+++ b/src/core/lib/surface/channel_stack_type.c
@@ -0,0 +1,54 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/surface/channel_stack_type.h"
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+
+bool grpc_channel_stack_type_is_client(grpc_channel_stack_type type) {
+  switch (type) {
+    case GRPC_CLIENT_CHANNEL:
+      return true;
+    case GRPC_CLIENT_SUBCHANNEL:
+      return true;
+    case GRPC_CLIENT_LAME_CHANNEL:
+      return true;
+    case GRPC_CLIENT_DIRECT_CHANNEL:
+      return true;
+    case GRPC_SERVER_CHANNEL:
+      return false;
+    case GRPC_NUM_CHANNEL_STACK_TYPES:
+      break;
+  }
+  GPR_UNREACHABLE_CODE(return true;);
+}
diff --git a/src/core/httpcli/parser.h b/src/core/lib/surface/channel_stack_type.h
index cd0383637f..16608fa386 100644
--- a/src/core/httpcli/parser.h
+++ b/src/core/lib/surface/channel_stack_type.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,34 +31,28 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_HTTPCLI_PARSER_H
-#define GRPC_INTERNAL_CORE_HTTPCLI_PARSER_H
+#ifndef GRPC_CORE_LIB_SURFACE_CHANNEL_STACK_TYPE_H
+#define GRPC_CORE_LIB_SURFACE_CHANNEL_STACK_TYPE_H
 
-#include "src/core/httpcli/httpcli.h"
-#include <grpc/support/port_platform.h>
-#include <grpc/support/slice.h>
+#include <stdbool.h>
 
 typedef enum {
-  GRPC_HTTPCLI_INITIAL_RESPONSE,
-  GRPC_HTTPCLI_HEADERS,
-  GRPC_HTTPCLI_BODY
-} grpc_httpcli_parser_state;
-
-typedef struct {
-  grpc_httpcli_parser_state state;
-
-  grpc_httpcli_response r;
-  size_t body_capacity;
-  size_t hdr_capacity;
-
-  uint8_t cur_line[GRPC_HTTPCLI_MAX_HEADER_LENGTH];
-  size_t cur_line_length;
-} grpc_httpcli_parser;
-
-void grpc_httpcli_parser_init(grpc_httpcli_parser* parser);
-void grpc_httpcli_parser_destroy(grpc_httpcli_parser* parser);
-
-int grpc_httpcli_parser_parse(grpc_httpcli_parser* parser, gpr_slice slice);
-int grpc_httpcli_parser_eof(grpc_httpcli_parser* parser);
-
-#endif /* GRPC_INTERNAL_CORE_HTTPCLI_PARSER_H */
+  // normal top-half client channel with load-balancing, connection management
+  GRPC_CLIENT_CHANNEL,
+  // bottom-half of a client channel: everything that happens post-load
+  // balancing (bound to a specific transport)
+  GRPC_CLIENT_SUBCHANNEL,
+  // a permanently broken client channel
+  GRPC_CLIENT_LAME_CHANNEL,
+  // a directly connected client channel (without load-balancing, directly talks
+  // to a transport)
+  GRPC_CLIENT_DIRECT_CHANNEL,
+  // server side channel
+  GRPC_SERVER_CHANNEL,
+  // must be last
+  GRPC_NUM_CHANNEL_STACK_TYPES
+} grpc_channel_stack_type;
+
+bool grpc_channel_stack_type_is_client(grpc_channel_stack_type type);
+
+#endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_STACK_TYPE_H */
diff --git a/src/core/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index f6a95ebbd3..a0d7002053 100644
--- a/src/core/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/surface/completion_queue.h"
+#include "src/core/lib/surface/completion_queue.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -41,14 +41,14 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/timer.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
-#include "src/core/surface/event_string.h"
-#include "src/core/surface/surface_trace.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/event_string.h"
+#include "src/core/lib/surface/surface_trace.h"
 
 typedef struct {
   grpc_pollset_worker **worker;
@@ -86,7 +86,7 @@ struct grpc_completion_queue {
 #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
 
 static gpr_mu g_freelist_mu;
-grpc_completion_queue *g_freelist;
+static grpc_completion_queue *g_freelist;
 
 static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc,
                                      bool success);
diff --git a/src/core/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index 27ef90f2d5..35591cb6f4 100644
--- a/src/core/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_COMPLETION_QUEUE_H
-#define GRPC_INTERNAL_CORE_SURFACE_COMPLETION_QUEUE_H
+#ifndef GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H
+#define GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H
 
 /* Internal API for completion queues */
 
-#include "src/core/iomgr/pollset.h"
 #include <grpc/grpc.h>
+#include "src/core/lib/iomgr/pollset.h"
 
 typedef struct grpc_cq_completion {
   /** user supplied tag */
@@ -88,4 +88,4 @@ int grpc_cq_is_server_cq(grpc_completion_queue *cc);
 void grpc_cq_global_init(void);
 void grpc_cq_global_shutdown(void);
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_COMPLETION_QUEUE_H */
+#endif /* GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H */
diff --git a/src/core/surface/event_string.c b/src/core/lib/surface/event_string.c
index 33cd4a43aa..360c718a17 100644
--- a/src/core/surface/event_string.c
+++ b/src/core/lib/surface/event_string.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,13 @@
  *
  */
 
-#include "src/core/surface/event_string.h"
+#include "src/core/lib/surface/event_string.h"
 
 #include <stdio.h>
 
-#include "src/core/support/string.h"
 #include <grpc/byte_buffer.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/support/string.h"
 
 static void addhdr(gpr_strvec *buf, grpc_event *ev) {
   char *tmp;
diff --git a/src/core/surface/event_string.h b/src/core/lib/surface/event_string.h
index 07c474e3a0..577e9c718f 100644
--- a/src/core/surface/event_string.h
+++ b/src/core/lib/surface/event_string.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_EVENT_STRING_H
-#define GRPC_INTERNAL_CORE_SURFACE_EVENT_STRING_H
+#ifndef GRPC_CORE_LIB_SURFACE_EVENT_STRING_H
+#define GRPC_CORE_LIB_SURFACE_EVENT_STRING_H
 
 #include <grpc/grpc.h>
 
 /* Returns a string describing an event. Must be later freed with gpr_free() */
 char *grpc_event_string(grpc_event *ev);
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_EVENT_STRING_H */
+#endif /* GRPC_CORE_LIB_SURFACE_EVENT_STRING_H */
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
new file mode 100644
index 0000000000..dbfc8a9336
--- /dev/null
+++ b/src/core/lib/surface/init.c
@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include <limits.h>
+#include <memory.h>
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/time.h>
+/* TODO(ctiller): find another way? - better not to include census here */
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/census/grpc_plugin.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/client_channel.h"
+#include "src/core/lib/channel/compress_filter.h"
+#include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/channel/http_server_filter.h"
+#include "src/core/lib/client_config/lb_policies/pick_first.h"
+#include "src/core/lib/client_config/lb_policies/round_robin.h"
+#include "src/core/lib/client_config/lb_policy_registry.h"
+#include "src/core/lib/client_config/resolver_registry.h"
+#include "src/core/lib/client_config/resolvers/dns_resolver.h"
+#include "src/core/lib/client_config/resolvers/sockaddr_resolver.h"
+#include "src/core/lib/client_config/subchannel.h"
+#include "src/core/lib/client_config/subchannel_index.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/init.h"
+#include "src/core/lib/surface/lame_client.h"
+#include "src/core/lib/surface/server.h"
+#include "src/core/lib/surface/surface_trace.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/transport_impl.h"
+
+#ifndef GRPC_DEFAULT_NAME_PREFIX
+#define GRPC_DEFAULT_NAME_PREFIX "dns:///"
+#endif
+
+#define MAX_PLUGINS 128
+
+static gpr_once g_basic_init = GPR_ONCE_INIT;
+static gpr_mu g_init_mu;
+static int g_initializations;
+
+static void do_basic_init(void) {
+  gpr_mu_init(&g_init_mu);
+  /* TODO(ctiller): ideally remove this strict linkage */
+  grpc_register_plugin(census_grpc_plugin_init, census_grpc_plugin_destroy);
+  g_initializations = 0;
+}
+
+static bool append_filter(grpc_channel_stack_builder *builder, void *arg) {
+  return grpc_channel_stack_builder_append_filter(
+      builder, (const grpc_channel_filter *)arg, NULL, NULL);
+}
+
+static bool prepend_filter(grpc_channel_stack_builder *builder, void *arg) {
+  return grpc_channel_stack_builder_prepend_filter(
+      builder, (const grpc_channel_filter *)arg, NULL, NULL);
+}
+
+static bool maybe_add_http_filter(grpc_channel_stack_builder *builder,
+                                  void *arg) {
+  grpc_transport *t = grpc_channel_stack_builder_get_transport(builder);
+  if (t && strstr(t->vtable->name, "http")) {
+    return grpc_channel_stack_builder_prepend_filter(
+        builder, (const grpc_channel_filter *)arg, NULL, NULL);
+  }
+  return true;
+}
+
+static void register_builtin_channel_init() {
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, prepend_filter,
+                                   (void *)&grpc_compress_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   prepend_filter,
+                                   (void *)&grpc_compress_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, prepend_filter,
+                                   (void *)&grpc_compress_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX,
+                                   maybe_add_http_filter,
+                                   (void *)&grpc_http_client_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX,
+                                   grpc_add_connected_filter, NULL);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   maybe_add_http_filter,
+                                   (void *)&grpc_http_client_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   grpc_add_connected_filter, NULL);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_http_filter,
+                                   (void *)&grpc_http_server_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   grpc_add_connected_filter, NULL);
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, append_filter,
+                                   (void *)&grpc_client_channel_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_LAME_CHANNEL, INT_MAX,
+                                   append_filter, (void *)&grpc_lame_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, prepend_filter,
+                                   (void *)&grpc_server_top_filter);
+}
+
+typedef struct grpc_plugin {
+  void (*init)();
+  void (*destroy)();
+} grpc_plugin;
+
+static grpc_plugin g_all_of_the_plugins[MAX_PLUGINS];
+static int g_number_of_plugins = 0;
+
+void grpc_register_plugin(void (*init)(void), void (*destroy)(void)) {
+  GRPC_API_TRACE("grpc_register_plugin(init=%p, destroy=%p)", 2,
+                 ((void *)(intptr_t)init, (void *)(intptr_t)destroy));
+  GPR_ASSERT(g_number_of_plugins != MAX_PLUGINS);
+  g_all_of_the_plugins[g_number_of_plugins].init = init;
+  g_all_of_the_plugins[g_number_of_plugins].destroy = destroy;
+  g_number_of_plugins++;
+}
+
+void grpc_init(void) {
+  int i;
+  gpr_once_init(&g_basic_init, do_basic_init);
+
+  gpr_mu_lock(&g_init_mu);
+  if (++g_initializations == 1) {
+    gpr_time_init();
+    grpc_mdctx_global_init();
+    grpc_channel_init_init();
+    grpc_lb_policy_registry_init(grpc_pick_first_lb_factory_create());
+    grpc_register_lb_policy(grpc_pick_first_lb_factory_create());
+    grpc_register_lb_policy(grpc_round_robin_lb_factory_create());
+    grpc_resolver_registry_init(GRPC_DEFAULT_NAME_PREFIX);
+    grpc_register_resolver_type(grpc_dns_resolver_factory_create());
+    grpc_register_resolver_type(grpc_ipv4_resolver_factory_create());
+    grpc_register_resolver_type(grpc_ipv6_resolver_factory_create());
+#ifdef GPR_POSIX_SOCKET
+    grpc_register_resolver_type(grpc_unix_resolver_factory_create());
+#endif
+    grpc_register_tracer("api", &grpc_api_trace);
+    grpc_register_tracer("channel", &grpc_trace_channel);
+    grpc_register_tracer("http", &grpc_http_trace);
+    grpc_register_tracer("flowctl", &grpc_flowctl_trace);
+    grpc_register_tracer("connectivity_state", &grpc_connectivity_state_trace);
+    grpc_register_tracer("channel_stack_builder",
+                         &grpc_trace_channel_stack_builder);
+    grpc_security_pre_init();
+    grpc_iomgr_init();
+    grpc_executor_init();
+    grpc_tracer_init("GRPC_TRACE");
+    gpr_timers_global_init();
+    grpc_cq_global_init();
+    grpc_subchannel_index_init();
+    for (i = 0; i < g_number_of_plugins; i++) {
+      if (g_all_of_the_plugins[i].init != NULL) {
+        g_all_of_the_plugins[i].init();
+      }
+    }
+    /* register channel finalization AFTER all plugins, to ensure that it's run
+     * at the appropriate time */
+    grpc_register_security_filters();
+    register_builtin_channel_init();
+    /* no more changes to channel init pipelines */
+    grpc_channel_init_finalize();
+  }
+  gpr_mu_unlock(&g_init_mu);
+  GRPC_API_TRACE("grpc_init(void)", 0, ());
+}
+
+void grpc_shutdown(void) {
+  int i;
+  GRPC_API_TRACE("grpc_shutdown(void)", 0, ());
+  gpr_mu_lock(&g_init_mu);
+  if (--g_initializations == 0) {
+    grpc_executor_shutdown();
+    grpc_cq_global_shutdown();
+    grpc_iomgr_shutdown();
+    grpc_subchannel_index_shutdown();
+    gpr_timers_global_destroy();
+    grpc_tracer_shutdown();
+    grpc_resolver_registry_shutdown();
+    grpc_lb_policy_registry_shutdown();
+    for (i = 0; i < g_number_of_plugins; i++) {
+      if (g_all_of_the_plugins[i].destroy != NULL) {
+        g_all_of_the_plugins[i].destroy();
+      }
+    }
+    grpc_channel_init_shutdown();
+    grpc_mdctx_global_shutdown();
+  }
+  gpr_mu_unlock(&g_init_mu);
+}
+
+int grpc_is_initialized(void) {
+  int r;
+  gpr_once_init(&g_basic_init, do_basic_init);
+  gpr_mu_lock(&g_init_mu);
+  r = g_initializations > 0;
+  gpr_mu_unlock(&g_init_mu);
+  return r;
+}
diff --git a/src/core/surface/init.h b/src/core/lib/surface/init.h
index 771c30f412..10e2a5896e 100644
--- a/src/core/surface/init.h
+++ b/src/core/lib/surface/init.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_INIT_H
-#define GRPC_INTERNAL_CORE_SURFACE_INIT_H
+#ifndef GRPC_CORE_LIB_SURFACE_INIT_H
+#define GRPC_CORE_LIB_SURFACE_INIT_H
 
+void grpc_register_security_filters(void);
 void grpc_security_pre_init(void);
 int grpc_is_initialized(void);
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_INIT_H */
+#endif /* GRPC_CORE_LIB_SURFACE_INIT_H */
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
new file mode 100644
index 0000000000..d3c2f645a7
--- /dev/null
+++ b/src/core/lib/surface/init_secure.c
@@ -0,0 +1,89 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/surface/init.h"
+
+#include <limits.h>
+#include <string.h>
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
+
+void grpc_security_pre_init(void) {
+  grpc_register_tracer("secure_endpoint", &grpc_trace_secure_endpoint);
+  grpc_register_tracer("transport_security", &tsi_tracing_enabled);
+}
+
+static bool maybe_prepend_client_auth_filter(
+    grpc_channel_stack_builder *builder, void *arg) {
+  const grpc_channel_args *args =
+      grpc_channel_stack_builder_get_channel_arguments(builder);
+  if (args) {
+    for (size_t i = 0; i < args->num_args; i++) {
+      if (0 == strcmp(GRPC_SECURITY_CONNECTOR_ARG, args->args[i].key)) {
+        return grpc_channel_stack_builder_prepend_filter(
+            builder, &grpc_client_auth_filter, NULL, NULL);
+      }
+    }
+  }
+  return true;
+}
+
+static bool maybe_prepend_server_auth_filter(
+    grpc_channel_stack_builder *builder, void *arg) {
+  const grpc_channel_args *args =
+      grpc_channel_stack_builder_get_channel_arguments(builder);
+  if (args) {
+    for (size_t i = 0; i < args->num_args; i++) {
+      if (0 == strcmp(GRPC_SERVER_CREDENTIALS_ARG, args->args[i].key)) {
+        return grpc_channel_stack_builder_prepend_filter(
+            builder, &grpc_server_auth_filter, NULL, NULL);
+      }
+    }
+  }
+  return true;
+}
+
+void grpc_register_security_filters(void) {
+  grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX,
+                                   maybe_prepend_client_auth_filter, NULL);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   maybe_prepend_client_auth_filter, NULL);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_prepend_server_auth_filter, NULL);
+}
diff --git a/src/core/surface/init_unsecure.c b/src/core/lib/surface/init_unsecure.c
index 630d564a7d..243c005d86 100644
--- a/src/core/surface/init_unsecure.c
+++ b/src/core/lib/surface/init_unsecure.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
  *
  */
 
-#include "src/core/surface/init.h"
+#include "src/core/lib/surface/init.h"
 
 void grpc_security_pre_init(void) {}
+
+void grpc_register_security_filters(void) {}
diff --git a/src/core/surface/lame_client.c b/src/core/lib/surface/lame_client.c
index 537069e984..95ec4b06c3 100644
--- a/src/core/surface/lame_client.c
+++ b/src/core/lib/surface/lame_client.c
@@ -31,17 +31,19 @@
  *
  */
 
+#include "src/core/lib/surface/lame_client.h"
+
 #include <grpc/grpc.h>
 
 #include <string.h>
 
-#include "src/core/channel/channel_stack.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/channel.h"
-#include "src/core/surface/call.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/channel.h"
 
 typedef struct {
   grpc_linked_mdelem status;
@@ -115,11 +117,18 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {}
 
-static const grpc_channel_filter lame_filter = {
-    lame_start_transport_stream_op, lame_start_transport_op, sizeof(call_data),
-    init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem,
-    sizeof(channel_data), init_channel_elem, destroy_channel_elem,
-    lame_get_peer, "lame-client",
+const grpc_channel_filter grpc_lame_filter = {
+    lame_start_transport_stream_op,
+    lame_start_transport_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    lame_get_peer,
+    "lame-client",
 };
 
 #define CHANNEL_STACK_FROM_CHANNEL(c) ((grpc_channel_stack *)((c) + 1))
@@ -127,19 +136,17 @@ static const grpc_channel_filter lame_filter = {
 grpc_channel *grpc_lame_client_channel_create(const char *target,
                                               grpc_status_code error_code,
                                               const char *error_message) {
-  grpc_channel *channel;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_channel_element *elem;
   channel_data *chand;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  static const grpc_channel_filter *filters[] = {&lame_filter};
-  channel =
-      grpc_channel_create_from_filters(&exec_ctx, target, filters, 1, NULL, 1);
+  grpc_channel *channel = grpc_channel_create(&exec_ctx, target, NULL,
+                                              GRPC_CLIENT_LAME_CHANNEL, NULL);
   elem = grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0);
   GRPC_API_TRACE(
       "grpc_lame_client_channel_create(target=%s, error_code=%d, "
       "error_message=%s)",
       3, (target, (int)error_code, error_message));
-  GPR_ASSERT(elem->filter == &lame_filter);
+  GPR_ASSERT(elem->filter == &grpc_lame_filter);
   chand = (channel_data *)elem->channel_data;
   chand->error_code = error_code;
   chand->error_message = error_message;
diff --git a/src/cpp/codegen/grpc_library.cc b/src/core/lib/surface/lame_client.h
index 48acec3f3d..5f6ea34d4b 100644
--- a/src/cpp/codegen/grpc_library.cc
+++ b/src/core/lib/surface/lame_client.h
@@ -31,10 +31,11 @@
  *
  */
 
-#include <grpc++/impl/codegen/grpc_library.h>
+#ifndef GRPC_CORE_LIB_SURFACE_LAME_CLIENT_H
+#define GRPC_CORE_LIB_SURFACE_LAME_CLIENT_H
 
-namespace grpc {
+#include "src/core/lib/channel/channel_stack.h"
 
-GrpcLibraryInterface *g_glip = nullptr;
+extern const grpc_channel_filter grpc_lame_filter;
 
-}  // namespace grpc
+#endif /* GRPC_CORE_LIB_SURFACE_LAME_CLIENT_H */
diff --git a/src/core/surface/metadata_array.c b/src/core/lib/surface/metadata_array.c
index 4c7bf17835..4436f2da87 100644
--- a/src/core/surface/metadata_array.c
+++ b/src/core/lib/surface/metadata_array.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,7 +36,7 @@
 
 #include <string.h>
 
-#include "src/core/surface/api_trace.h"
+#include "src/core/lib/surface/api_trace.h"
 
 void grpc_metadata_array_init(grpc_metadata_array* array) {
   GRPC_API_TRACE("grpc_metadata_array_init(array=%p)", 1, (array));
diff --git a/src/core/surface/server.c b/src/core/lib/surface/server.c
index 5b13d4ba52..55e5e49e11 100644
--- a/src/core/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/surface/server.h"
+#include "src/core/lib/surface/server.h"
 
 #include <limits.h>
 #include <stdlib.h>
@@ -42,19 +42,18 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/census/grpc_filter.h"
-#include "src/core/channel/channel_args.h"
-#include "src/core/channel/connected_channel.h"
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/support/stack_lockfree.h"
-#include "src/core/support/string.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
-#include "src/core/surface/channel.h"
-#include "src/core/surface/completion_queue.h"
-#include "src/core/surface/init.h"
-#include "src/core/transport/metadata.h"
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/support/stack_lockfree.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/init.h"
+#include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 typedef struct listener {
   void *arg;
@@ -101,6 +100,7 @@ typedef struct requested_call {
 
 typedef struct channel_registered_method {
   registered_method *server_registered_method;
+  uint32_t flags;
   grpc_mdstr *method;
   grpc_mdstr *host;
 } channel_registered_method;
@@ -153,6 +153,7 @@ struct call_data {
   grpc_completion_queue *cq_new;
 
   grpc_metadata_batch *recv_initial_metadata;
+  bool recv_idempotent_request;
   grpc_metadata_array initial_metadata;
 
   grpc_closure got_initial_metadata;
@@ -172,6 +173,7 @@ struct request_matcher {
 struct registered_method {
   char *method;
   char *host;
+  uint32_t flags;
   request_matcher request_matcher;
   registered_method *next;
 };
@@ -182,8 +184,6 @@ typedef struct {
 } channel_broadcaster;
 
 struct grpc_server {
-  size_t channel_filter_count;
-  grpc_channel_filter const **channel_filters;
   grpc_channel_args *channel_args;
 
   grpc_completion_queue **cqs;
@@ -355,7 +355,6 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
   grpc_channel_args_destroy(server->channel_args);
   gpr_mu_destroy(&server->mu_global);
   gpr_mu_destroy(&server->mu_call);
-  gpr_free((void *)server->channel_filters);
   while ((rm = server->registered_methods) != NULL) {
     server->registered_methods = rm->next;
     request_matcher_destroy(&rm->request_matcher);
@@ -472,6 +471,9 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
       if (!rm) break;
       if (rm->host != calld->host) continue;
       if (rm->method != calld->path) continue;
+      if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) &&
+          !calld->recv_idempotent_request)
+        continue;
       finish_start_new_rpc(exec_ctx, server, elem,
                            &rm->server_registered_method->request_matcher);
       return;
@@ -484,6 +486,9 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
       if (!rm) break;
       if (rm->host != NULL) continue;
       if (rm->method != calld->path) continue;
+      if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) &&
+          !calld->recv_idempotent_request)
+        continue;
       finish_start_new_rpc(exec_ctx, server, elem,
                            &rm->server_registered_method->request_matcher);
       return;
@@ -602,9 +607,11 @@ static void server_mutate_op(grpc_call_element *elem,
   call_data *calld = elem->call_data;
 
   if (op->recv_initial_metadata != NULL) {
+    GPR_ASSERT(op->recv_idempotent_request == NULL);
     calld->recv_initial_metadata = op->recv_initial_metadata;
     calld->on_done_recv_initial_metadata = op->recv_initial_metadata_ready;
     op->recv_initial_metadata_ready = &calld->server_on_recv_initial_metadata;
+    op->recv_idempotent_request = &calld->recv_idempotent_request;
   }
 }
 
@@ -757,11 +764,18 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static const grpc_channel_filter server_surface_filter = {
-    server_start_transport_stream_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem,
-    sizeof(channel_data), init_channel_elem, destroy_channel_elem,
-    grpc_call_next_get_peer, "server",
+const grpc_channel_filter grpc_server_top_filter = {
+    server_start_transport_stream_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "server",
 };
 
 void grpc_server_register_completion_queue(grpc_server *server,
@@ -783,11 +797,10 @@ void grpc_server_register_completion_queue(grpc_server *server,
   server->cqs[n] = cq;
 }
 
-grpc_server *grpc_server_create_from_filters(
-    const grpc_channel_filter **filters, size_t filter_count,
-    const grpc_channel_args *args) {
+grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   size_t i;
-  int census_enabled = grpc_channel_args_is_census_enabled(args);
+
+  GRPC_API_TRACE("grpc_server_create(%p, %p)", 2, (args, reserved));
 
   grpc_server *server = gpr_malloc(sizeof(grpc_server));
 
@@ -815,23 +828,6 @@ grpc_server *grpc_server_create_from_filters(
   server->requested_calls = gpr_malloc(server->max_requested_calls *
                                        sizeof(*server->requested_calls));
 
-  /* Server filter stack is:
-
-     server_surface_filter - for making surface API calls
-     grpc_server_census_filter (optional) - for stats collection and tracing
-     {passed in filter stack}
-     grpc_connected_channel_filter - for interfacing with transports */
-  server->channel_filter_count = filter_count + 1u + (census_enabled ? 1u : 0u);
-  server->channel_filters =
-      gpr_malloc(server->channel_filter_count * sizeof(grpc_channel_filter *));
-  server->channel_filters[0] = &server_surface_filter;
-  if (census_enabled) {
-    server->channel_filters[1] = &grpc_server_census_filter;
-  }
-  for (i = 0; i < filter_count; i++) {
-    server->channel_filters[i + 1u + (census_enabled ? 1u : 0u)] = filters[i];
-  }
-
   server->channel_args = grpc_channel_args_copy(args);
 
   return server;
@@ -845,10 +841,12 @@ static int streq(const char *a, const char *b) {
 }
 
 void *grpc_server_register_method(grpc_server *server, const char *method,
-                                  const char *host) {
+                                  const char *host, uint32_t flags) {
   registered_method *m;
-  GRPC_API_TRACE("grpc_server_register_method(server=%p, method=%s, host=%s)",
-                 3, (server, method, host));
+  GRPC_API_TRACE(
+      "grpc_server_register_method(server=%p, method=%s, host=%s, "
+      "flags=0x%08x)",
+      4, (server, method, host, flags));
   if (!method) {
     gpr_log(GPR_ERROR,
             "grpc_server_register_method method string cannot be NULL");
@@ -861,12 +859,18 @@ void *grpc_server_register_method(grpc_server *server, const char *method,
       return NULL;
     }
   }
+  if ((flags & ~GRPC_INITIAL_METADATA_USED_MASK) != 0) {
+    gpr_log(GPR_ERROR, "grpc_server_register_method invalid flags 0x%08x",
+            flags);
+    return NULL;
+  }
   m = gpr_malloc(sizeof(registered_method));
   memset(m, 0, sizeof(*m));
   request_matcher_init(&m->request_matcher, server->max_requested_calls);
   m->method = gpr_strdup(method);
   m->host = gpr_strdup(host);
   m->next = server->registered_methods;
+  m->flags = flags;
   server->registered_methods = m;
   return m;
 }
@@ -892,12 +896,7 @@ void grpc_server_start(grpc_server *server) {
 
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
                                  grpc_transport *transport,
-                                 grpc_channel_filter const **extra_filters,
-                                 size_t num_extra_filters,
                                  const grpc_channel_args *args) {
-  size_t num_filters = s->channel_filter_count + num_extra_filters + 1;
-  grpc_channel_filter const **filters =
-      gpr_malloc(sizeof(grpc_channel_filter *) * num_filters);
   size_t i;
   size_t num_registered_methods;
   size_t alloc;
@@ -913,24 +912,17 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   uint32_t max_probes = 0;
   grpc_transport_op op;
 
-  for (i = 0; i < s->channel_filter_count; i++) {
-    filters[i] = s->channel_filters[i];
-  }
-  for (; i < s->channel_filter_count + num_extra_filters; i++) {
-    filters[i] = extra_filters[i - s->channel_filter_count];
-  }
-  filters[i] = &grpc_connected_channel_filter;
-
   for (i = 0; i < s->cq_count; i++) {
     memset(&op, 0, sizeof(op));
     op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
     grpc_transport_perform_op(exec_ctx, transport, &op);
   }
 
-  channel = grpc_channel_create_from_filters(exec_ctx, NULL, filters,
-                                             num_filters, args, 0);
+  channel =
+      grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
   chand = (channel_data *)grpc_channel_stack_element(
-              grpc_channel_get_channel_stack(channel), 0)->channel_data;
+              grpc_channel_get_channel_stack(channel), 0)
+              ->channel_data;
   chand->server = s;
   server_ref(s);
   chand->channel = channel;
@@ -951,12 +943,13 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
       method = grpc_mdstr_from_string(rm->method);
       hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
       for (probes = 0; chand->registered_methods[(hash + probes) % slots]
-                               .server_registered_method != NULL;
+                           .server_registered_method != NULL;
            probes++)
         ;
       if (probes > max_probes) max_probes = probes;
       crm = &chand->registered_methods[(hash + probes) % slots];
       crm->server_registered_method = rm;
+      crm->flags = rm->flags;
       crm->host = host;
       crm->method = method;
     }
@@ -965,17 +958,12 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
     chand->registered_method_max_probes = max_probes;
   }
 
-  grpc_connected_channel_bind_transport(grpc_channel_get_channel_stack(channel),
-                                        transport);
-
   gpr_mu_lock(&s->mu_global);
   chand->next = &s->root_channel_data;
   chand->prev = chand->next->prev;
   chand->next->prev = chand->prev->next = chand;
   gpr_mu_unlock(&s->mu_global);
 
-  gpr_free((void *)filters);
-
   GRPC_CHANNEL_INTERNAL_REF(channel, "connectivity");
   memset(&op, 0, sizeof(op));
   op.set_accept_stream = true;
@@ -1279,6 +1267,10 @@ static void begin_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
       cpstr(&rc->data.batch.details->method,
             &rc->data.batch.details->method_capacity, calld->path);
       rc->data.batch.details->deadline = calld->deadline;
+      rc->data.batch.details->flags =
+          0 | (calld->recv_idempotent_request
+                   ? GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST
+                   : 0);
       break;
     case REGISTERED_CALL:
       *rc->data.registered.deadline = calld->deadline;
diff --git a/src/core/surface/server.h b/src/core/lib/surface/server.h
index a957fdb360..3845eb2981 100644
--- a/src/core/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,17 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_SERVER_H
-#define GRPC_INTERNAL_CORE_SURFACE_SERVER_H
+#ifndef GRPC_CORE_LIB_SURFACE_SERVER_H
+#define GRPC_CORE_LIB_SURFACE_SERVER_H
 
-#include "src/core/channel/channel_stack.h"
 #include <grpc/grpc.h>
-#include "src/core/transport/transport.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/transport/transport.h"
 
-/* Create a server */
-grpc_server *grpc_server_create_from_filters(
-    const grpc_channel_filter **filters, size_t filter_count,
-    const grpc_channel_args *args);
+extern const grpc_channel_filter grpc_server_top_filter;
 
 /* Add a listener to the server: when the server starts, it will call start,
    and when it shuts down, it will call destroy */
@@ -56,12 +53,10 @@ void grpc_server_add_listener(
    server */
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *server,
                                  grpc_transport *transport,
-                                 grpc_channel_filter const **extra_filters,
-                                 size_t num_extra_filters,
                                  const grpc_channel_args *args);
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
 
 int grpc_server_has_open_connections(grpc_server *server);
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_SERVER_H */
+#endif /* GRPC_CORE_LIB_SURFACE_SERVER_H */
diff --git a/src/core/surface/surface_trace.h b/src/core/lib/surface/surface_trace.h
index 93b2859ac5..6b3f673924 100644
--- a/src/core/surface/surface_trace.h
+++ b/src/core/lib/surface/surface_trace.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SURFACE_SURFACE_TRACE_H
-#define GRPC_INTERNAL_CORE_SURFACE_SURFACE_TRACE_H
+#ifndef GRPC_CORE_LIB_SURFACE_SURFACE_TRACE_H
+#define GRPC_CORE_LIB_SURFACE_SURFACE_TRACE_H
 
-#include "src/core/debug/trace.h"
-#include "src/core/surface/api_trace.h"
 #include <grpc/support/log.h>
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/surface/api_trace.h"
 
 #define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)    \
   if (grpc_api_trace) {                                 \
@@ -45,4 +45,4 @@
     gpr_free(_ev);                                      \
   }
 
-#endif /* GRPC_INTERNAL_CORE_SURFACE_SURFACE_TRACE_H */
+#endif /* GRPC_CORE_LIB_SURFACE_SURFACE_TRACE_H */
diff --git a/src/core/surface/validate_metadata.c b/src/core/lib/surface/validate_metadata.c
index bf4126867f..bf4126867f 100644
--- a/src/core/surface/validate_metadata.c
+++ b/src/core/lib/surface/validate_metadata.c
diff --git a/src/core/surface/version.c b/src/core/lib/surface/version.c
index 7723f39401..7723f39401 100644
--- a/src/core/surface/version.c
+++ b/src/core/lib/surface/version.c
diff --git a/src/core/transport/byte_stream.c b/src/core/lib/transport/byte_stream.c
index 8e6fb2cbef..79981aa154 100644
--- a/src/core/transport/byte_stream.c
+++ b/src/core/lib/transport/byte_stream.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/byte_stream.h"
+#include "src/core/lib/transport/byte_stream.h"
 
 #include <stdlib.h>
 
diff --git a/src/core/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h
index d2e51e7b45..e7346dafc3 100644
--- a/src/core/transport/byte_stream.h
+++ b/src/core/lib/transport/byte_stream.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_BYTE_STREAM_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_BYTE_STREAM_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H
+#define GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H
 
-#include "src/core/iomgr/exec_ctx.h"
 #include <grpc/support/slice_buffer.h>
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 /** Internal bit flag for grpc_begin_message's \a flags signaling the use of
  * compression for the message */
@@ -86,4 +86,4 @@ void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream,
                                    gpr_slice_buffer *slice_buffer,
                                    uint32_t flags);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_BYTE_STREAM_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H */
diff --git a/src/core/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c
index 87765b9799..123eab8b36 100644
--- a/src/core/transport/connectivity_state.c
+++ b/src/core/lib/transport/connectivity_state.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/connectivity_state.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 #include <string.h>
 
diff --git a/src/core/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h
index a4eb6652e5..6f92132438 100644
--- a/src/core/transport/connectivity_state.h
+++ b/src/core/lib/transport/connectivity_state.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CONNECTIVITY_STATE_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_CONNECTIVITY_STATE_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H
+#define GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H
 
 #include <grpc/grpc.h>
-#include "src/core/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef struct grpc_connectivity_state_watcher {
   /** we keep watchers in a linked list */
@@ -82,4 +82,4 @@ int grpc_connectivity_state_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
     grpc_connectivity_state *current, grpc_closure *notify);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CONNECTIVITY_STATE_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H */
diff --git a/src/core/transport/metadata.c b/src/core/lib/transport/metadata.c
index 807ae071a3..451c8d1cd3 100644
--- a/src/core/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/transport/metadata.h"
+#include "src/core/lib/transport/metadata.h"
 
 #include <assert.h>
 #include <stddef.h>
@@ -44,12 +44,12 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
-#include "src/core/profiling/timers.h"
-#include "src/core/support/murmur_hash.h"
-#include "src/core/support/string.h"
-#include "src/core/transport/chttp2/bin_encoder.h"
-#include "src/core/transport/static_metadata.h"
-#include "src/core/iomgr/iomgr_internal.h"
+#include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/murmur_hash.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 /* There are two kinds of mdelem and mdstr instances.
  * Static instances are declared in static_metadata.{h,c} and
diff --git a/src/core/transport/metadata.h b/src/core/lib/transport/metadata.h
index 8742846be7..d72ec9accc 100644
--- a/src/core/transport/metadata.h
+++ b/src/core/lib/transport/metadata.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_METADATA_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_METADATA_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_METADATA_H
+#define GRPC_CORE_LIB_TRANSPORT_METADATA_H
 
 #include <grpc/support/slice.h>
 #include <grpc/support/useful.h>
@@ -153,4 +153,4 @@ int grpc_mdstr_is_bin_suffixed(grpc_mdstr *s);
 void grpc_mdctx_global_init(void);
 void grpc_mdctx_global_shutdown(void);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_METADATA_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_METADATA_H */
diff --git a/src/core/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c
index 1266862f82..bb79b8fa96 100644
--- a/src/core/transport/metadata_batch.c
+++ b/src/core/lib/transport/metadata_batch.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/transport/metadata_batch.h"
+#include "src/core/lib/transport/metadata_batch.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
 static void assert_valid_list(grpc_mdelem_list *list) {
 #ifndef NDEBUG
diff --git a/src/core/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h
index 1b0d1fda3e..f1d4726989 100644
--- a/src/core/transport/metadata_batch.h
+++ b/src/core/lib/transport/metadata_batch.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_STREAM_OP_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_STREAM_OP_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_METADATA_BATCH_H
+#define GRPC_CORE_LIB_TRANSPORT_METADATA_BATCH_H
 
 #include <grpc/grpc.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/time.h>
-#include "src/core/transport/metadata.h"
+#include "src/core/lib/transport/metadata.h"
 
 typedef struct grpc_linked_mdelem {
   grpc_mdelem *md;
@@ -122,4 +122,4 @@ void grpc_metadata_batch_assert_ok(grpc_metadata_batch *comd);
   } while (0)
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_STREAM_OP_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_METADATA_BATCH_H */
diff --git a/src/core/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c
index eeedae0619..bde1fafc88 100644
--- a/src/core/transport/static_metadata.c
+++ b/src/core/lib/transport/static_metadata.c
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
@@ -28,31 +27,28 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
  */
 
 /*
  * WARNING: Auto-generated code.
  *
  * To make changes to this file, change
- * tools/codegen/core/gen_static_metadata.py,
- * and then re-run it.
+ * tools/codegen/core/gen_static_metadata.py, and then re-run it.
  *
  * See metadata.h for an explanation of the interface here, and metadata.c for
- * an
- * explanation of what's going on.
+ * an explanation of what's going on.
  */
 
-#include "src/core/transport/static_metadata.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 
 grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 3, 7, 5, 2, 4, 8, 6, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 4, 8, 6, 2, 4, 8, 6, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
     {11, 35, 10, 35, 12, 35, 12, 49, 13, 35, 14, 35, 15, 35, 16, 35, 17, 35,
@@ -60,30 +56,102 @@ const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
      30, 18, 30, 35, 31, 35, 32, 35, 36, 35, 37, 35, 38, 35, 39, 35, 42, 33,
      42, 34, 42, 48, 42, 53, 42, 54, 42, 55, 42, 56, 43, 33, 43, 48, 43, 53,
      46, 0,  46, 1,  46, 2,  50, 35, 57, 35, 58, 35, 59, 35, 60, 35, 61, 35,
-     62, 35, 63, 35, 64, 35, 65, 35, 66, 40, 66, 68, 67, 78, 67, 79, 69, 35,
-     70, 35, 71, 35, 72, 35, 73, 35, 74, 35, 75, 41, 75, 51, 75, 52, 76, 35,
-     77, 35, 80, 3,  80, 4,  80, 5,  80, 6,  80, 7,  80, 8,  80, 9,  81, 35,
-     82, 83, 84, 35, 85, 35, 86, 35, 87, 35, 88, 35};
+     62, 35, 63, 35, 64, 35, 65, 35, 66, 40, 66, 68, 66, 71, 67, 79, 67, 80,
+     69, 35, 70, 35, 72, 35, 73, 35, 74, 35, 75, 35, 76, 41, 76, 51, 76, 52,
+     77, 35, 78, 35, 81, 3,  81, 4,  81, 5,  81, 6,  81, 7,  81, 8,  81, 9,
+     82, 35, 83, 84, 85, 35, 86, 35, 87, 35, 88, 35, 89, 35};
 
 const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
-    "0", "1", "2", "200", "204", "206", "304", "400", "404", "500", "accept",
-    "accept-charset", "accept-encoding", "accept-language", "accept-ranges",
-    "access-control-allow-origin", "age", "allow", "application/grpc",
-    ":authority", "authorization", "cache-control", "census-bin",
-    "census-binary-bin", "content-disposition", "content-encoding",
-    "content-language", "content-length", "content-location", "content-range",
-    "content-type", "cookie", "date", "deflate", "deflate,gzip", "", "etag",
-    "expect", "expires", "from", "GET", "grpc", "grpc-accept-encoding",
-    "grpc-encoding", "grpc-internal-encoding-request", "grpc-message",
-    "grpc-status", "grpc-timeout", "gzip", "gzip, deflate", "host", "http",
-    "https", "identity", "identity,deflate", "identity,deflate,gzip",
-    "identity,gzip", "if-match", "if-modified-since", "if-none-match",
-    "if-range", "if-unmodified-since", "last-modified", "link", "location",
-    "max-forwards", ":method", ":path", "POST", "proxy-authenticate",
-    "proxy-authorization", "range", "referer", "refresh", "retry-after",
-    ":scheme", "server", "set-cookie", "/", "/index.html", ":status",
-    "strict-transport-security", "te", "trailers", "transfer-encoding",
-    "user-agent", "vary", "via", "www-authenticate"};
+    "0",
+    "1",
+    "2",
+    "200",
+    "204",
+    "206",
+    "304",
+    "400",
+    "404",
+    "500",
+    "accept",
+    "accept-charset",
+    "accept-encoding",
+    "accept-language",
+    "accept-ranges",
+    "access-control-allow-origin",
+    "age",
+    "allow",
+    "application/grpc",
+    ":authority",
+    "authorization",
+    "cache-control",
+    "census-bin",
+    "census-binary-bin",
+    "content-disposition",
+    "content-encoding",
+    "content-language",
+    "content-length",
+    "content-location",
+    "content-range",
+    "content-type",
+    "cookie",
+    "date",
+    "deflate",
+    "deflate,gzip",
+    "",
+    "etag",
+    "expect",
+    "expires",
+    "from",
+    "GET",
+    "grpc",
+    "grpc-accept-encoding",
+    "grpc-encoding",
+    "grpc-internal-encoding-request",
+    "grpc-message",
+    "grpc-status",
+    "grpc-timeout",
+    "gzip",
+    "gzip, deflate",
+    "host",
+    "http",
+    "https",
+    "identity",
+    "identity,deflate",
+    "identity,deflate,gzip",
+    "identity,gzip",
+    "if-match",
+    "if-modified-since",
+    "if-none-match",
+    "if-range",
+    "if-unmodified-since",
+    "last-modified",
+    "link",
+    "location",
+    "max-forwards",
+    ":method",
+    ":path",
+    "POST",
+    "proxy-authenticate",
+    "proxy-authorization",
+    "PUT",
+    "range",
+    "referer",
+    "refresh",
+    "retry-after",
+    ":scheme",
+    "server",
+    "set-cookie",
+    "/",
+    "/index.html",
+    ":status",
+    "strict-transport-security",
+    "te",
+    "trailers",
+    "transfer-encoding",
+    "user-agent",
+    "vary",
+    "via",
+    "www-authenticate"};
 
 const uint8_t grpc_static_accept_encoding_metadata[8] = {0,  29, 26, 30,
                                                          28, 32, 27, 31};
diff --git a/src/core/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h
index ef72b802b5..a05553a870 100644
--- a/src/core/transport/static_metadata.h
+++ b/src/core/lib/transport/static_metadata.h
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
@@ -28,27 +27,24 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
  */
 
 /*
  * WARNING: Auto-generated code.
  *
  * To make changes to this file, change
- * tools/codegen/core/gen_static_metadata.py,
- * and then re-run it.
+ * tools/codegen/core/gen_static_metadata.py, and then re-run it.
  *
  * See metadata.h for an explanation of the interface here, and metadata.c for
- * an
- * explanation of what's going on.
+ * an explanation of what's going on.
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_STATIC_METADATA_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_STATIC_METADATA_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H
+#define GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H
 
-#include "src/core/transport/metadata.h"
+#include "src/core/lib/transport/metadata.h"
 
-#define GRPC_STATIC_MDSTR_COUNT 89
+#define GRPC_STATIC_MDSTR_COUNT 90
 extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 /* "0" */
 #define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0])
@@ -193,44 +189,46 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[69])
 /* "proxy-authorization" */
 #define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[70])
+/* "PUT" */
+#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[71])
 /* "range" */
-#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[71])
+#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[72])
 /* "referer" */
-#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[72])
+#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[73])
 /* "refresh" */
-#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[73])
+#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[74])
 /* "retry-after" */
-#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[74])
+#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[75])
 /* ":scheme" */
-#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[75])
+#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[76])
 /* "server" */
-#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[76])
+#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[77])
 /* "set-cookie" */
-#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[77])
+#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[78])
 /* "/" */
-#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[78])
+#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[79])
 /* "/index.html" */
-#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[79])
+#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[80])
 /* ":status" */
-#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[80])
+#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[81])
 /* "strict-transport-security" */
-#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[81])
+#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[82])
 /* "te" */
-#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[82])
+#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[83])
 /* "trailers" */
-#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[83])
+#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[84])
 /* "transfer-encoding" */
-#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[84])
+#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[85])
 /* "user-agent" */
-#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[85])
+#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[86])
 /* "vary" */
-#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[86])
+#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[87])
 /* "via" */
-#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[87])
+#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[88])
 /* "www-authenticate" */
-#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[88])
+#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[89])
 
-#define GRPC_STATIC_MDELEM_COUNT 78
+#define GRPC_STATIC_MDELEM_COUNT 79
 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 /* "accept-charset": "" */
@@ -343,61 +341,63 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 #define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[49])
 /* ":method": "POST" */
 #define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[50])
+/* ":method": "PUT" */
+#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[51])
 /* ":path": "/" */
-#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[51])
+#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[52])
 /* ":path": "/index.html" */
-#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[52])
+#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[53])
 /* "proxy-authenticate": "" */
-#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[53])
+#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[54])
 /* "proxy-authorization": "" */
-#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[54])
+#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[55])
 /* "range": "" */
-#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[55])
+#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[56])
 /* "referer": "" */
-#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[56])
+#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[57])
 /* "refresh": "" */
-#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[57])
+#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[58])
 /* "retry-after": "" */
-#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[58])
+#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[59])
 /* ":scheme": "grpc" */
-#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[59])
+#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[60])
 /* ":scheme": "http" */
-#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[60])
+#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[61])
 /* ":scheme": "https" */
-#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[61])
+#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[62])
 /* "server": "" */
-#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[62])
+#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[63])
 /* "set-cookie": "" */
-#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[63])
+#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[64])
 /* ":status": "200" */
-#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[64])
+#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[65])
 /* ":status": "204" */
-#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[65])
+#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[66])
 /* ":status": "206" */
-#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[66])
+#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[67])
 /* ":status": "304" */
-#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[67])
+#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[68])
 /* ":status": "400" */
-#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[68])
+#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[69])
 /* ":status": "404" */
-#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[69])
+#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[70])
 /* ":status": "500" */
-#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[70])
+#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[71])
 /* "strict-transport-security": "" */
 #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \
-  (&grpc_static_mdelem_table[71])
+  (&grpc_static_mdelem_table[72])
 /* "te": "trailers" */
-#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[72])
+#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[73])
 /* "transfer-encoding": "" */
-#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[73])
+#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[74])
 /* "user-agent": "" */
-#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[74])
+#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[75])
 /* "vary": "" */
-#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[75])
+#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[76])
 /* "via": "" */
-#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[76])
+#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[77])
 /* "www-authenticate": "" */
-#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[77])
+#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[78])
 
 extern const uint8_t
     grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2];
@@ -405,4 +405,4 @@ extern const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT];
 extern const uint8_t grpc_static_accept_encoding_metadata[8];
 #define GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS(algs) \
   (&grpc_static_mdelem_table[grpc_static_accept_encoding_metadata[(algs)]])
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_STATIC_METADATA_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H */
diff --git a/src/core/transport/transport.c b/src/core/lib/transport/transport.c
index 3b555fa933..411e4f140a 100644
--- a/src/core/transport/transport.c
+++ b/src/core/lib/transport/transport.c
@@ -31,11 +31,12 @@
  *
  */
 
-#include "src/core/transport/transport.h"
+#include "src/core/lib/transport/transport.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
-#include "src/core/transport/transport_impl.h"
+#include <grpc/support/sync.h>
+#include "src/core/lib/transport/transport_impl.h"
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason) {
@@ -76,6 +77,24 @@ void grpc_stream_ref_init(grpc_stream_refcount *refcount, int initial_refs,
   grpc_closure_init(&refcount->destroy, cb, cb_arg);
 }
 
+static void move64(uint64_t *from, uint64_t *to) {
+  *to += *from;
+  *from = 0;
+}
+
+void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats *from,
+                                       grpc_transport_one_way_stats *to) {
+  move64(&from->framing_bytes, &to->framing_bytes);
+  move64(&from->data_bytes, &to->data_bytes);
+  move64(&from->header_bytes, &to->header_bytes);
+}
+
+void grpc_transport_move_stats(grpc_transport_stream_stats *from,
+                               grpc_transport_stream_stats *to) {
+  grpc_transport_move_one_way_stats(&from->incoming, &to->incoming);
+  grpc_transport_move_one_way_stats(&from->outgoing, &to->outgoing);
+}
+
 size_t grpc_transport_stream_size(grpc_transport *transport) {
   return transport->vtable->sizeof_stream;
 }
diff --git a/src/core/transport/transport.h b/src/core/lib/transport/transport.h
index ed6e121c9c..6dd782c19e 100644
--- a/src/core/transport/transport.h
+++ b/src/core/lib/transport/transport.h
@@ -31,16 +31,16 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_TRANSPORT_H
+#define GRPC_CORE_LIB_TRANSPORT_TRANSPORT_H
 
 #include <stddef.h>
 
-#include "src/core/iomgr/pollset.h"
-#include "src/core/iomgr/pollset_set.h"
-#include "src/core/transport/metadata_batch.h"
-#include "src/core/transport/byte_stream.h"
-#include "src/core/channel/context.h"
+#include "src/core/lib/channel/context.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/transport/byte_stream.h"
+#include "src/core/lib/transport/metadata_batch.h"
 
 /* forward declarations */
 typedef struct grpc_transport grpc_transport;
@@ -78,11 +78,32 @@ void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount);
   grpc_stream_ref_init(rc, ir, cb, cb_arg)
 #endif
 
+typedef struct {
+  uint64_t framing_bytes;
+  uint64_t data_bytes;
+  uint64_t header_bytes;
+} grpc_transport_one_way_stats;
+
+typedef struct grpc_transport_stream_stats {
+  grpc_transport_one_way_stats incoming;
+  grpc_transport_one_way_stats outgoing;
+} grpc_transport_stream_stats;
+
+void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats *from,
+                                       grpc_transport_one_way_stats *to);
+
+void grpc_transport_move_stats(grpc_transport_stream_stats *from,
+                               grpc_transport_stream_stats *to);
+
 /* Transport stream op: a set of operations to perform on a transport
    against a single stream */
 typedef struct grpc_transport_stream_op {
-  /** Send initial metadata to the peer, from the provided metadata batch. */
+  /** Send initial metadata to the peer, from the provided metadata batch.
+      idempotent_request MUST be set if this is non-null */
   grpc_metadata_batch *send_initial_metadata;
+  /** Iff send_initial_metadata != NULL, flags if this is an idempotent request
+      or not */
+  bool send_idempotent_request;
 
   /** Send trailing metadata to the peer, from the provided metadata batch. */
   grpc_metadata_batch *send_trailing_metadata;
@@ -92,6 +113,7 @@ typedef struct grpc_transport_stream_op {
 
   /** Receive initial metadata from the stream, into provided metadata batch. */
   grpc_metadata_batch *recv_initial_metadata;
+  bool *recv_idempotent_request;
   /** Should be enqueued when initial metadata is ready to be processed. */
   grpc_closure *recv_initial_metadata_ready;
 
@@ -104,6 +126,9 @@ typedef struct grpc_transport_stream_op {
    */
   grpc_metadata_batch *recv_trailing_metadata;
 
+  /** Collect any stats into provided buffer, zero internal stat counters */
+  grpc_transport_stream_stats *collect_stats;
+
   /** Should be enqueued when all requested operations (excluding recv_message
       and recv_initial_metadata which have their own closures) in a given batch
       have been completed. */
@@ -239,4 +264,4 @@ void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, grpc_transport *transport);
 char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
                               grpc_transport *transport);
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_TRANSPORT_H */
diff --git a/src/core/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h
index 40bfb4b13a..92fa5d519d 100644
--- a/src/core/transport/transport_impl.h
+++ b/src/core/lib/transport/transport_impl.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,19 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_IMPL_H
-#define GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_IMPL_H
+#ifndef GRPC_CORE_LIB_TRANSPORT_TRANSPORT_IMPL_H
+#define GRPC_CORE_LIB_TRANSPORT_TRANSPORT_IMPL_H
 
-#include "src/core/transport/transport.h"
+#include "src/core/lib/transport/transport.h"
 
 typedef struct grpc_transport_vtable {
   /* Memory required for a single stream element - this is allocated by upper
      layers and initialized by the transport */
   size_t sizeof_stream; /* = sizeof(transport stream) */
 
+  /* name of this transport implementation */
+  const char *name;
+
   /* implementation of grpc_transport_init_stream */
   int (*init_stream)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                      grpc_stream *stream, grpc_stream_refcount *refcount,
@@ -75,4 +78,4 @@ struct grpc_transport {
   const grpc_transport_vtable *vtable;
 };
 
-#endif /* GRPC_INTERNAL_CORE_TRANSPORT_TRANSPORT_IMPL_H */
+#endif /* GRPC_CORE_LIB_TRANSPORT_TRANSPORT_IMPL_H */
diff --git a/src/core/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c
index 98b51afc88..1fa8fa5d4f 100644
--- a/src/core/transport/transport_op_string.c
+++ b/src/core/lib/transport/transport_op_string.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,16 @@
  *
  */
 
-#include "src/core/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack.h"
 
 #include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/support/string.h"
 
 /* These routines are here to facilitate debugging - they produce string
    representations of various transport data structures */
diff --git a/src/core/tsi/fake_transport_security.c b/src/core/lib/tsi/fake_transport_security.c
index 72ac32a171..4b812f4803 100644
--- a/src/core/tsi/fake_transport_security.c
+++ b/src/core/lib/tsi/fake_transport_security.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/tsi/fake_transport_security.h"
+#include "src/core/lib/tsi/fake_transport_security.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -39,7 +39,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/useful.h>
-#include "src/core/tsi/transport_security.h"
+#include "src/core/lib/tsi/transport_security.h"
 
 /* --- Constants. ---*/
 #define TSI_FAKE_FRAME_HEADER_SIZE 4
@@ -493,8 +493,10 @@ static void fake_handshaker_destroy(tsi_handshaker *self) {
 
 static const tsi_handshaker_vtable handshaker_vtable = {
     fake_handshaker_get_bytes_to_send_to_peer,
-    fake_handshaker_process_bytes_from_peer, fake_handshaker_get_result,
-    fake_handshaker_extract_peer, fake_handshaker_create_frame_protector,
+    fake_handshaker_process_bytes_from_peer,
+    fake_handshaker_get_result,
+    fake_handshaker_extract_peer,
+    fake_handshaker_create_frame_protector,
     fake_handshaker_destroy,
 };
 
diff --git a/src/core/tsi/fake_transport_security.h b/src/core/lib/tsi/fake_transport_security.h
index fe295aa536..b887dfcb09 100644
--- a/src/core/tsi/fake_transport_security.h
+++ b/src/core/lib/tsi/fake_transport_security.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TSI_FAKE_TRANSPORT_SECURITY_H
-#define GRPC_INTERNAL_CORE_TSI_FAKE_TRANSPORT_SECURITY_H
+#ifndef GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H
+#define GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H
 
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -58,4 +58,4 @@ tsi_frame_protector *tsi_create_fake_protector(
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_TSI_FAKE_TRANSPORT_SECURITY_H */
+#endif /* GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H */
diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/lib/tsi/ssl_transport_security.c
index 6adcaac9ed..d03201eec6 100644
--- a/src/core/tsi/ssl_transport_security.c
+++ b/src/core/lib/tsi/ssl_transport_security.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,20 @@
  *
  */
 
-#include "src/core/tsi/ssl_transport_security.h"
+#include "src/core/lib/tsi/ssl_transport_security.h"
+
+#include <grpc/support/port_platform.h>
 
 #include <limits.h>
 #include <string.h>
 
+/* TODO(jboeuf): refactor inet_ntop into a portability header. */
+#ifdef GPR_WINSOCK_SOCKET
+#include <ws2tcpip.h>
+#else
+#include <arpa/inet.h>
+#endif
+
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/thd.h>
@@ -48,8 +57,8 @@
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
 
-#include "src/core/tsi/ssl_types.h"
-#include "src/core/tsi/transport_security.h"
+#include "src/core/lib/tsi/ssl_types.h"
+#include "src/core/lib/tsi/transport_security.h"
 
 /* --- Constants. ---*/
 
@@ -197,13 +206,16 @@ static void ssl_info_callback(const SSL *ssl, int where, int ret) {
 }
 
 /* Returns 1 if name looks like an IP address, 0 otherwise.
-   This is a very rough heuristic as it does not handle IPV6 or things like:
-   0300.0250.00.01, 0xC0.0Xa8.0x0.0x1, 000030052000001, 0xc0.052000001 */
+   This is a very rough heuristic, and only handles IPv6 in hexadecimal form. */
 static int looks_like_ip_address(const char *name) {
   size_t i;
   size_t dot_count = 0;
   size_t num_size = 0;
   for (i = 0; i < strlen(name); i++) {
+    if (name[i] == ':') {
+      /* IPv6 Address in hexadecimal form, : is not allowed in DNS names. */
+      return 1;
+    }
     if (name[i] >= '0' && name[i] <= '9') {
       if (num_size > 3) return 0;
       num_size++;
@@ -281,6 +293,26 @@ static tsi_result peer_property_from_x509_common_name(
   return result;
 }
 
+/* Gets the X509 cert in PEM format as a tsi_peer_property. */
+static tsi_result add_pem_certificate(X509 *cert, tsi_peer_property *property) {
+  BIO *bio = BIO_new(BIO_s_mem());
+  if (!PEM_write_bio_X509(bio, cert)) {
+    BIO_free(bio);
+    return TSI_INTERNAL_ERROR;
+  }
+  char *contents;
+  long len = BIO_get_mem_data(bio, &contents);
+  if (len <= 0) {
+    BIO_free(bio);
+    return TSI_INTERNAL_ERROR;
+  }
+  tsi_result result = tsi_construct_string_peer_property(
+      TSI_X509_PEM_CERT_PROPERTY, (const char *)contents, (size_t)len,
+      property);
+  BIO_free(bio);
+  return result;
+}
+
 /* Gets the subject SANs from an X509 cert as a tsi_peer_property. */
 static tsi_result add_subject_alt_names_properties_to_peer(
     tsi_peer *peer, GENERAL_NAMES *subject_alt_names,
@@ -296,21 +328,44 @@ static tsi_result add_subject_alt_names_properties_to_peer(
         sk_GENERAL_NAME_value(subject_alt_names, TSI_SIZE_AS_SIZE(i));
     /* Filter out the non-dns entries names. */
     if (subject_alt_name->type == GEN_DNS) {
-      unsigned char *dns_name = NULL;
-      int dns_name_size =
-          ASN1_STRING_to_UTF8(&dns_name, subject_alt_name->d.dNSName);
-      if (dns_name_size < 0) {
+      unsigned char *name = NULL;
+      int name_size;
+      name_size = ASN1_STRING_to_UTF8(&name, subject_alt_name->d.dNSName);
+      if (name_size < 0) {
         gpr_log(GPR_ERROR, "Could not get utf8 from asn1 string.");
         result = TSI_INTERNAL_ERROR;
         break;
       }
       result = tsi_construct_string_peer_property(
-          TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY,
-          (const char *)dns_name, (size_t)dns_name_size,
+          TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, (const char *)name,
+          (size_t)name_size, &peer->properties[peer->property_count++]);
+      OPENSSL_free(name);
+    } else if (subject_alt_name->type == GEN_IPADD) {
+      char ntop_buf[INET6_ADDRSTRLEN];
+      int af;
+
+      if (subject_alt_name->d.iPAddress->length == 4) {
+        af = AF_INET;
+      } else if (subject_alt_name->d.iPAddress->length == 16) {
+        af = AF_INET6;
+      } else {
+        gpr_log(GPR_ERROR, "SAN IP Address contained invalid IP");
+        result = TSI_INTERNAL_ERROR;
+        break;
+      }
+      const char *name = inet_ntop(af, subject_alt_name->d.iPAddress->data,
+                                   ntop_buf, INET6_ADDRSTRLEN);
+      if (name == NULL) {
+        gpr_log(GPR_ERROR, "Could not get IP string from asn1 octet.");
+        result = TSI_INTERNAL_ERROR;
+        break;
+      }
+
+      result = tsi_construct_string_peer_property_from_cstring(
+          TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, name,
           &peer->properties[peer->property_count++]);
-      OPENSSL_free(dns_name);
-      if (result != TSI_OK) break;
     }
+    if (result != TSI_OK) break;
   }
   return result;
 }
@@ -328,7 +383,8 @@ static tsi_result peer_from_x509(X509 *cert, int include_certificate_type,
   tsi_result result;
   GPR_ASSERT(subject_alt_name_count >= 0);
   property_count = (include_certificate_type ? (size_t)1 : 0) +
-                   1 /* common name */ + (size_t)subject_alt_name_count;
+                   2 /* common name, certificate */ +
+                   (size_t)subject_alt_name_count;
   result = tsi_construct_peer(property_count, peer);
   if (result != TSI_OK) return result;
   do {
@@ -342,6 +398,10 @@ static tsi_result peer_from_x509(X509 *cert, int include_certificate_type,
         cert, &peer->properties[include_certificate_type ? 1 : 0]);
     if (result != TSI_OK) break;
 
+    result = add_pem_certificate(
+        cert, &peer->properties[include_certificate_type ? 2 : 1]);
+    if (result != TSI_OK) break;
+
     if (subject_alt_name_count != 0) {
       result = add_subject_alt_names_properties_to_peer(
           peer, subject_alt_names, (size_t)subject_alt_name_count);
@@ -979,8 +1039,10 @@ static void ssl_handshaker_destroy(tsi_handshaker *self) {
 
 static const tsi_handshaker_vtable handshaker_vtable = {
     ssl_handshaker_get_bytes_to_send_to_peer,
-    ssl_handshaker_process_bytes_from_peer, ssl_handshaker_get_result,
-    ssl_handshaker_extract_peer, ssl_handshaker_create_frame_protector,
+    ssl_handshaker_process_bytes_from_peer,
+    ssl_handshaker_get_result,
+    ssl_handshaker_extract_peer,
+    ssl_handshaker_create_frame_protector,
     ssl_handshaker_destroy,
 };
 
@@ -1436,9 +1498,7 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) {
   size_t i = 0;
   size_t san_count = 0;
   const tsi_peer_property *cn_property = NULL;
-
-  /* For now reject what looks like an IP address. */
-  if (looks_like_ip_address(name)) return 0;
+  int like_ip = looks_like_ip_address(name);
 
   /* Check the SAN first. */
   for (i = 0; i < peer->property_count; i++) {
@@ -1447,8 +1507,15 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) {
     if (strcmp(property->name,
                TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY) == 0) {
       san_count++;
-      if (does_entry_match_name(property->value.data, property->value.length,
-                                name)) {
+
+      if (!like_ip && does_entry_match_name(property->value.data,
+                                            property->value.length, name)) {
+        return 1;
+      } else if (like_ip &&
+                 strncmp(name, property->value.data, property->value.length) ==
+                     0 &&
+                 strlen(name) == property->value.length) {
+        /* IP Addresses are exact matches only. */
         return 1;
       }
     } else if (strcmp(property->name,
@@ -1457,8 +1524,8 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) {
     }
   }
 
-  /* If there's no SAN, try the CN. */
-  if (san_count == 0 && cn_property != NULL) {
+  /* If there's no SAN, try the CN, but only if its not like an IP Address */
+  if (san_count == 0 && cn_property != NULL && !like_ip) {
     if (does_entry_match_name(cn_property->value.data,
                               cn_property->value.length, name)) {
       return 1;
diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/lib/tsi/ssl_transport_security.h
index 51c0003a85..c9b9e8f54b 100644
--- a/src/core/tsi/ssl_transport_security.h
+++ b/src/core/lib/tsi/ssl_transport_security.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TSI_SSL_TRANSPORT_SECURITY_H
-#define GRPC_INTERNAL_CORE_TSI_SSL_TRANSPORT_SECURITY_H
+#ifndef GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H
+#define GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H
 
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -48,6 +48,8 @@ extern "C" {
 #define TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY \
   "x509_subject_alternative_name"
 
+#define TSI_X509_PEM_CERT_PROPERTY "x509_pem_cert"
+
 #define TSI_SSL_ALPN_SELECTED_PROTOCOL "ssl_alpn_selected_protocol"
 
 /* --- tsi_ssl_handshaker_factory object ---
@@ -162,12 +164,11 @@ void tsi_ssl_handshaker_factory_destroy(tsi_ssl_handshaker_factory *self);
    Still TODO(jboeuf):
    - handle mixed case.
    - handle %encoded chars.
-   - handle public suffix wildchar more strictly (e.g. *.co.uk)
-   - handle IP addresses in SAN. */
+   - handle public suffix wildchar more strictly (e.g. *.co.uk) */
 int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_TSI_SSL_TRANSPORT_SECURITY_H */
+#endif /* GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H */
diff --git a/src/core/tsi/ssl_types.h b/src/core/lib/tsi/ssl_types.h
index 0d0225e9c7..c6e68c01ad 100644
--- a/src/core/tsi/ssl_types.h
+++ b/src/core/lib/tsi/ssl_types.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TSI_SSL_TYPES_H
-#define GRPC_INTERNAL_CORE_TSI_SSL_TYPES_H
+#ifndef GRPC_CORE_LIB_TSI_SSL_TYPES_H
+#define GRPC_CORE_LIB_TSI_SSL_TYPES_H
 
 /* A collection of macros to cast between various integer types that are
  * used differently between BoringSSL and OpenSSL:
@@ -52,4 +52,4 @@
 #define TSI_SIZE_AS_SIZE(x) ((int)(x))
 #endif
 
-#endif
+#endif /* GRPC_CORE_LIB_TSI_SSL_TYPES_H */
diff --git a/src/core/tsi/test_creds/README b/src/core/lib/tsi/test_creds/README
index eb8482d648..eb8482d648 100644
--- a/src/core/tsi/test_creds/README
+++ b/src/core/lib/tsi/test_creds/README
diff --git a/src/core/tsi/test_creds/badclient.key b/src/core/lib/tsi/test_creds/badclient.key
index 5832685122..5832685122 100644
--- a/src/core/tsi/test_creds/badclient.key
+++ b/src/core/lib/tsi/test_creds/badclient.key
diff --git a/src/core/tsi/test_creds/badclient.pem b/src/core/lib/tsi/test_creds/badclient.pem
index 1785970221..1785970221 100644
--- a/src/core/tsi/test_creds/badclient.pem
+++ b/src/core/lib/tsi/test_creds/badclient.pem
diff --git a/src/core/tsi/test_creds/badserver.key b/src/core/lib/tsi/test_creds/badserver.key
index abfbde10ff..abfbde10ff 100644
--- a/src/core/tsi/test_creds/badserver.key
+++ b/src/core/lib/tsi/test_creds/badserver.key
diff --git a/src/core/tsi/test_creds/badserver.pem b/src/core/lib/tsi/test_creds/badserver.pem
index 983c979f31..983c979f31 100644
--- a/src/core/tsi/test_creds/badserver.pem
+++ b/src/core/lib/tsi/test_creds/badserver.pem
diff --git a/src/core/tsi/test_creds/ca-openssl.cnf b/src/core/lib/tsi/test_creds/ca-openssl.cnf
index e97b945e4b..e97b945e4b 100644
--- a/src/core/tsi/test_creds/ca-openssl.cnf
+++ b/src/core/lib/tsi/test_creds/ca-openssl.cnf
diff --git a/src/core/tsi/test_creds/ca.key b/src/core/lib/tsi/test_creds/ca.key
index 03c4f950e3..03c4f950e3 100644
--- a/src/core/tsi/test_creds/ca.key
+++ b/src/core/lib/tsi/test_creds/ca.key
diff --git a/src/core/tsi/test_creds/ca.pem b/src/core/lib/tsi/test_creds/ca.pem
index 6c8511a73c..6c8511a73c 100644
--- a/src/core/tsi/test_creds/ca.pem
+++ b/src/core/lib/tsi/test_creds/ca.pem
diff --git a/src/core/tsi/test_creds/client.key b/src/core/lib/tsi/test_creds/client.key
index f48d0735d9..f48d0735d9 100644
--- a/src/core/tsi/test_creds/client.key
+++ b/src/core/lib/tsi/test_creds/client.key
diff --git a/src/core/tsi/test_creds/client.pem b/src/core/lib/tsi/test_creds/client.pem
index e332091019..e332091019 100644
--- a/src/core/tsi/test_creds/client.pem
+++ b/src/core/lib/tsi/test_creds/client.pem
diff --git a/src/core/tsi/test_creds/server0.key b/src/core/lib/tsi/test_creds/server0.key
index add153c9ae..add153c9ae 100644
--- a/src/core/tsi/test_creds/server0.key
+++ b/src/core/lib/tsi/test_creds/server0.key
diff --git a/src/core/tsi/test_creds/server0.pem b/src/core/lib/tsi/test_creds/server0.pem
index ade75d8563..ade75d8563 100644
--- a/src/core/tsi/test_creds/server0.pem
+++ b/src/core/lib/tsi/test_creds/server0.pem
diff --git a/src/core/tsi/test_creds/server1-openssl.cnf b/src/core/lib/tsi/test_creds/server1-openssl.cnf
index 8a02108289..8a02108289 100644
--- a/src/core/tsi/test_creds/server1-openssl.cnf
+++ b/src/core/lib/tsi/test_creds/server1-openssl.cnf
diff --git a/src/core/tsi/test_creds/server1.key b/src/core/lib/tsi/test_creds/server1.key
index 143a5b8765..143a5b8765 100644
--- a/src/core/tsi/test_creds/server1.key
+++ b/src/core/lib/tsi/test_creds/server1.key
diff --git a/src/core/tsi/test_creds/server1.pem b/src/core/lib/tsi/test_creds/server1.pem
index f3d43fcc5b..f3d43fcc5b 100644
--- a/src/core/tsi/test_creds/server1.pem
+++ b/src/core/lib/tsi/test_creds/server1.pem
diff --git a/src/core/tsi/transport_security.c b/src/core/lib/tsi/transport_security.c
index db219a50a6..a2c0d46196 100644
--- a/src/core/tsi/transport_security.c
+++ b/src/core/lib/tsi/transport_security.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/tsi/transport_security.h"
+#include "src/core/lib/tsi/transport_security.h"
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/src/core/tsi/transport_security.h b/src/core/lib/tsi/transport_security.h
index 4077737473..349dd0ae9c 100644
--- a/src/core/tsi/transport_security.h
+++ b/src/core/lib/tsi/transport_security.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TSI_TRANSPORT_SECURITY_H
-#define GRPC_INTERNAL_CORE_TSI_TRANSPORT_SECURITY_H
+#ifndef GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H
+#define GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H
 
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -108,4 +108,4 @@ char *tsi_strdup(const char *src); /* Sadly, no strdup in C89. */
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_TSI_TRANSPORT_SECURITY_H */
+#endif /* GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H */
diff --git a/src/core/tsi/transport_security_interface.h b/src/core/lib/tsi/transport_security_interface.h
index 69ee17ae91..f88f1516a9 100644
--- a/src/core/tsi/transport_security_interface.h
+++ b/src/core/lib/tsi/transport_security_interface.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_TSI_TRANSPORT_SECURITY_INTERFACE_H
-#define GRPC_INTERNAL_CORE_TSI_TRANSPORT_SECURITY_INTERFACE_H
+#ifndef GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_INTERFACE_H
+#define GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_INTERFACE_H
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -341,4 +341,4 @@ void tsi_handshaker_destroy(tsi_handshaker *self);
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_TSI_TRANSPORT_SECURITY_INTERFACE_H */
+#endif /* GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_INTERFACE_H */
diff --git a/src/core/surface/init.c b/src/core/surface/init.c
deleted file mode 100644
index a4a53d3ec1..0000000000
--- a/src/core/surface/init.c
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- *
- * Copyright 2015-2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <grpc/support/port_platform.h>
-
-#include <memory.h>
-
-#include <grpc/census.h>
-#include <grpc/grpc.h>
-#include <grpc/support/alloc.h>
-#include <grpc/support/time.h>
-#include "src/core/channel/channel_stack.h"
-#include "src/core/client_config/lb_policy_registry.h"
-#include "src/core/client_config/lb_policies/pick_first.h"
-#include "src/core/client_config/lb_policies/round_robin.h"
-#include "src/core/client_config/resolver_registry.h"
-#include "src/core/client_config/resolvers/dns_resolver.h"
-#include "src/core/client_config/resolvers/sockaddr_resolver.h"
-#include "src/core/client_config/subchannel.h"
-#include "src/core/client_config/subchannel_index.h"
-#include "src/core/debug/trace.h"
-#include "src/core/iomgr/executor.h"
-#include "src/core/iomgr/iomgr.h"
-#include "src/core/profiling/timers.h"
-#include "src/core/surface/api_trace.h"
-#include "src/core/surface/call.h"
-#include "src/core/surface/completion_queue.h"
-#include "src/core/surface/init.h"
-#include "src/core/surface/surface_trace.h"
-#include "src/core/transport/chttp2_transport.h"
-#include "src/core/transport/connectivity_state.h"
-
-#ifndef GRPC_DEFAULT_NAME_PREFIX
-#define GRPC_DEFAULT_NAME_PREFIX "dns:///"
-#endif
-
-#define MAX_PLUGINS 128
-
-static gpr_once g_basic_init = GPR_ONCE_INIT;
-static gpr_mu g_init_mu;
-static int g_initializations;
-
-static void do_basic_init(void) {
-  gpr_mu_init(&g_init_mu);
-  g_initializations = 0;
-}
-
-typedef struct grpc_plugin {
-  void (*init)();
-  void (*destroy)();
-} grpc_plugin;
-
-static grpc_plugin g_all_of_the_plugins[MAX_PLUGINS];
-static int g_number_of_plugins = 0;
-
-void grpc_register_plugin(void (*init)(void), void (*destroy)(void)) {
-  GRPC_API_TRACE("grpc_register_plugin(init=%p, destroy=%p)", 2,
-                 ((void*)(intptr_t)init, (void*)(intptr_t)destroy));
-  GPR_ASSERT(g_number_of_plugins != MAX_PLUGINS);
-  g_all_of_the_plugins[g_number_of_plugins].init = init;
-  g_all_of_the_plugins[g_number_of_plugins].destroy = destroy;
-  g_number_of_plugins++;
-}
-
-void grpc_init(void) {
-  int i;
-  gpr_once_init(&g_basic_init, do_basic_init);
-
-  gpr_mu_lock(&g_init_mu);
-  if (++g_initializations == 1) {
-    gpr_time_init();
-    grpc_mdctx_global_init();
-    grpc_lb_policy_registry_init(grpc_pick_first_lb_factory_create());
-    grpc_register_lb_policy(grpc_pick_first_lb_factory_create());
-    grpc_register_lb_policy(grpc_round_robin_lb_factory_create());
-    grpc_resolver_registry_init(GRPC_DEFAULT_NAME_PREFIX);
-    grpc_register_resolver_type(grpc_dns_resolver_factory_create());
-    grpc_register_resolver_type(grpc_ipv4_resolver_factory_create());
-    grpc_register_resolver_type(grpc_ipv6_resolver_factory_create());
-#ifdef GPR_POSIX_SOCKET
-    grpc_register_resolver_type(grpc_unix_resolver_factory_create());
-#endif
-    grpc_register_tracer("api", &grpc_api_trace);
-    grpc_register_tracer("channel", &grpc_trace_channel);
-    grpc_register_tracer("http", &grpc_http_trace);
-    grpc_register_tracer("flowctl", &grpc_flowctl_trace);
-    grpc_register_tracer("connectivity_state", &grpc_connectivity_state_trace);
-    grpc_security_pre_init();
-    grpc_iomgr_init();
-    grpc_executor_init();
-    grpc_tracer_init("GRPC_TRACE");
-    /* Only initialize census if no one else has and some features are
-     * available. */
-    if (census_enabled() == CENSUS_FEATURE_NONE &&
-        census_supported() != CENSUS_FEATURE_NONE) {
-      if (census_initialize(census_supported())) { /* enable all features. */
-        gpr_log(GPR_ERROR, "Could not initialize census.");
-      }
-    }
-    gpr_timers_global_init();
-    grpc_cq_global_init();
-    grpc_subchannel_index_init();
-    for (i = 0; i < g_number_of_plugins; i++) {
-      if (g_all_of_the_plugins[i].init != NULL) {
-        g_all_of_the_plugins[i].init();
-      }
-    }
-  }
-  gpr_mu_unlock(&g_init_mu);
-  GRPC_API_TRACE("grpc_init(void)", 0, ());
-}
-
-void grpc_shutdown(void) {
-  int i;
-  GRPC_API_TRACE("grpc_shutdown(void)", 0, ());
-  gpr_mu_lock(&g_init_mu);
-  if (--g_initializations == 0) {
-    grpc_executor_shutdown();
-    grpc_cq_global_shutdown();
-    grpc_iomgr_shutdown();
-    grpc_subchannel_index_shutdown();
-    census_shutdown();
-    gpr_timers_global_destroy();
-    grpc_tracer_shutdown();
-    grpc_resolver_registry_shutdown();
-    grpc_lb_policy_registry_shutdown();
-    for (i = 0; i < g_number_of_plugins; i++) {
-      if (g_all_of_the_plugins[i].destroy != NULL) {
-        g_all_of_the_plugins[i].destroy();
-      }
-    }
-    grpc_mdctx_global_shutdown();
-  }
-  gpr_mu_unlock(&g_init_mu);
-}
-
-int grpc_is_initialized(void) {
-  int r;
-  gpr_once_init(&g_basic_init, do_basic_init);
-  gpr_mu_lock(&g_init_mu);
-  r = g_initializations > 0;
-  gpr_mu_unlock(&g_init_mu);
-  return r;
-}
diff --git a/src/core/transport/chttp2/huffsyms.c b/src/core/transport/chttp2/huffsyms.c
deleted file mode 100644
index 7b138e9b5d..0000000000
--- a/src/core/transport/chttp2/huffsyms.c
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/transport/chttp2/huffsyms.h"
-
-/* Constants pulled from the HPACK spec, and converted to C using the vim
-   command:
-   :%s/.*   \([0-9a-f]\+\)  \[ *\([0-9]\+\)\]/{0x\1, \2},/g */
-const grpc_chttp2_huffsym grpc_chttp2_huffsyms[GRPC_CHTTP2_NUM_HUFFSYMS] = {
-    {0x1ff8, 13},
-    {0x7fffd8, 23},
-    {0xfffffe2, 28},
-    {0xfffffe3, 28},
-    {0xfffffe4, 28},
-    {0xfffffe5, 28},
-    {0xfffffe6, 28},
-    {0xfffffe7, 28},
-    {0xfffffe8, 28},
-    {0xffffea, 24},
-    {0x3ffffffc, 30},
-    {0xfffffe9, 28},
-    {0xfffffea, 28},
-    {0x3ffffffd, 30},
-    {0xfffffeb, 28},
-    {0xfffffec, 28},
-    {0xfffffed, 28},
-    {0xfffffee, 28},
-    {0xfffffef, 28},
-    {0xffffff0, 28},
-    {0xffffff1, 28},
-    {0xffffff2, 28},
-    {0x3ffffffe, 30},
-    {0xffffff3, 28},
-    {0xffffff4, 28},
-    {0xffffff5, 28},
-    {0xffffff6, 28},
-    {0xffffff7, 28},
-    {0xffffff8, 28},
-    {0xffffff9, 28},
-    {0xffffffa, 28},
-    {0xffffffb, 28},
-    {0x14, 6},
-    {0x3f8, 10},
-    {0x3f9, 10},
-    {0xffa, 12},
-    {0x1ff9, 13},
-    {0x15, 6},
-    {0xf8, 8},
-    {0x7fa, 11},
-    {0x3fa, 10},
-    {0x3fb, 10},
-    {0xf9, 8},
-    {0x7fb, 11},
-    {0xfa, 8},
-    {0x16, 6},
-    {0x17, 6},
-    {0x18, 6},
-    {0x0, 5},
-    {0x1, 5},
-    {0x2, 5},
-    {0x19, 6},
-    {0x1a, 6},
-    {0x1b, 6},
-    {0x1c, 6},
-    {0x1d, 6},
-    {0x1e, 6},
-    {0x1f, 6},
-    {0x5c, 7},
-    {0xfb, 8},
-    {0x7ffc, 15},
-    {0x20, 6},
-    {0xffb, 12},
-    {0x3fc, 10},
-    {0x1ffa, 13},
-    {0x21, 6},
-    {0x5d, 7},
-    {0x5e, 7},
-    {0x5f, 7},
-    {0x60, 7},
-    {0x61, 7},
-    {0x62, 7},
-    {0x63, 7},
-    {0x64, 7},
-    {0x65, 7},
-    {0x66, 7},
-    {0x67, 7},
-    {0x68, 7},
-    {0x69, 7},
-    {0x6a, 7},
-    {0x6b, 7},
-    {0x6c, 7},
-    {0x6d, 7},
-    {0x6e, 7},
-    {0x6f, 7},
-    {0x70, 7},
-    {0x71, 7},
-    {0x72, 7},
-    {0xfc, 8},
-    {0x73, 7},
-    {0xfd, 8},
-    {0x1ffb, 13},
-    {0x7fff0, 19},
-    {0x1ffc, 13},
-    {0x3ffc, 14},
-    {0x22, 6},
-    {0x7ffd, 15},
-    {0x3, 5},
-    {0x23, 6},
-    {0x4, 5},
-    {0x24, 6},
-    {0x5, 5},
-    {0x25, 6},
-    {0x26, 6},
-    {0x27, 6},
-    {0x6, 5},
-    {0x74, 7},
-    {0x75, 7},
-    {0x28, 6},
-    {0x29, 6},
-    {0x2a, 6},
-    {0x7, 5},
-    {0x2b, 6},
-    {0x76, 7},
-    {0x2c, 6},
-    {0x8, 5},
-    {0x9, 5},
-    {0x2d, 6},
-    {0x77, 7},
-    {0x78, 7},
-    {0x79, 7},
-    {0x7a, 7},
-    {0x7b, 7},
-    {0x7ffe, 15},
-    {0x7fc, 11},
-    {0x3ffd, 14},
-    {0x1ffd, 13},
-    {0xffffffc, 28},
-    {0xfffe6, 20},
-    {0x3fffd2, 22},
-    {0xfffe7, 20},
-    {0xfffe8, 20},
-    {0x3fffd3, 22},
-    {0x3fffd4, 22},
-    {0x3fffd5, 22},
-    {0x7fffd9, 23},
-    {0x3fffd6, 22},
-    {0x7fffda, 23},
-    {0x7fffdb, 23},
-    {0x7fffdc, 23},
-    {0x7fffdd, 23},
-    {0x7fffde, 23},
-    {0xffffeb, 24},
-    {0x7fffdf, 23},
-    {0xffffec, 24},
-    {0xffffed, 24},
-    {0x3fffd7, 22},
-    {0x7fffe0, 23},
-    {0xffffee, 24},
-    {0x7fffe1, 23},
-    {0x7fffe2, 23},
-    {0x7fffe3, 23},
-    {0x7fffe4, 23},
-    {0x1fffdc, 21},
-    {0x3fffd8, 22},
-    {0x7fffe5, 23},
-    {0x3fffd9, 22},
-    {0x7fffe6, 23},
-    {0x7fffe7, 23},
-    {0xffffef, 24},
-    {0x3fffda, 22},
-    {0x1fffdd, 21},
-    {0xfffe9, 20},
-    {0x3fffdb, 22},
-    {0x3fffdc, 22},
-    {0x7fffe8, 23},
-    {0x7fffe9, 23},
-    {0x1fffde, 21},
-    {0x7fffea, 23},
-    {0x3fffdd, 22},
-    {0x3fffde, 22},
-    {0xfffff0, 24},
-    {0x1fffdf, 21},
-    {0x3fffdf, 22},
-    {0x7fffeb, 23},
-    {0x7fffec, 23},
-    {0x1fffe0, 21},
-    {0x1fffe1, 21},
-    {0x3fffe0, 22},
-    {0x1fffe2, 21},
-    {0x7fffed, 23},
-    {0x3fffe1, 22},
-    {0x7fffee, 23},
-    {0x7fffef, 23},
-    {0xfffea, 20},
-    {0x3fffe2, 22},
-    {0x3fffe3, 22},
-    {0x3fffe4, 22},
-    {0x7ffff0, 23},
-    {0x3fffe5, 22},
-    {0x3fffe6, 22},
-    {0x7ffff1, 23},
-    {0x3ffffe0, 26},
-    {0x3ffffe1, 26},
-    {0xfffeb, 20},
-    {0x7fff1, 19},
-    {0x3fffe7, 22},
-    {0x7ffff2, 23},
-    {0x3fffe8, 22},
-    {0x1ffffec, 25},
-    {0x3ffffe2, 26},
-    {0x3ffffe3, 26},
-    {0x3ffffe4, 26},
-    {0x7ffffde, 27},
-    {0x7ffffdf, 27},
-    {0x3ffffe5, 26},
-    {0xfffff1, 24},
-    {0x1ffffed, 25},
-    {0x7fff2, 19},
-    {0x1fffe3, 21},
-    {0x3ffffe6, 26},
-    {0x7ffffe0, 27},
-    {0x7ffffe1, 27},
-    {0x3ffffe7, 26},
-    {0x7ffffe2, 27},
-    {0xfffff2, 24},
-    {0x1fffe4, 21},
-    {0x1fffe5, 21},
-    {0x3ffffe8, 26},
-    {0x3ffffe9, 26},
-    {0xffffffd, 28},
-    {0x7ffffe3, 27},
-    {0x7ffffe4, 27},
-    {0x7ffffe5, 27},
-    {0xfffec, 20},
-    {0xfffff3, 24},
-    {0xfffed, 20},
-    {0x1fffe6, 21},
-    {0x3fffe9, 22},
-    {0x1fffe7, 21},
-    {0x1fffe8, 21},
-    {0x7ffff3, 23},
-    {0x3fffea, 22},
-    {0x3fffeb, 22},
-    {0x1ffffee, 25},
-    {0x1ffffef, 25},
-    {0xfffff4, 24},
-    {0xfffff5, 24},
-    {0x3ffffea, 26},
-    {0x7ffff4, 23},
-    {0x3ffffeb, 26},
-    {0x7ffffe6, 27},
-    {0x3ffffec, 26},
-    {0x3ffffed, 26},
-    {0x7ffffe7, 27},
-    {0x7ffffe8, 27},
-    {0x7ffffe9, 27},
-    {0x7ffffea, 27},
-    {0x7ffffeb, 27},
-    {0xffffffe, 28},
-    {0x7ffffec, 27},
-    {0x7ffffed, 27},
-    {0x7ffffee, 27},
-    {0x7ffffef, 27},
-    {0x7fffff0, 27},
-    {0x3ffffee, 26},
-    {0x3fffffff, 30},
-};
diff --git a/src/cpp/README.md b/src/cpp/README.md
index baeba08315..f2935e52d9 100644
--- a/src/cpp/README.md
+++ b/src/cpp/README.md
@@ -6,3 +6,77 @@ This directory contains source code for C++ implementation of gRPC.
 #Status
 
 Beta
+
+#Pre-requisites
+
+##Linux
+
+```sh
+ $ [sudo] apt-get install build-essential autoconf libtool
+```
+
+##Mac OSX
+
+For a Mac system, git is not available by default. You will first need to
+install Xcode from the Mac AppStore and then run the following command from a
+terminal:
+
+```sh
+ $ [sudo] xcode-select --install
+```
+
+##Protoc
+
+By default gRPC uses [protocol buffers](https://github.com/google/protobuf),
+you will need the `protoc` compiler to generate stub server and client code.
+
+If you compile gRPC from source, as described below, this also installs the
+`protoc` compiler.
+
+If it hasn't been installed, you can run the following commands to install it.
+
+```sh
+$ cd grpc/third_party/protobuf
+$ sudo make install   # 'make' should have been run by core grpc
+```
+
+Alternatively, you can download `protoc` binaries from
+[the protocol buffers Github repository](https://github.com/google/protobuf/releases).
+
+#Installation
+
+Currently to install gRPC for C++, you need to build from source as described
+below.
+
+#Build from Source
+
+```sh
+ $ git clone https://github.com/grpc/grpc.git
+ $ cd grpc
+ $ git submodule update --init
+ $ make
+ $ [sudo] make install
+```
+
+#Documentation
+
+You can find out how to build and run our simplest gRPC C++ example in our
+[C++ quick start](../../examples/cpp).
+
+For more detailed documentation on using gRPC in C++ , see our main
+documentation site at [grpc.io](http://grpc.io), specifically:
+
+* [Overview](http://www.grpc.io/docs/): An introduction to gRPC with a simple
+  Hello World example in all our supported languages, including C++.
+* [gRPC Basics - C++](http://www.grpc.io/docs/tutorials/basic/c.html):
+  A tutorial that steps you through creating a simple gRPC C++ example
+  application.
+* [Asynchronous Basics - C++](http://www.grpc.io/docs/tutorials/async/helloasync-cpp.html):
+  A tutorial that shows you how to use gRPC C++'s asynchronous/non-blocking
+  APIs.
+
+
+# Examples
+
+Code examples for gRPC C++ live in this repository's
+[examples/cpp](../../examples/cpp) directory.
diff --git a/src/cpp/client/channel.cc b/src/cpp/client/channel.cc
index ae20392d11..f174676172 100644
--- a/src/cpp/client/channel.cc
+++ b/src/cpp/client/channel.cc
@@ -49,7 +49,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
 namespace grpc {
 
diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc
index 73147fd7bb..de8b2db6e3 100644
--- a/src/cpp/client/client_context.cc
+++ b/src/cpp/client/client_context.cc
@@ -33,15 +33,15 @@
 
 #include <grpc++/client_context.h>
 
+#include <grpc++/security/credentials.h>
+#include <grpc++/server_context.h>
+#include <grpc++/support/time.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
-#include <grpc++/security/credentials.h>
-#include <grpc++/server_context.h>
-#include <grpc++/support/time.h>
 
-#include "src/core/channel/compress_filter.h"
+#include "src/core/lib/channel/compress_filter.h"
 #include "src/cpp/common/create_auth_context.h"
 
 namespace grpc {
diff --git a/src/cpp/client/insecure_credentials.cc b/src/cpp/client/insecure_credentials.cc
index 1293203b93..efea02995a 100644
--- a/src/cpp/client/insecure_credentials.cc
+++ b/src/cpp/client/insecure_credentials.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,11 +33,11 @@
 
 #include <grpc++/security/credentials.h>
 
-#include <grpc/grpc.h>
-#include <grpc/support/log.h>
 #include <grpc++/channel.h>
 #include <grpc++/support/channel_arguments.h>
 #include <grpc++/support/config.h>
+#include <grpc/grpc.h>
+#include <grpc/support/log.h>
 #include "src/cpp/client/create_channel_internal.h"
 
 namespace grpc {
diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc
index 074dae7ca7..cdc8406f24 100644
--- a/src/cpp/client/secure_credentials.cc
+++ b/src/cpp/client/secure_credentials.cc
@@ -31,12 +31,12 @@
  *
  */
 
+#include "src/cpp/client/secure_credentials.h"
 #include <grpc++/channel.h>
 #include <grpc++/impl/grpc_library.h>
 #include <grpc++/support/channel_arguments.h>
 #include <grpc/support/log.h>
 #include "src/cpp/client/create_channel_internal.h"
-#include "src/cpp/client/secure_credentials.h"
 #include "src/cpp/common/secure_auth_context.h"
 
 namespace grpc {
@@ -60,8 +60,7 @@ std::shared_ptr<grpc::Channel> SecureChannelCredentials::CreateChannel(
 
 SecureCallCredentials::SecureCallCredentials(grpc_call_credentials* c_creds)
     : c_creds_(c_creds) {
-  internal::GrpcLibraryInitializer gli_initializer;
-  gli_initializer.summon();
+  g_gli_initializer.summon();
 }
 
 bool SecureCallCredentials::ApplyToCall(grpc_call* call) {
@@ -83,14 +82,14 @@ std::shared_ptr<CallCredentials> WrapCallCredentials(
 }  // namespace
 
 std::shared_ptr<ChannelCredentials> GoogleDefaultCredentials() {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   return WrapChannelCredentials(grpc_google_default_credentials_create());
 }
 
 // Builds SSL Credentials given SSL specific options
 std::shared_ptr<ChannelCredentials> SslCredentials(
     const SslCredentialsOptions& options) {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {
       options.pem_private_key.c_str(), options.pem_cert_chain.c_str()};
 
@@ -102,7 +101,7 @@ std::shared_ptr<ChannelCredentials> SslCredentials(
 
 // Builds credentials for use when running in GCE
 std::shared_ptr<CallCredentials> GoogleComputeEngineCredentials() {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   return WrapCallCredentials(
       grpc_google_compute_engine_credentials_create(nullptr));
 }
@@ -110,7 +109,7 @@ std::shared_ptr<CallCredentials> GoogleComputeEngineCredentials() {
 // Builds JWT credentials.
 std::shared_ptr<CallCredentials> ServiceAccountJWTAccessCredentials(
     const grpc::string& json_key, long token_lifetime_seconds) {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   if (token_lifetime_seconds <= 0) {
     gpr_log(GPR_ERROR,
             "Trying to create JWTCredentials with non-positive lifetime");
@@ -125,7 +124,7 @@ std::shared_ptr<CallCredentials> ServiceAccountJWTAccessCredentials(
 // Builds refresh token credentials.
 std::shared_ptr<CallCredentials> GoogleRefreshTokenCredentials(
     const grpc::string& json_refresh_token) {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   return WrapCallCredentials(grpc_google_refresh_token_credentials_create(
       json_refresh_token.c_str(), nullptr));
 }
@@ -133,7 +132,7 @@ std::shared_ptr<CallCredentials> GoogleRefreshTokenCredentials(
 // Builds access token credentials.
 std::shared_ptr<CallCredentials> AccessTokenCredentials(
     const grpc::string& access_token) {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   return WrapCallCredentials(
       grpc_access_token_credentials_create(access_token.c_str(), nullptr));
 }
@@ -142,7 +141,7 @@ std::shared_ptr<CallCredentials> AccessTokenCredentials(
 std::shared_ptr<CallCredentials> GoogleIAMCredentials(
     const grpc::string& authorization_token,
     const grpc::string& authority_selector) {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   return WrapCallCredentials(grpc_google_iam_credentials_create(
       authorization_token.c_str(), authority_selector.c_str(), nullptr));
 }
@@ -224,7 +223,7 @@ MetadataCredentialsPluginWrapper::MetadataCredentialsPluginWrapper(
 
 std::shared_ptr<CallCredentials> MetadataCredentialsFromPlugin(
     std::unique_ptr<MetadataCredentialsPlugin> plugin) {
-  GrpcLibrary init;  // To call grpc_init().
+  GrpcLibraryCodegen init;  // To call grpc_init().
   const char* type = plugin->GetType();
   MetadataCredentialsPluginWrapper* wrapper =
       new MetadataCredentialsPluginWrapper(std::move(plugin));
diff --git a/src/cpp/client/secure_credentials.h b/src/cpp/client/secure_credentials.h
index 9e84102154..fd82331a44 100644
--- a/src/cpp/client/secure_credentials.h
+++ b/src/cpp/client/secure_credentials.h
@@ -36,8 +36,8 @@
 
 #include <grpc/grpc_security.h>
 
-#include <grpc++/support/config.h>
 #include <grpc++/security/credentials.h>
+#include <grpc++/support/config.h>
 
 #include "src/cpp/server/thread_pool_interface.h"
 
diff --git a/src/cpp/codegen/codegen_init.cc b/src/cpp/codegen/codegen_init.cc
new file mode 100644
index 0000000000..c5d22124b7
--- /dev/null
+++ b/src/cpp/codegen/codegen_init.cc
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/impl/codegen/core_codegen_interface.h>
+#include <grpc++/impl/codegen/grpc_library.h>
+
+/// Initializes the global gRPC variables for the codegen library. These will
+/// stay null in the absence of of grpc++ library. In this case, no gRPC
+/// features such as the ability to perform calls will be available. Trying to
+/// perform them would result in a segmentation fault when trying to deference
+/// the following nulled globals. These should be associated with actual
+/// as part of the instantiation of a \a grpc::GrpcLibraryInitializer variable.
+
+grpc::CoreCodegenInterface* grpc::g_core_codegen_interface = nullptr;
+grpc::GrpcLibraryInterface* grpc::g_glip = nullptr;
diff --git a/src/cpp/common/call.cc b/src/cpp/common/call.cc
deleted file mode 100644
index 5b87c2a806..0000000000
--- a/src/cpp/common/call.cc
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <grpc++/impl/call.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc++/channel.h>
-#include <grpc++/client_context.h>
-#include <grpc++/support/byte_buffer.h>
-#include "src/core/profiling/timers.h"
-
-namespace grpc {
-
-void FillMetadataMap(
-    grpc_metadata_array* arr,
-    std::multimap<grpc::string_ref, grpc::string_ref>* metadata) {
-  for (size_t i = 0; i < arr->count; i++) {
-    // TODO(yangg) handle duplicates?
-    metadata->insert(std::pair<grpc::string_ref, grpc::string_ref>(
-        arr->metadata[i].key, grpc::string_ref(arr->metadata[i].value,
-                                               arr->metadata[i].value_length)));
-  }
-  grpc_metadata_array_destroy(arr);
-  grpc_metadata_array_init(arr);
-}
-
-// TODO(yangg) if the map is changed before we send, the pointers will be a
-// mess. Make sure it does not happen.
-grpc_metadata* FillMetadataArray(
-    const std::multimap<grpc::string, grpc::string>& metadata) {
-  if (metadata.empty()) {
-    return nullptr;
-  }
-  grpc_metadata* metadata_array =
-      (grpc_metadata*)gpr_malloc(metadata.size() * sizeof(grpc_metadata));
-  size_t i = 0;
-  for (auto iter = metadata.cbegin(); iter != metadata.cend(); ++iter, ++i) {
-    metadata_array[i].key = iter->first.c_str();
-    metadata_array[i].value = iter->second.c_str();
-    metadata_array[i].value_length = iter->second.size();
-  }
-  return metadata_array;
-}
-
-Call::Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq)
-    : call_hook_(call_hook), cq_(cq), call_(call), max_message_size_(-1) {}
-
-Call::Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq,
-           int max_message_size)
-    : call_hook_(call_hook),
-      cq_(cq),
-      call_(call),
-      max_message_size_(max_message_size) {}
-
-void Call::PerformOps(CallOpSetInterface* ops) {
-  if (max_message_size_ > 0) {
-    ops->set_max_message_size(max_message_size_);
-  }
-  call_hook_->PerformOpsOnCall(ops, this);
-}
-
-}  // namespace grpc
diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc
index d7faa5e173..3bdb4398ab 100644
--- a/src/cpp/common/channel_arguments.cc
+++ b/src/cpp/common/channel_arguments.cc
@@ -36,7 +36,7 @@
 
 #include <grpc/impl/codegen/grpc_types.h>
 #include <grpc/support/log.h>
-#include "src/core/channel/channel_args.h"
+#include "src/core/lib/channel/channel_args.h"
 
 namespace grpc {
 
diff --git a/src/cpp/common/completion_queue.cc b/src/cpp/common/completion_queue.cc
index 4f76dfff1d..729dc33749 100644
--- a/src/cpp/common/completion_queue.cc
+++ b/src/cpp/common/completion_queue.cc
@@ -34,7 +34,6 @@
 
 #include <memory>
 
-#include <grpc++/impl/codegen/completion_queue_tag.h>
 #include <grpc++/impl/grpc_library.h>
 #include <grpc++/support/time.h>
 #include <grpc/grpc.h>
@@ -43,16 +42,13 @@
 namespace grpc {
 
 static internal::GrpcLibraryInitializer g_gli_initializer;
-CompletionQueue::CompletionQueue() {
-  g_gli_initializer.summon();
-  cq_ = grpc_completion_queue_create(nullptr);
-}
 
 CompletionQueue::CompletionQueue(grpc_completion_queue* take) : cq_(take) {}
 
-CompletionQueue::~CompletionQueue() { grpc_completion_queue_destroy(cq_); }
-
-void CompletionQueue::Shutdown() { grpc_completion_queue_shutdown(cq_); }
+void CompletionQueue::Shutdown() {
+  g_gli_initializer.summon();
+  grpc_completion_queue_shutdown(cq_);
+}
 
 CompletionQueue::NextStatus CompletionQueue::AsyncNextInternal(
     void** tag, bool* ok, gpr_timespec deadline) {
@@ -75,25 +71,4 @@ CompletionQueue::NextStatus CompletionQueue::AsyncNextInternal(
   }
 }
 
-bool CompletionQueue::Pluck(CompletionQueueTag* tag) {
-  auto deadline = gpr_inf_future(GPR_CLOCK_REALTIME);
-  auto ev = grpc_completion_queue_pluck(cq_, tag, deadline, nullptr);
-  bool ok = ev.success != 0;
-  void* ignored = tag;
-  GPR_ASSERT(tag->FinalizeResult(&ignored, &ok));
-  GPR_ASSERT(ignored == tag);
-  // Ignore mutations by FinalizeResult: Pluck returns the C API status
-  return ev.success != 0;
-}
-
-void CompletionQueue::TryPluck(CompletionQueueTag* tag) {
-  auto deadline = gpr_time_0(GPR_CLOCK_REALTIME);
-  auto ev = grpc_completion_queue_pluck(cq_, tag, deadline, nullptr);
-  if (ev.type == GRPC_QUEUE_TIMEOUT) return;
-  bool ok = ev.success != 0;
-  void* ignored = tag;
-  // the tag must be swallowed if using TryPluck
-  GPR_ASSERT(!tag->FinalizeResult(&ignored, &ok));
-}
-
 }  // namespace grpc
diff --git a/src/cpp/proto/proto_utils.cc b/src/cpp/common/core_codegen.cc
index 79e7bf1801..33a8f755e6 100644
--- a/src/cpp/proto/proto_utils.cc
+++ b/src/cpp/common/core_codegen.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,28 +31,31 @@
  *
  */
 
-#include <grpc++/impl/proto_utils.h>
+#include "src/cpp/common/core_codegen.h"
 
-#include <climits>
+#include <stdlib.h>
 
-#include <grpc/grpc.h>
+#include <grpc++/support/config.h>
 #include <grpc/byte_buffer.h>
 #include <grpc/byte_buffer_reader.h>
-#include <grpc/support/log.h>
+#include <grpc/grpc.h>
+#include <grpc/impl/codegen/alloc.h>
+#include <grpc/impl/codegen/byte_buffer.h>
+#include <grpc/impl/codegen/log.h>
+#include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
-#include <grpc/support/port_platform.h>
-#include <grpc++/support/config.h>
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 
-const int kMaxBufferLength = 8192;
+namespace {
+
+const int kGrpcBufferWriterMaxBufferLength = 8192;
 
 class GrpcBufferWriter GRPC_FINAL
     : public ::grpc::protobuf::io::ZeroCopyOutputStream {
  public:
-  explicit GrpcBufferWriter(grpc_byte_buffer** bp,
-                            int block_size = kMaxBufferLength)
+  explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size)
       : block_size_(block_size), byte_count_(0), have_backup_(false) {
     *bp = grpc_raw_byte_buffer_create(NULL, 0);
     slice_buffer_ = &(*bp)->data.raw.slice_buffer;
@@ -161,14 +164,56 @@ class GrpcBufferReader GRPC_FINAL
   grpc_byte_buffer_reader reader_;
   gpr_slice slice_;
 };
+}  // namespace
 
 namespace grpc {
 
-Status SerializeProto(const grpc::protobuf::Message& msg,
-                      grpc_byte_buffer** bp) {
+grpc_completion_queue* CoreCodegen::grpc_completion_queue_create(
+    void* reserved) {
+  return ::grpc_completion_queue_create(reserved);
+}
+
+void CoreCodegen::grpc_completion_queue_destroy(grpc_completion_queue* cq) {
+  ::grpc_completion_queue_destroy(cq);
+}
+
+grpc_event CoreCodegen::grpc_completion_queue_pluck(grpc_completion_queue* cq,
+                                                    void* tag,
+                                                    gpr_timespec deadline,
+                                                    void* reserved) {
+  return ::grpc_completion_queue_pluck(cq, tag, deadline, reserved);
+}
+
+void* CoreCodegen::gpr_malloc(size_t size) { return ::gpr_malloc(size); }
+
+void CoreCodegen::gpr_free(void* p) { return ::gpr_free(p); }
+
+void CoreCodegen::grpc_byte_buffer_destroy(grpc_byte_buffer* bb) {
+  ::grpc_byte_buffer_destroy(bb);
+}
+
+void CoreCodegen::grpc_metadata_array_init(grpc_metadata_array* array) {
+  ::grpc_metadata_array_init(array);
+}
+
+void CoreCodegen::grpc_metadata_array_destroy(grpc_metadata_array* array) {
+  ::grpc_metadata_array_destroy(array);
+}
+
+gpr_timespec CoreCodegen::gpr_inf_future(gpr_clock_type type) {
+  return ::gpr_inf_future(type);
+}
+
+void CoreCodegen::assert_fail(const char* failed_assertion) {
+  gpr_log(GPR_ERROR, "assertion failed: %s", failed_assertion);
+  abort();
+}
+
+Status CoreCodegen::SerializeProto(const grpc::protobuf::Message& msg,
+                                   grpc_byte_buffer** bp) {
   GPR_TIMER_SCOPE("SerializeProto", 0);
   int byte_size = msg.ByteSize();
-  if (byte_size <= kMaxBufferLength) {
+  if (byte_size <= kGrpcBufferWriterMaxBufferLength) {
     gpr_slice slice = gpr_slice_malloc(byte_size);
     GPR_ASSERT(GPR_SLICE_END_PTR(slice) ==
                msg.SerializeWithCachedSizesToArray(GPR_SLICE_START_PTR(slice)));
@@ -176,31 +221,36 @@ Status SerializeProto(const grpc::protobuf::Message& msg,
     gpr_slice_unref(slice);
     return Status::OK;
   } else {
-    GrpcBufferWriter writer(bp);
+    GrpcBufferWriter writer(bp, kGrpcBufferWriterMaxBufferLength);
     return msg.SerializeToZeroCopyStream(&writer)
                ? Status::OK
                : Status(StatusCode::INTERNAL, "Failed to serialize message");
   }
 }
 
-Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
-                        int max_message_size) {
+Status CoreCodegen::DeserializeProto(grpc_byte_buffer* buffer,
+                                     grpc::protobuf::Message* msg,
+                                     int max_message_size) {
   GPR_TIMER_SCOPE("DeserializeProto", 0);
-  if (!buffer) {
+  if (buffer == nullptr) {
     return Status(StatusCode::INTERNAL, "No payload");
   }
-  GrpcBufferReader reader(buffer);
-  ::grpc::protobuf::io::CodedInputStream decoder(&reader);
-  if (max_message_size > 0) {
-    decoder.SetTotalBytesLimit(max_message_size, max_message_size);
-  }
-  if (!msg->ParseFromCodedStream(&decoder)) {
-    return Status(StatusCode::INTERNAL, msg->InitializationErrorString());
-  }
-  if (!decoder.ConsumedEntireMessage()) {
-    return Status(StatusCode::INTERNAL, "Did not read entire message");
+  Status result = Status::OK;
+  {
+    GrpcBufferReader reader(buffer);
+    ::grpc::protobuf::io::CodedInputStream decoder(&reader);
+    if (max_message_size > 0) {
+      decoder.SetTotalBytesLimit(max_message_size, max_message_size);
+    }
+    if (!msg->ParseFromCodedStream(&decoder)) {
+      result = Status(StatusCode::INTERNAL, msg->InitializationErrorString());
+    }
+    if (!decoder.ConsumedEntireMessage()) {
+      result = Status(StatusCode::INTERNAL, "Did not read entire message");
+    }
   }
-  return Status::OK;
+  grpc_byte_buffer_destroy(buffer);
+  return result;
 }
 
 }  // namespace grpc
diff --git a/src/cpp/common/core_codegen.h b/src/cpp/common/core_codegen.h
new file mode 100644
index 0000000000..e15cb4c34a
--- /dev/null
+++ b/src/cpp/common/core_codegen.h
@@ -0,0 +1,71 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+// This file should be compiled as part of grpc++.
+
+#include <grpc++/impl/codegen/core_codegen_interface.h>
+#include <grpc/byte_buffer.h>
+#include <grpc/impl/codegen/grpc_types.h>
+
+namespace grpc {
+
+/// Implementation of the core codegen interface.
+class CoreCodegen : public CoreCodegenInterface {
+ private:
+  Status SerializeProto(const grpc::protobuf::Message& msg,
+                        grpc_byte_buffer** bp) override;
+
+  Status DeserializeProto(grpc_byte_buffer* buffer,
+                          grpc::protobuf::Message* msg,
+                          int max_message_size) override;
+
+  grpc_completion_queue* grpc_completion_queue_create(void* reserved) override;
+  void grpc_completion_queue_destroy(grpc_completion_queue* cq) override;
+  grpc_event grpc_completion_queue_pluck(grpc_completion_queue* cq, void* tag,
+                                         gpr_timespec deadline,
+                                         void* reserved) override;
+
+  void* gpr_malloc(size_t size) override;
+  void gpr_free(void* p) override;
+
+  void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) override;
+
+  void grpc_metadata_array_init(grpc_metadata_array* array) override;
+  void grpc_metadata_array_destroy(grpc_metadata_array* array) override;
+
+  gpr_timespec gpr_inf_future(gpr_clock_type type) override;
+
+  void assert_fail(const char* failed_assertion) override;
+};
+
+}  // namespace grpc
diff --git a/src/cpp/common/create_auth_context.h b/src/cpp/common/create_auth_context.h
index 4f3da397ba..c53055503f 100644
--- a/src/cpp/common/create_auth_context.h
+++ b/src/cpp/common/create_auth_context.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,8 +32,8 @@
  */
 #include <memory>
 
-#include <grpc/grpc.h>
 #include <grpc++/security/auth_context.h>
+#include <grpc/grpc.h>
 
 namespace grpc {
 
diff --git a/src/cpp/common/insecure_create_auth_context.cc b/src/cpp/common/insecure_create_auth_context.cc
index b2e153229a..7ec5d9bd12 100644
--- a/src/cpp/common/insecure_create_auth_context.cc
+++ b/src/cpp/common/insecure_create_auth_context.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,8 +32,8 @@
  */
 #include <memory>
 
-#include <grpc/grpc.h>
 #include <grpc++/security/auth_context.h>
+#include <grpc/grpc.h>
 
 namespace grpc {
 
diff --git a/src/cpp/common/secure_channel_arguments.cc b/src/cpp/common/secure_channel_arguments.cc
index e17d3b58b0..82e02f0238 100644
--- a/src/cpp/common/secure_channel_arguments.cc
+++ b/src/cpp/common/secure_channel_arguments.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,7 +34,7 @@
 #include <grpc++/support/channel_arguments.h>
 
 #include <grpc/grpc_security.h>
-#include "src/core/channel/channel_args.h"
+#include "src/core/lib/channel/channel_args.h"
 
 namespace grpc {
 
diff --git a/src/cpp/common/secure_create_auth_context.cc b/src/cpp/common/secure_create_auth_context.cc
index 40bc298b64..d7cf803fde 100644
--- a/src/cpp/common/secure_create_auth_context.cc
+++ b/src/cpp/common/secure_create_auth_context.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,9 +32,9 @@
  */
 #include <memory>
 
+#include <grpc++/security/auth_context.h>
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
-#include <grpc++/security/auth_context.h>
 #include "src/cpp/common/secure_auth_context.h"
 
 namespace grpc {
diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc
index 6d31a608c8..5ec05e4dcc 100644
--- a/src/cpp/server/server.cc
+++ b/src/cpp/server/server.cc
@@ -49,7 +49,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/profiling/timers.h"
+#include "src/core/lib/profiling/timers.h"
 #include "src/cpp/server/thread_pool_interface.h"
 
 namespace grpc {
@@ -264,6 +264,7 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
   void* const tag_;
   bool in_flight_;
   const bool has_request_payload_;
+  uint32_t incoming_flags_;
   grpc_call* call_;
   grpc_call_details* call_details_;
   gpr_timespec deadline_;
@@ -334,7 +335,7 @@ bool Server::RegisterService(const grpc::string* host, Service* service) {
     }
     RpcServiceMethod* method = it->get();
     void* tag = grpc_server_register_method(server_, method->name(),
-                                            host ? host->c_str() : nullptr);
+                                            host ? host->c_str() : nullptr, 0);
     if (tag == nullptr) {
       gpr_log(GPR_DEBUG, "Attempt to register %s multiple times",
               method->name());
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 134e5f1d5f..1947d68e3e 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -33,10 +33,10 @@
 
 #include <grpc++/server_builder.h>
 
-#include <grpc/support/cpu.h>
-#include <grpc/support/log.h>
 #include <grpc++/impl/service_type.h>
 #include <grpc++/server.h>
+#include <grpc/support/cpu.h>
+#include <grpc/support/log.h>
 #include "src/cpp/server/thread_pool_interface.h"
 
 namespace grpc {
diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc
index e205a1969b..0422650953 100644
--- a/src/cpp/server/server_context.cc
+++ b/src/cpp/server/server_context.cc
@@ -42,7 +42,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/channel/compress_filter.h"
+#include "src/core/lib/channel/compress_filter.h"
+#include "src/core/lib/surface/call.h"
 #include "src/cpp/common/create_auth_context.h"
 
 namespace grpc {
@@ -62,7 +63,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface {
   void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE;
   bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE;
 
-  bool CheckCancelled(CompletionQueue* cq);
+  bool CheckCancelled(CompletionQueue* cq) {
+    cq->TryPluck(this);
+    return CheckCancelledNoPluck();
+  }
+  bool CheckCancelledAsync() { return CheckCancelledNoPluck(); }
 
   void set_tag(void* tag) {
     has_tag_ = true;
@@ -72,6 +77,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface {
   void Unref();
 
  private:
+  bool CheckCancelledNoPluck() {
+    grpc::lock_guard<grpc::mutex> g(mu_);
+    return finalized_ ? (cancelled_ != 0) : false;
+  }
+
   bool has_tag_;
   void* tag_;
   grpc::mutex mu_;
@@ -88,12 +98,6 @@ void ServerContext::CompletionOp::Unref() {
   }
 }
 
-bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) {
-  cq->TryPluck(this);
-  grpc::lock_guard<grpc::mutex> g(mu_);
-  return finalized_ ? cancelled_ != 0 : false;
-}
-
 void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) {
   ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
   ops->data.recv_close_on_server.cancelled = &cancelled_;
@@ -182,12 +186,19 @@ void ServerContext::TryCancel() const {
 }
 
 bool ServerContext::IsCancelled() const {
-  return completion_op_ && completion_op_->CheckCancelled(cq_);
+  if (has_notify_when_done_tag_) {
+    // when using async API, but the result is only valid
+    // if the tag has already been delivered at the completion queue
+    return completion_op_ && completion_op_->CheckCancelledAsync();
+  } else {
+    // when using sync API
+    return completion_op_ && completion_op_->CheckCancelled(cq_);
+  }
 }
 
 void ServerContext::set_compression_level(grpc_compression_level level) {
   const grpc_compression_algorithm algorithm_for_level =
-      grpc_compression_algorithm_for_level(level);
+      grpc_call_compression_for_level(call_, level);
   set_compression_algorithm(algorithm_for_level);
 }
 
diff --git a/src/cpp/util/string_ref.cc b/src/cpp/util/string_ref.cc
index 66c79a1818..b55019b5f2 100644
--- a/src/cpp/util/string_ref.cc
+++ b/src/cpp/util/string_ref.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,72 +33,8 @@
 
 #include <grpc++/support/string_ref.h>
 
-#include <string.h>
-
-#include <algorithm>
-#include <iostream>
-
 namespace grpc {
 
 const size_t string_ref::npos = size_t(-1);
 
-string_ref& string_ref::operator=(const string_ref& rhs) {
-  data_ = rhs.data_;
-  length_ = rhs.length_;
-  return *this;
-}
-
-string_ref::string_ref(const char* s) : data_(s), length_(strlen(s)) {}
-
-string_ref string_ref::substr(size_t pos, size_t n) const {
-  if (pos > length_) pos = length_;
-  if (n > (length_ - pos)) n = length_ - pos;
-  return string_ref(data_ + pos, n);
-}
-
-int string_ref::compare(string_ref x) const {
-  size_t min_size = length_ < x.length_ ? length_ : x.length_;
-  int r = memcmp(data_, x.data_, min_size);
-  if (r < 0) return -1;
-  if (r > 0) return 1;
-  if (length_ < x.length_) return -1;
-  if (length_ > x.length_) return 1;
-  return 0;
-}
-
-bool string_ref::starts_with(string_ref x) const {
-  return length_ >= x.length_ && (memcmp(data_, x.data_, x.length_) == 0);
-}
-
-bool string_ref::ends_with(string_ref x) const {
-  return length_ >= x.length_ &&
-         (memcmp(data_ + (length_ - x.length_), x.data_, x.length_) == 0);
-}
-
-size_t string_ref::find(string_ref s) const {
-  auto it = std::search(cbegin(), cend(), s.cbegin(), s.cend());
-  return it == cend() ? npos : std::distance(cbegin(), it);
-}
-
-size_t string_ref::find(char c) const {
-  auto it = std::find(cbegin(), cend(), c);
-  return it == cend() ? npos : std::distance(cbegin(), it);
-}
-
-bool operator==(string_ref x, string_ref y) { return x.compare(y) == 0; }
-
-bool operator!=(string_ref x, string_ref y) { return x.compare(y) != 0; }
-
-bool operator<(string_ref x, string_ref y) { return x.compare(y) < 0; }
-
-bool operator<=(string_ref x, string_ref y) { return x.compare(y) <= 0; }
-
-bool operator>(string_ref x, string_ref y) { return x.compare(y) > 0; }
-
-bool operator>=(string_ref x, string_ref y) { return x.compare(y) >= 0; }
-
-std::ostream& operator<<(std::ostream& out, const string_ref& string) {
-  return out << grpc::string(string.begin(), string.end());
-}
-
 }  // namespace grpc
diff --git a/src/cpp/util/time.cc b/src/cpp/util/time.cc
index 2685e31ee6..bb5fce389d 100644
--- a/src/cpp/util/time.cc
+++ b/src/cpp/util/time.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,8 +35,8 @@
 
 #ifndef GRPC_CXX0X_NO_CHRONO
 
-#include <grpc/support/time.h>
 #include <grpc++/support/time.h>
+#include <grpc/support/time.h>
 
 using std::chrono::duration_cast;
 using std::chrono::nanoseconds;
diff --git a/src/csharp/Grpc.Core/Internal/NativeExtension.cs b/src/csharp/Grpc.Core/Internal/NativeExtension.cs
index 4c742ab6c3..282816d51e 100644
--- a/src/csharp/Grpc.Core/Internal/NativeExtension.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeExtension.cs
@@ -32,6 +32,7 @@
 #endregion
 
 using System;
+using System.Globalization;
 using System.IO;
 using System.Reflection;
 
@@ -99,14 +100,30 @@ namespace Grpc.Core.Internal
             // TODO: allow customizing path to native extension (possibly through exposing a GrpcEnvironment property).
 
             var libraryFlavor = string.Format("{0}_{1}", GetPlatformString(), GetArchitectureString());
-            var fullPath = Path.Combine(GetExecutingAssemblyDirectory(),
+            var fullPath = Path.Combine(Path.GetDirectoryName(GetAssemblyPath()),
                 NativeLibrariesDir, libraryFlavor, GetNativeLibraryFilename());
             return new UnmanagedLibrary(fullPath);
         }
 
-        private static string GetExecutingAssemblyDirectory()
+        private static string GetAssemblyPath()
         {
-            return Path.GetDirectoryName(typeof(NativeExtension).GetTypeInfo().Assembly.Location);
+            var assembly = typeof(NativeExtension).GetTypeInfo().Assembly;
+
+            // If assembly is shadowed (e.g. in a webapp), EscapedCodeBase is pointing
+            // to the original location of the assembly, and Location is pointing
+            // to the shadow copy. We care about the original location because
+            // the native dlls don't get shadowed.
+            var escapedCodeBase = assembly.EscapedCodeBase;
+            if (IsFileUri(escapedCodeBase))
+            {
+                return new Uri(escapedCodeBase).LocalPath;
+            }
+            return assembly.Location;
+        }
+
+        private static bool IsFileUri(string uri)
+        {
+            return uri.ToLowerInvariant().StartsWith(Uri.UriSchemeFile);
         }
 
         private static string GetPlatformString()
diff --git a/src/csharp/Grpc.Core/Logging/ConsoleLogger.cs b/src/csharp/Grpc.Core/Logging/ConsoleLogger.cs
index 5c5b802164..d41b1b9f26 100644
--- a/src/csharp/Grpc.Core/Logging/ConsoleLogger.cs
+++ b/src/csharp/Grpc.Core/Logging/ConsoleLogger.cs
@@ -1,6 +1,6 @@
 #region Copyright notice and license
 
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -33,12 +33,16 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 
 namespace Grpc.Core.Logging
 {
     /// <summary>Logger that logs to System.Console.</summary>
     public class ConsoleLogger : ILogger
     {
+        // Format similar enough to C core log format except nanosecond precision is not supported.
+        const string DateTimeFormatString = "MMdd HH:mm:ss.ffffff";
+
         readonly Type forType;
         readonly string forTypeString;
 
@@ -142,7 +146,7 @@ namespace Grpc.Core.Logging
         {
             Console.Error.WriteLine("{0}{1} {2}{3}",
                 severityString,
-                DateTime.Now,
+                DateTime.Now.ToString(DateTimeFormatString, CultureInfo.InvariantCulture),
                 forTypeString,
                 message);
         }
diff --git a/src/csharp/Grpc.Core/Metadata.cs b/src/csharp/Grpc.Core/Metadata.cs
index aa22f840d6..52cef96f40 100644
--- a/src/csharp/Grpc.Core/Metadata.cs
+++ b/src/csharp/Grpc.Core/Metadata.cs
@@ -323,7 +323,7 @@ namespace Grpc.Core
 
             private static string NormalizeKey(string key)
             {
-                var normalized = GrpcPreconditions.CheckNotNull(key, "key").ToLower(CultureInfo.InvariantCulture);
+                var normalized = GrpcPreconditions.CheckNotNull(key, "key").ToLowerInvariant();
                 GrpcPreconditions.CheckArgument(ValidKeyRegex.IsMatch(normalized), 
                     "Metadata entry key not valid. Keys can only contain lowercase alphanumeric characters, underscores and hyphens.");
                 return normalized;
diff --git a/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceImpl.cs
index 47a15224f1..1edeedae2f 100644
--- a/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceImpl.cs
+++ b/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceImpl.cs
@@ -1,6 +1,6 @@
 #region Copyright notice and license
 
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -46,16 +46,13 @@ namespace Grpc.Testing
     /// </summary>
     public class BenchmarkServiceImpl : BenchmarkService.IBenchmarkService
     {
-        private readonly int responseSize;
-
-        public BenchmarkServiceImpl(int responseSize)
+        public BenchmarkServiceImpl()
         {
-            this.responseSize = responseSize;
         }
 
         public Task<SimpleResponse> UnaryCall(SimpleRequest request, ServerCallContext context)
         {
-            var response = new SimpleResponse { Payload = CreateZerosPayload(responseSize) };
+            var response = new SimpleResponse { Payload = CreateZerosPayload(request.ResponseSize) };
             return Task.FromResult(response);
         }
 
@@ -63,7 +60,7 @@ namespace Grpc.Testing
         {
             await requestStream.ForEachAsync(async request =>
             {
-                var response = new SimpleResponse { Payload = CreateZerosPayload(responseSize) };
+                var response = new SimpleResponse { Payload = CreateZerosPayload(request.ResponseSize) };
                 await responseStream.WriteAsync(response);
             });
         }
diff --git a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
index c4016012cb..e6dc2321c4 100644
--- a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
@@ -41,6 +41,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Google.Protobuf;
 using Grpc.Core;
+using Grpc.Core.Logging;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 using Grpc.Testing;
@@ -50,42 +51,65 @@ namespace Grpc.IntegrationTesting
     /// <summary>
     /// Helper methods to start client runners for performance testing.
     /// </summary>
-    public static class ClientRunners
+    public class ClientRunners
     {
+        static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<ClientRunners>();
+
         /// <summary>
         /// Creates a started client runner.
         /// </summary>
         public static IClientRunner CreateStarted(ClientConfig config)
         {
+            Logger.Debug("ClientConfig: {0}", config);
             string target = config.ServerTargets.Single();
-            GrpcPreconditions.CheckArgument(config.LoadParams.LoadCase == LoadParams.LoadOneofCase.ClosedLoop);
+            GrpcPreconditions.CheckArgument(config.LoadParams.LoadCase == LoadParams.LoadOneofCase.ClosedLoop,
+                "Only closed loop scenario supported for C#");
+            GrpcPreconditions.CheckArgument(config.ClientChannels == 1, "ClientConfig.ClientChannels needs to be 1");
 
-            var credentials = config.SecurityParams != null ? TestCredentials.CreateSslCredentials() : ChannelCredentials.Insecure;
-            var channel = new Channel(target, credentials);
+            if (config.OutstandingRpcsPerChannel != 0)
+            {
+                Logger.Warning("ClientConfig.OutstandingRpcsPerChannel is not supported for C#. Ignoring the value");
+            }
+            if (config.AsyncClientThreads != 0)
+            {
+                Logger.Warning("ClientConfig.AsyncClientThreads is not supported for C#. Ignoring the value");
+            }
+            if (config.CoreLimit != 0)
+            {
+                Logger.Warning("ClientConfig.CoreLimit is not supported for C#. Ignoring the value");
+            }
+            if (config.CoreList.Count > 0)
+            {
+                Logger.Warning("ClientConfig.CoreList is not supported for C#. Ignoring the value");
+            }
 
-            switch (config.RpcType)
+            var credentials = config.SecurityParams != null ? TestCredentials.CreateSslCredentials() : ChannelCredentials.Insecure;
+            List<ChannelOption> channelOptions = null;
+            if (config.SecurityParams != null && config.SecurityParams.ServerHostOverride != "")
             {
-                case RpcType.UNARY:
-                    return new SyncUnaryClientRunner(channel,
-                        config.PayloadConfig.SimpleParams.ReqSize,
-                        config.HistogramParams);
-
-                case RpcType.STREAMING:
-                default:
-                    throw new ArgumentException("Unsupported RpcType.");
+                channelOptions = new List<ChannelOption>
+                {
+                    new ChannelOption(ChannelOptions.SslTargetNameOverride, config.SecurityParams.ServerHostOverride)
+                };
             }
+            var channel = new Channel(target, credentials, channelOptions);
+
+            return new ClientRunnerImpl(channel,
+                config.ClientType,
+                config.RpcType,
+                config.PayloadConfig,
+                config.HistogramParams);
         }
     }
 
-    /// <summary>
-    /// Client that starts synchronous unary calls in a closed loop.
-    /// </summary>
-    public class SyncUnaryClientRunner : IClientRunner
+    public class ClientRunnerImpl : IClientRunner
     {
         const double SecondsToNanos = 1e9;
 
         readonly Channel channel;
-        readonly int payloadSize;
+        readonly ClientType clientType;
+        readonly RpcType rpcType;
+        readonly PayloadConfig payloadConfig;
         readonly Histogram histogram;
 
         readonly BenchmarkService.IBenchmarkServiceClient client;
@@ -93,15 +117,19 @@ namespace Grpc.IntegrationTesting
         readonly CancellationTokenSource stoppedCts;
         readonly WallClockStopwatch wallClockStopwatch = new WallClockStopwatch();
         
-        public SyncUnaryClientRunner(Channel channel, int payloadSize, HistogramParams histogramParams)
+        public ClientRunnerImpl(Channel channel, ClientType clientType, RpcType rpcType, PayloadConfig payloadConfig, HistogramParams histogramParams)
         {
             this.channel = GrpcPreconditions.CheckNotNull(channel);
-            this.payloadSize = payloadSize;
+            this.clientType = clientType;
+            this.rpcType = rpcType;
+            this.payloadConfig = payloadConfig;
             this.histogram = new Histogram(histogramParams.Resolution, histogramParams.MaxPossible);
 
             this.stoppedCts = new CancellationTokenSource();
             this.client = BenchmarkService.NewClient(channel);
-            this.runnerTask = Task.Factory.StartNew(Run, TaskCreationOptions.LongRunning);
+
+            var threadBody = GetThreadBody();
+            this.runnerTask = Task.Factory.StartNew(threadBody, TaskCreationOptions.LongRunning);
         }
 
         public ClientStats GetStats(bool reset)
@@ -126,12 +154,9 @@ namespace Grpc.IntegrationTesting
             await channel.ShutdownAsync();
         }
 
-        private void Run()
+        private void RunClosedLoopUnary()
         {
-            var request = new SimpleRequest
-            {
-                Payload = CreateZerosPayload(payloadSize)
-            };
+            var request = CreateSimpleRequest();
             var stopwatch = new Stopwatch();
 
             while (!stoppedCts.Token.IsCancellationRequested)
@@ -145,6 +170,124 @@ namespace Grpc.IntegrationTesting
             }
         }
 
+        private async Task RunClosedLoopUnaryAsync()
+        {
+            var request = CreateSimpleRequest();
+            var stopwatch = new Stopwatch();
+
+            while (!stoppedCts.Token.IsCancellationRequested)
+            {
+                stopwatch.Restart();
+                await client.UnaryCallAsync(request);
+                stopwatch.Stop();
+
+                // spec requires data point in nanoseconds.
+                histogram.AddObservation(stopwatch.Elapsed.TotalSeconds * SecondsToNanos);
+            }
+        }
+
+        private async Task RunClosedLoopStreamingAsync()
+        {
+            var request = CreateSimpleRequest();
+            var stopwatch = new Stopwatch();
+
+            using (var call = client.StreamingCall())
+            {
+                while (!stoppedCts.Token.IsCancellationRequested)
+                {
+                    stopwatch.Restart();
+                    await call.RequestStream.WriteAsync(request);
+                    await call.ResponseStream.MoveNext();
+                    stopwatch.Stop();
+
+                    // spec requires data point in nanoseconds.
+                    histogram.AddObservation(stopwatch.Elapsed.TotalSeconds * SecondsToNanos);
+                }
+
+                // finish the streaming call
+                await call.RequestStream.CompleteAsync();
+                Assert.IsFalse(await call.ResponseStream.MoveNext());
+            }
+        }
+
+        private async Task RunGenericClosedLoopStreamingAsync()
+        {
+            var request = CreateByteBufferRequest();
+            var stopwatch = new Stopwatch();
+
+            var callDetails = new CallInvocationDetails<byte[], byte[]>(channel, GenericService.StreamingCallMethod, new CallOptions());
+
+            using (var call = Calls.AsyncDuplexStreamingCall(callDetails))
+            {
+                while (!stoppedCts.Token.IsCancellationRequested)
+                {
+                    stopwatch.Restart();
+                    await call.RequestStream.WriteAsync(request);
+                    await call.ResponseStream.MoveNext();
+                    stopwatch.Stop();
+
+                    // spec requires data point in nanoseconds.
+                    histogram.AddObservation(stopwatch.Elapsed.TotalSeconds * SecondsToNanos);
+                }
+
+                // finish the streaming call
+                await call.RequestStream.CompleteAsync();
+                Assert.IsFalse(await call.ResponseStream.MoveNext());
+            }
+        }
+
+        private Action GetThreadBody()
+        {
+            if (payloadConfig.PayloadCase == PayloadConfig.PayloadOneofCase.BytebufParams)
+            {
+                GrpcPreconditions.CheckArgument(clientType == ClientType.ASYNC_CLIENT, "Generic client only supports async API");
+                GrpcPreconditions.CheckArgument(rpcType == RpcType.STREAMING, "Generic client only supports streaming calls");
+                return () =>
+                {
+                    RunGenericClosedLoopStreamingAsync().Wait();
+                };
+            }
+
+            GrpcPreconditions.CheckNotNull(payloadConfig.SimpleParams);
+            if (clientType == ClientType.SYNC_CLIENT)
+            {
+                GrpcPreconditions.CheckArgument(rpcType == RpcType.UNARY, "Sync client can only be used for Unary calls in C#");
+                return RunClosedLoopUnary;
+            }
+            else if (clientType == ClientType.ASYNC_CLIENT)
+            {
+                switch (rpcType)
+                {
+                    case RpcType.UNARY:
+                        return () =>
+                        {
+                            RunClosedLoopUnaryAsync().Wait();
+                        };
+                    case RpcType.STREAMING:
+                        return () =>
+                        {
+                            RunClosedLoopStreamingAsync().Wait();
+                        };
+                }
+            }
+            throw new ArgumentException("Unsupported configuration.");
+        }
+
+        private SimpleRequest CreateSimpleRequest()
+        {
+            GrpcPreconditions.CheckNotNull(payloadConfig.SimpleParams);
+            return new SimpleRequest
+            {
+                Payload = CreateZerosPayload(payloadConfig.SimpleParams.ReqSize),
+                ResponseSize = payloadConfig.SimpleParams.RespSize
+            };
+        }
+
+        private byte[] CreateByteBufferRequest()
+        {
+            return new byte[payloadConfig.BytebufParams.ReqSize];
+        }
+
         private static Payload CreateZerosPayload(int size)
         {
             return new Payload { Body = ByteString.CopyFrom(new byte[size]) };
diff --git a/src/csharp/Grpc.IntegrationTesting/GenericService.cs b/src/csharp/Grpc.IntegrationTesting/GenericService.cs
new file mode 100644
index 0000000000..c6128264ac
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting/GenericService.cs
@@ -0,0 +1,71 @@
+#region Copyright notice and license
+
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using Google.Protobuf;
+using Grpc.Core;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+using Grpc.Testing;
+
+namespace Grpc.IntegrationTesting
+{
+    /// <summary>
+    /// Utility methods for defining and calling a service that doesn't use protobufs
+    /// for serialization/deserialization.
+    /// </summary>
+    public static class GenericService
+    {
+        readonly static Marshaller<byte[]> ByteArrayMarshaller = new Marshaller<byte[]>((b) => b, (b) => b);
+
+        public readonly static Method<byte[], byte[]> StreamingCallMethod = new Method<byte[], byte[]>(
+            MethodType.DuplexStreaming,
+            "grpc.testing.BenchmarkService",
+            "StreamingCall",
+            ByteArrayMarshaller,
+            ByteArrayMarshaller
+        );
+
+        public static ServerServiceDefinition BindHandler(DuplexStreamingServerMethod<byte[], byte[]> handler)
+        {
+            return ServerServiceDefinition.CreateBuilder(StreamingCallMethod.ServiceName)
+                .AddMethod(StreamingCallMethod, handler).Build();
+        }
+    }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
index 372991374e..4c049944ea 100644
--- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
@@ -120,6 +120,7 @@
     <Compile Include="WorkerServiceImpl.cs" />
     <Compile Include="QpsWorker.cs" />
     <Compile Include="WallClockStopwatch.cs" />
+    <Compile Include="GenericService.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>
diff --git a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
index 06d5ee93d8..a8cf75bd81 100644
--- a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
@@ -55,14 +55,7 @@ namespace Grpc.IntegrationTesting
         {
             var serverConfig = new ServerConfig
             {
-                ServerType = ServerType.ASYNC_SERVER,
-                PayloadConfig = new PayloadConfig
-                {
-                    SimpleParams = new SimpleProtoParams
-                    {
-                        RespSize = 100
-                    }
-                }
+                ServerType = ServerType.ASYNC_SERVER
             };
             serverRunner = ServerRunners.CreateStarted(serverConfig);
         }
@@ -88,7 +81,8 @@ namespace Grpc.IntegrationTesting
                 {
                     SimpleParams = new SimpleProtoParams
                     {
-                        ReqSize = 100
+                        ReqSize = 100,
+                        RespSize = 100
                     }
                 },
                 HistogramParams = new HistogramParams
diff --git a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs
index 4a73645e6c..c326378cfa 100644
--- a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs
@@ -41,6 +41,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Google.Protobuf;
 using Grpc.Core;
+using Grpc.Core.Logging;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 using Grpc.Testing;
@@ -50,27 +51,78 @@ namespace Grpc.IntegrationTesting
     /// <summary>
     /// Helper methods to start server runners for performance testing.
     /// </summary>
-    public static class ServerRunners
+    public class ServerRunners
     {
+        static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<ServerRunners>();
+
         /// <summary>
         /// Creates a started server runner.
         /// </summary>
         public static IServerRunner CreateStarted(ServerConfig config)
         {
-            GrpcPreconditions.CheckArgument(config.ServerType == ServerType.ASYNC_SERVER);
+            Logger.Debug("ServerConfig: {0}", config);
             var credentials = config.SecurityParams != null ? TestCredentials.CreateSslServerCredentials() : ServerCredentials.Insecure;
 
-            // TODO: qps_driver needs to setup payload properly...
-            int responseSize = config.PayloadConfig != null ? config.PayloadConfig.SimpleParams.RespSize : 0;
+            if (config.AsyncServerThreads != 0)
+            {
+                Logger.Warning("ServerConfig.AsyncServerThreads is not supported for C#. Ignoring the value");
+            }
+            if (config.CoreLimit != 0)
+            {
+                Logger.Warning("ServerConfig.CoreLimit is not supported for C#. Ignoring the value");
+            }
+            if (config.CoreList.Count > 0)
+            {
+                Logger.Warning("ServerConfig.CoreList is not supported for C#. Ignoring the value");
+            }
+
+            ServerServiceDefinition service = null;
+            if (config.ServerType == ServerType.ASYNC_SERVER)
+            {
+                GrpcPreconditions.CheckArgument(config.PayloadConfig == null,
+                    "ServerConfig.PayloadConfig shouldn't be set for BenchmarkService based server.");    
+                service = BenchmarkService.BindService(new BenchmarkServiceImpl());
+            }
+            else if (config.ServerType == ServerType.ASYNC_GENERIC_SERVER)
+            {
+                var genericService = new GenericServiceImpl(config.PayloadConfig.BytebufParams.RespSize);
+                service = GenericService.BindHandler(genericService.StreamingCall);
+            }
+            else
+            {
+                throw new ArgumentException("Unsupported ServerType");
+            }
+
             var server = new Server
             {
-                Services = { BenchmarkService.BindService(new BenchmarkServiceImpl(responseSize)) },
+                Services = { service },
                 Ports = { new ServerPort("[::]", config.Port, credentials) }
             };
 
             server.Start();
             return new ServerRunnerImpl(server);
         }
+
+        private class GenericServiceImpl
+        {
+            readonly byte[] response;
+
+            public GenericServiceImpl(int responseSize)
+            {
+                this.response = new byte[responseSize];
+            }
+
+            /// <summary>
+            /// Generic streaming call handler.
+            /// </summary>
+            public async Task StreamingCall(IAsyncStreamReader<byte[]> requestStream, IServerStreamWriter<byte[]> responseStream, ServerCallContext context)
+            {
+                await requestStream.ForEachAsync(async request =>
+                {
+                    await responseStream.WriteAsync(response);
+                });
+            }
+        }
     }
 
     /// <summary>
@@ -119,6 +171,5 @@ namespace Grpc.IntegrationTesting
         {
             return server.ShutdownAsync();
         }
-    }
-        
+    }        
 }
diff --git a/src/csharp/README.md b/src/csharp/README.md
index b4fa945ac9..201c5ab0b5 100644
--- a/src/csharp/README.md
+++ b/src/csharp/README.md
@@ -55,16 +55,11 @@ If you are a user of gRPC C#, go to Usage section above.
 
 **Windows**
 
-- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution. You can 
-  either build the native solution in `vsprojects/grpc_csharp_ext.sln` from Visual Studio manually, or you can use
-  a convenience batch script that builds everything for you.
+- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution. Open the
+  solution `vsprojects/grpc_csharp_ext.sln` in Visual Studio and build it.
 
-  ```
-  > REM From src/csharp directory
-  > buildall.bat
-  ```
-
-- Open Grpc.sln using Visual Studio.
+- Open `src\csharp\Grpc.sln` (path is relative to gRPC repository root)
+  using Visual Studio
 
 **Linux**
 
@@ -79,7 +74,7 @@ If you are a user of gRPC C#, go to Usage section above.
 **Mac OS X**
 
 - The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution.
-  
+
   ```sh
   # from the gRPC repository root
   $ tools/run_tests/run_tests.py -c dbg -l csharp --build_only
diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c
index 1df74a0993..642dc9ef42 100644
--- a/src/csharp/ext/grpc_csharp_ext.c
+++ b/src/csharp/ext/grpc_csharp_ext.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/support/string.h"
+#include "src/core/lib/support/string.h"
 
 #include <grpc/byte_buffer_reader.h>
 #include <grpc/support/port_platform.h>
diff --git a/src/node/README.md b/src/node/README.md
index 3501b54a66..15d4c6d02f 100644
--- a/src/node/README.md
+++ b/src/node/README.md
@@ -7,6 +7,8 @@ Beta
 ## PREREQUISITES
 - `node`: This requires `node` to be installed, version `0.12` or above. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package.
 
+- **Note:** If you installed `node` via a package manager and the version is still less than `0.12`, try directly installing it from [nodejs.org](https://nodejs.org).
+
 ## INSTALLATION
 
 Install the gRPC NPM package
@@ -17,7 +19,21 @@ npm install grpc
 
 ## BUILD FROM SOURCE
  1. Clone [the grpc Git Repository](https://github.com/grpc/grpc).
- 3. Run `npm install`.
+ 2. Run `npm install` from the repository root.
+
+ - **Note:** On Windows, this might fail due to [nodejs issue #4932](https://github.com/nodejs/node/issues/4932) in which case, you will see something like the following in `npm install`'s output (towards the very beginning):
+
+    ```
+     ..
+     Building the projects in this solution one at a time. To enable parallel build, please add the "/m" switch.
+     WINDOWS_BUILD_WARNING
+      "..\IMPORTANT: Due to https:\github.com\nodejs\node\issues\4932, to build this library on Windows, you must first remove C:\Users\jenkins\.node-gyp\4.4.0\include\node\openssl"
+      ...
+      ..
+    ```
+
+    To fix this, you will have to delete the folder `C:\Users\<username>\.node-gyp\<node_version>\include\node\openssl` and retry `npm install`
+
 
 ## TESTING
 To run the test suite, simply run `npm test` in the install location.
diff --git a/src/node/ext/call_credentials.cc b/src/node/ext/call_credentials.cc
index 98696db232..bd2d146bbc 100644
--- a/src/node/ext/call_credentials.cc
+++ b/src/node/ext/call_credentials.cc
@@ -35,6 +35,8 @@
 #include <nan.h>
 #include <uv.h>
 
+#include <list>
+
 #include "grpc/grpc.h"
 #include "grpc/grpc_security.h"
 #include "grpc/support/log.h"
@@ -161,6 +163,15 @@ NAN_METHOD(CallCredentials::CreateFromPlugin) {
   grpc_metadata_credentials_plugin plugin;
   plugin_state *state = new plugin_state;
   state->callback = new Nan::Callback(info[0].As<Function>());
+  state->pending_callbacks = new std::list<plugin_callback_data*>();
+  uv_mutex_init(&state->plugin_mutex);
+  uv_async_init(uv_default_loop(),
+                &state->plugin_async,
+                SendPluginCallback);
+  uv_unref((uv_handle_t*)&state->plugin_async);
+
+  state->plugin_async.data = state;
+
   plugin.get_metadata = plugin_get_metadata;
   plugin.destroy = plugin_destroy_state;
   plugin.state = reinterpret_cast<void*>(state);
@@ -208,48 +219,60 @@ NAN_METHOD(PluginCallback) {
 
 NAUV_WORK_CB(SendPluginCallback) {
   Nan::HandleScope scope;
-  plugin_callback_data *data = reinterpret_cast<plugin_callback_data*>(
-      async->data);
-  // Attach cb and user_data to plugin_callback so that it can access them later
-  v8::Local<v8::Function> plugin_callback = Nan::GetFunction(
-      Nan::New<v8::FunctionTemplate>(PluginCallback)).ToLocalChecked();
-  Nan::Set(plugin_callback, Nan::New("cb").ToLocalChecked(),
-           Nan::New<v8::External>(reinterpret_cast<void*>(data->cb)));
-  Nan::Set(plugin_callback, Nan::New("user_data").ToLocalChecked(),
-           Nan::New<v8::External>(data->user_data));
-  const int argc = 2;
-  v8::Local<v8::Value> argv[argc] = {
-    Nan::New(data->service_url).ToLocalChecked(),
-    plugin_callback
-  };
-  Nan::Callback *callback = data->state->callback;
-  callback->Call(argc, argv);
-  delete data;
-  uv_unref((uv_handle_t *)async);
-  uv_close((uv_handle_t *)async, (uv_close_cb)free);
+  plugin_state *state = reinterpret_cast<plugin_state*>(async->data);
+  std::list<plugin_callback_data*> callbacks;
+  uv_mutex_lock(&state->plugin_mutex);
+  callbacks.splice(callbacks.begin(), *state->pending_callbacks);
+  uv_mutex_unlock(&state->plugin_mutex);
+  while (!callbacks.empty()) {
+    plugin_callback_data *data = callbacks.front();
+    callbacks.pop_front();
+    // Attach cb and user_data to plugin_callback so that it can access them later
+    v8::Local<v8::Function> plugin_callback = Nan::GetFunction(
+        Nan::New<v8::FunctionTemplate>(PluginCallback)).ToLocalChecked();
+    Nan::Set(plugin_callback, Nan::New("cb").ToLocalChecked(),
+             Nan::New<v8::External>(reinterpret_cast<void*>(data->cb)));
+    Nan::Set(plugin_callback, Nan::New("user_data").ToLocalChecked(),
+             Nan::New<v8::External>(data->user_data));
+    const int argc = 2;
+    v8::Local<v8::Value> argv[argc] = {
+      Nan::New(data->service_url).ToLocalChecked(),
+      plugin_callback
+    };
+    Nan::Callback *callback = state->callback;
+    callback->Call(argc, argv);
+    delete data;
+  }
 }
 
 void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
                          grpc_credentials_plugin_metadata_cb cb,
                          void *user_data) {
-  uv_async_t *async = static_cast<uv_async_t*>(malloc(sizeof(uv_async_t)));
-  uv_async_init(uv_default_loop(),
-                async,
-                SendPluginCallback);
+  plugin_state *p_state = reinterpret_cast<plugin_state*>(state);
   plugin_callback_data *data = new plugin_callback_data;
-  data->state = reinterpret_cast<plugin_state*>(state);
   data->service_url = context.service_url;
   data->cb = cb;
   data->user_data = user_data;
-  async->data = data;
-  /* libuv says that it will coalesce calls to uv_async_send. If there is ever a
-   * problem with a callback not getting called, that is probably the reason */
-  uv_async_send(async);
+
+  uv_mutex_lock(&p_state->plugin_mutex);
+  p_state->pending_callbacks->push_back(data);
+  uv_mutex_unlock(&p_state->plugin_mutex);
+
+  uv_async_send(&p_state->plugin_async);
+}
+
+void plugin_uv_close_cb(uv_handle_t *handle) {
+  uv_async_t *async = reinterpret_cast<uv_async_t*>(handle);
+  plugin_state *state = reinterpret_cast<plugin_state *>(async->data);
+  uv_mutex_destroy(&state->plugin_mutex);
+  delete state->pending_callbacks;
+  delete state->callback;
+  delete state;
 }
 
 void plugin_destroy_state(void *ptr) {
   plugin_state *state = reinterpret_cast<plugin_state *>(ptr);
-  delete state->callback;
+  uv_close((uv_handle_t*)&state->plugin_async, plugin_uv_close_cb);
 }
 
 }  // namespace node
diff --git a/src/node/ext/call_credentials.h b/src/node/ext/call_credentials.h
index a9bfe30f94..1f35595f3d 100644
--- a/src/node/ext/call_credentials.h
+++ b/src/node/ext/call_credentials.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,8 +34,11 @@
 #ifndef GRPC_NODE_CALL_CREDENTIALS_H_
 #define GRPC_NODE_CALL_CREDENTIALS_H_
 
+#include <list>
+
 #include <node.h>
 #include <nan.h>
+#include <uv.h>
 #include "grpc/grpc_security.h"
 
 namespace grpc {
@@ -73,17 +76,20 @@ class CallCredentials : public Nan::ObjectWrap {
 
 /* Auth metadata plugin functionality */
 
-typedef struct plugin_state {
-  Nan::Callback *callback;
-} plugin_state;
-
 typedef struct plugin_callback_data {
-  plugin_state *state;
   const char *service_url;
   grpc_credentials_plugin_metadata_cb cb;
   void *user_data;
 } plugin_callback_data;
 
+typedef struct plugin_state {
+  Nan::Callback *callback;
+  std::list<plugin_callback_data*> *pending_callbacks;
+  uv_mutex_t plugin_mutex;
+  // async.data == this
+  uv_async_t plugin_async;
+} plugin_state;
+
 void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
                          grpc_credentials_plugin_metadata_cb cb,
                          void *user_data);
diff --git a/src/node/index.js b/src/node/index.js
index 1c197729d7..6567d56260 100644
--- a/src/node/index.js
+++ b/src/node/index.js
@@ -87,6 +87,10 @@ var loadObject = exports.loadObject;
  *   Buffers. Defaults to false
  * - longsAsStrings: deserialize long values as strings instead of objects.
  *   Defaults to true
+ * - deprecatedArgumentOrder: Use the beta method argument order for client
+ *   methods, with optional arguments after the callback. Defaults to false.
+ *   This option is only a temporary stopgap measure to smooth an API breakage.
+ *   It is deprecated, and new code should not use it.
  * @param {string|{root: string, file: string}} filename The file to load
  * @param {string=} format The file format to expect. Must be either 'proto' or
  *     'json'. Defaults to 'proto'
diff --git a/src/node/interop/interop_client.js b/src/node/interop/interop_client.js
index 5602011a8e..ac0eddcf45 100644
--- a/src/node/interop/interop_client.js
+++ b/src/node/interop/interop_client.js
@@ -286,7 +286,7 @@ function cancelAfterFirstResponse(client, done) {
 function timeoutOnSleepingServer(client, done) {
   var deadline = new Date();
   deadline.setMilliseconds(deadline.getMilliseconds() + 1);
-  var call = client.fullDuplexCall(null, {deadline: deadline});
+  var call = client.fullDuplexCall({deadline: deadline});
   call.write({
     payload: {body: zeroBuffer(27182)}
   });
@@ -316,10 +316,10 @@ function customMetadata(client, done) {
       body: zeroBuffer(271828)
     }
   };
-  var unary = client.unaryCall(arg, function(err, resp) {
+  var unary = client.unaryCall(arg, metadata, function(err, resp) {
     assert.ifError(err);
     done();
-  }, metadata);
+  });
   unary.on('metadata', function(metadata) {
     assert.deepEqual(metadata.get(ECHO_INITIAL_KEY),
                      ['test_initial_metadata_value']);
@@ -455,14 +455,14 @@ function perRpcAuthTest(client, done, extra) {
       credential = credential.createScoped(scope);
     }
     var creds = grpc.credentials.createFromGoogleCredential(credential);
-    client.unaryCall(arg, function(err, resp) {
+    client.unaryCall(arg, {credentials: creds}, function(err, resp) {
       assert.ifError(err);
       assert.strictEqual(resp.username, SERVICE_ACCOUNT_EMAIL);
       assert(extra.oauth_scope.indexOf(resp.oauth_scope) > -1);
       if (done) {
         done();
       }
-    }, null, {credentials: creds});
+    });
   });
 }
 
diff --git a/src/node/performance/benchmark_client.js b/src/node/performance/benchmark_client.js
index 620aecde97..80bec0b73e 100644
--- a/src/node/performance/benchmark_client.js
+++ b/src/node/performance/benchmark_client.js
@@ -45,6 +45,9 @@ var EventEmitter = require('events');
 var _ = require('lodash');
 var PoissonProcess = require('poisson-process');
 var Histogram = require('./histogram');
+
+var genericService = require('./generic_service');
+
 var grpc = require('../../../');
 var serviceProto = grpc.load({
   root: __dirname + '/../../..',
@@ -104,10 +107,14 @@ function BenchmarkClient(server_targets, channels, histogram_params,
   }
 
   this.clients = [];
+  var GenericClient = grpc.makeGenericClientConstructor(genericService);
+  this.genericClients = [];
 
   for (var i = 0; i < channels; i++) {
     this.clients[i] = new serviceProto.BenchmarkService(
         server_targets[i % server_targets.length], creds, options);
+    this.genericClients[i] = new GenericClient(
+        server_targets[i % server_targets.length], creds, options);
   }
 
   this.histogram = new Histogram(histogram_params.resolution,
@@ -130,9 +137,11 @@ util.inherits(BenchmarkClient, EventEmitter);
  *     'STREAMING'
  * @param {number} req_size The size of the payload to send with each request
  * @param {number} resp_size The size of payload to request be sent in responses
+ * @param {boolean} generic Indicates that the generic (non-proto) clients
+ *     should be used
  */
 BenchmarkClient.prototype.startClosedLoop = function(
-    outstanding_rpcs_per_channel, rpc_type, req_size, resp_size) {
+    outstanding_rpcs_per_channel, rpc_type, req_size, resp_size, generic) {
   var self = this;
 
   self.running = true;
@@ -141,12 +150,20 @@ BenchmarkClient.prototype.startClosedLoop = function(
 
   var makeCall;
 
-  var argument = {
-    response_size: resp_size,
-    payload: {
-      body: zeroBuffer(req_size)
-    }
-  };
+  var argument;
+  var client_list;
+  if (generic) {
+    argument = zeroBuffer(req_size);
+    client_list = self.genericClients;
+  } else {
+    argument = {
+      response_size: resp_size,
+      payload: {
+        body: zeroBuffer(req_size)
+      }
+    };
+    client_list = self.clients;
+  }
 
   if (rpc_type == 'UNARY') {
     makeCall = function(client) {
@@ -195,7 +212,7 @@ BenchmarkClient.prototype.startClosedLoop = function(
     };
   }
 
-  _.each(self.clients, function(client) {
+  _.each(client_list, function(client) {
     _.times(outstanding_rpcs_per_channel, function() {
       makeCall(client);
     });
@@ -213,9 +230,12 @@ BenchmarkClient.prototype.startClosedLoop = function(
  * @param {number} req_size The size of the payload to send with each request
  * @param {number} resp_size The size of payload to request be sent in responses
  * @param {number} offered_load The load parameter for the Poisson process
+ * @param {boolean} generic Indicates that the generic (non-proto) clients
+ *     should be used
  */
 BenchmarkClient.prototype.startPoisson = function(
-    outstanding_rpcs_per_channel, rpc_type, req_size, resp_size, offered_load) {
+    outstanding_rpcs_per_channel, rpc_type, req_size, resp_size, offered_load,
+    generic) {
   var self = this;
 
   self.running = true;
@@ -224,12 +244,20 @@ BenchmarkClient.prototype.startPoisson = function(
 
   var makeCall;
 
-  var argument = {
-    response_size: resp_size,
-    payload: {
-      body: zeroBuffer(req_size)
-    }
-  };
+  var argument;
+  var client_list;
+  if (generic) {
+    argument = zeroBuffer(req_size);
+    client_list = self.genericClients;
+  } else {
+    argument = {
+      response_size: resp_size,
+      payload: {
+        body: zeroBuffer(req_size)
+      }
+    };
+    client_list = self.clients;
+  }
 
   if (rpc_type == 'UNARY') {
     makeCall = function(client, poisson) {
@@ -282,7 +310,7 @@ BenchmarkClient.prototype.startPoisson = function(
 
   var averageIntervalMs = (1 / offered_load) * 1000;
 
-  _.each(self.clients, function(client) {
+  _.each(client_list, function(client) {
     _.times(outstanding_rpcs_per_channel, function() {
       var p = PoissonProcess.create(averageIntervalMs, function() {
         makeCall(client, p);
diff --git a/src/node/performance/benchmark_server.js b/src/node/performance/benchmark_server.js
index e48acd48f5..b1b0bd12ab 100644
--- a/src/node/performance/benchmark_server.js
+++ b/src/node/performance/benchmark_server.js
@@ -41,6 +41,8 @@
 var fs = require('fs');
 var path = require('path');
 
+var genericService = require('./generic_service');
+
 var grpc = require('../../../');
 var serviceProto = grpc.load({
   root: __dirname + '/../../..',
@@ -84,14 +86,28 @@ function streamingCall(call) {
   });
 }
 
+function makeStreamingGenericCall(response_size) {
+  var response = zeroBuffer(response_size);
+  return function streamingGenericCall(call) {
+    call.on('data', function(value) {
+      call.write(response);
+    });
+    call.on('end', function() {
+      call.end();
+    });
+  };
+}
+
 /**
  * BenchmarkServer class. Constructed based on parameters from the driver and
  * stores statistics.
  * @param {string} host The host to serve on
  * @param {number} port The port to listen to
- * @param {tls} Indicates whether TLS should be used
+ * @param {boolean} tls Indicates whether TLS should be used
+ * @param {boolean} generic Indicates whether to use the generic service
+ * @param {number=} response_size The response size for the generic service
  */
-function BenchmarkServer(host, port, tls) {
+function BenchmarkServer(host, port, tls, generic, response_size) {
   var server_creds;
   var host_override;
   if (tls) {
@@ -109,10 +125,16 @@ function BenchmarkServer(host, port, tls) {
 
   var server = new grpc.Server();
   this.port = server.bind(host + ':' + port, server_creds);
-  server.addProtoService(serviceProto.BenchmarkService.service, {
-    unaryCall: unaryCall,
-    streamingCall: streamingCall
-  });
+  if (generic) {
+    server.addService(genericService, {
+      streamingCall: makeStreamingGenericCall(response_size)
+    });
+  } else {
+    server.addProtoService(serviceProto.BenchmarkService.service, {
+      unaryCall: unaryCall,
+      streamingCall: streamingCall
+    });
+  }
   this.server = server;
 }
 
diff --git a/src/node/performance/generic_service.js b/src/node/performance/generic_service.js
new file mode 100644
index 0000000000..ce09cc4336
--- /dev/null
+++ b/src/node/performance/generic_service.js
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+var _ = require('lodash');
+
+module.exports = {
+  'streamingCall' : {
+    path: '/grpc.testing/BenchmarkService',
+    requestStream: true,
+    responseStream: true,
+    requestSerialize: _.identity,
+    requestDeserialize: _.identity,
+    responseSerialize: _.identity,
+    responseDeserialize: _.identity
+  }
+};
diff --git a/src/node/performance/worker_service_impl.js b/src/node/performance/worker_service_impl.js
index 1439249878..2c4651370f 100644
--- a/src/node/performance/worker_service_impl.js
+++ b/src/node/performance/worker_service_impl.js
@@ -56,18 +56,31 @@ exports.runClient = function runClient(call) {
       client.on('error', function(error) {
         call.emit('error', error);
       });
+      var req_size, resp_size, generic;
+      switch (setup.payload_config.payload) {
+        case 'bytebuf_params':
+        req_size = setup.payload_config.bytebuf_params.req_size;
+        resp_size = setup.payload_config.bytebuf_params.resp_size;
+        generic = true;
+        break;
+        case 'simple_params':
+        req_size = setup.payload_config.simple_params.req_size;
+        resp_size = setup.payload_config.simple_params.resp_size;
+        generic = false;
+        break;
+        default:
+        call.emit('error', new Error('Unsupported PayloadConfig type' +
+            setup.payload_config.payload));
+      }
       switch (setup.load_params.load) {
         case 'closed_loop':
         client.startClosedLoop(setup.outstanding_rpcs_per_channel,
-                               setup.rpc_type,
-                               setup.payload_config.simple_params.req_size,
-                               setup.payload_config.simple_params.resp_size);
+                               setup.rpc_type, req_size, resp_size, generic);
         break;
         case 'poisson':
         client.startPoisson(setup.outstanding_rpcs_per_channel,
-                            setup.rpc_type, setup.payload_config.req_size,
-                            setup.payload_config.resp_size,
-                            setup.load_params.poisson.offered_load);
+                            setup.rpc_type, req_size, resp_size,
+                            setup.load_params.poisson.offered_load, generic);
         break;
         default:
         call.emit('error', new Error('Unsupported LoadParams type' +
diff --git a/src/node/src/client.js b/src/node/src/client.js
index 9acf51bd98..82142379da 100644
--- a/src/node/src/client.js
+++ b/src/node/src/client.js
@@ -50,6 +50,7 @@
 'use strict';
 
 var _ = require('lodash');
+var arguejs = require('arguejs');
 
 var grpc = require('./grpc_extension');
 
@@ -149,6 +150,9 @@ function _readsDone(status) {
   if (!status) {
     status = {code: grpc.status.OK, details: 'OK'};
   }
+  if (status.code !== grpc.status.OK) {
+    this.call.cancelWithStatus(status.code, status.details);
+  }
   this.finished = true;
   this.read_status = status;
   this._emitStatusIfDone();
@@ -350,21 +354,23 @@ function makeUnaryRequestFunction(method, serialize, deserialize) {
    * @this {Client} Client object. Must have a channel member.
    * @param {*} argument The argument to the call. Should be serializable with
    *     serialize
-   * @param {function(?Error, value=)} callback The callback to for when the
-   *     response is received
    * @param {Metadata=} metadata Metadata to add to the call
    * @param {Object=} options Options map
+   * @param {function(?Error, value=)} callback The callback to for when the
+   *     response is received
    * @return {EventEmitter} An event emitter for stream related events
    */
-  function makeUnaryRequest(argument, callback, metadata, options) {
+  function makeUnaryRequest(argument, metadata, options, callback) {
     /* jshint validthis: true */
+    /* While the arguments are listed in the function signature, those variables
+     * are not used directly. Instead, ArgueJS processes the arguments
+     * object. This allows for simple handling of optional arguments in the
+     * middle of the argument list, and also provides type checking. */
+    var args = arguejs({argument: null, metadata: [Metadata, new Metadata()],
+                        options: [Object], callback: Function}, arguments);
     var emitter = new EventEmitter();
-    var call = getCall(this.$channel, method, options);
-    if (metadata === null || metadata === undefined) {
-      metadata = new Metadata();
-    } else {
-      metadata = metadata.clone();
-    }
+    var call = getCall(this.$channel, method, args.options);
+    metadata = args.metadata.clone();
     emitter.cancel = function cancel() {
       call.cancel();
     };
@@ -372,9 +378,9 @@ function makeUnaryRequestFunction(method, serialize, deserialize) {
       return call.getPeer();
     };
     var client_batch = {};
-    var message = serialize(argument);
-    if (options) {
-      message.grpcWriteFlags = options.flags;
+    var message = serialize(args.argument);
+    if (args.options) {
+      message.grpcWriteFlags = args.options.flags;
     }
     client_batch[grpc.opType.SEND_INITIAL_METADATA] =
         metadata._getCoreRepresentation();
@@ -392,7 +398,7 @@ function makeUnaryRequestFunction(method, serialize, deserialize) {
       if (status.code === grpc.status.OK) {
         if (err) {
           // Got a batch error, but OK status. Something went wrong
-          callback(err);
+          args.callback(err);
           return;
         } else {
           try {
@@ -408,12 +414,12 @@ function makeUnaryRequestFunction(method, serialize, deserialize) {
         }
       }
       if (status.code !== grpc.status.OK) {
-        error = new Error(response.status.details);
+        error = new Error(status.details);
         error.code = status.code;
         error.metadata = status.metadata;
-        callback(error);
+        args.callback(error);
       } else {
-        callback(null, deserialized);
+        args.callback(null, deserialized);
       }
       emitter.emit('status', status);
       emitter.emit('metadata', Metadata._fromCoreRepresentation(
@@ -437,21 +443,23 @@ function makeClientStreamRequestFunction(method, serialize, deserialize) {
    * Make a client stream request with this method on the given channel with the
    * given callback, etc.
    * @this {Client} Client object. Must have a channel member.
-   * @param {function(?Error, value=)} callback The callback to for when the
-   *     response is received
    * @param {Metadata=} metadata Array of metadata key/value pairs to add to the
    *     call
    * @param {Object=} options Options map
+   * @param {function(?Error, value=)} callback The callback to for when the
+   *     response is received
    * @return {EventEmitter} An event emitter for stream related events
    */
-  function makeClientStreamRequest(callback, metadata, options) {
+  function makeClientStreamRequest(metadata, options, callback) {
     /* jshint validthis: true */
-    var call = getCall(this.$channel, method, options);
-    if (metadata === null || metadata === undefined) {
-      metadata = new Metadata();
-    } else {
-      metadata = metadata.clone();
-    }
+    /* While the arguments are listed in the function signature, those variables
+     * are not used directly. Instead, ArgueJS processes the arguments
+     * object. This allows for simple handling of optional arguments in the
+     * middle of the argument list, and also provides type checking. */
+    var args = arguejs({metadata: [Metadata, new Metadata()],
+                        options: [Object], callback: Function}, arguments);
+    var call = getCall(this.$channel, method, args.options);
+    metadata = args.metadata.clone();
     var stream = new ClientWritableStream(call, serialize);
     var metadata_batch = {};
     metadata_batch[grpc.opType.SEND_INITIAL_METADATA] =
@@ -478,7 +486,7 @@ function makeClientStreamRequestFunction(method, serialize, deserialize) {
       if (status.code === grpc.status.OK) {
         if (err) {
           // Got a batch error, but OK status. Something went wrong
-          callback(err);
+          args.callback(err);
           return;
         } else {
           try {
@@ -497,9 +505,9 @@ function makeClientStreamRequestFunction(method, serialize, deserialize) {
         error = new Error(response.status.details);
         error.code = status.code;
         error.metadata = status.metadata;
-        callback(error);
+        args.callback(error);
       } else {
-        callback(null, deserialized);
+        args.callback(null, deserialized);
       }
       stream.emit('status', status);
     });
@@ -530,17 +538,18 @@ function makeServerStreamRequestFunction(method, serialize, deserialize) {
    */
   function makeServerStreamRequest(argument, metadata, options) {
     /* jshint validthis: true */
-    var call = getCall(this.$channel, method, options);
-    if (metadata === null || metadata === undefined) {
-      metadata = new Metadata();
-    } else {
-      metadata = metadata.clone();
-    }
+    /* While the arguments are listed in the function signature, those variables
+     * are not used directly. Instead, ArgueJS processes the arguments
+     * object. */
+    var args = arguejs({argument: null, metadata: [Metadata, new Metadata()],
+                        options: [Object]}, arguments);
+    var call = getCall(this.$channel, method, args.options);
+    metadata = args.metadata.clone();
     var stream = new ClientReadableStream(call, deserialize);
     var start_batch = {};
-    var message = serialize(argument);
-    if (options) {
-      message.grpcWriteFlags = options.flags;
+    var message = serialize(args.argument);
+    if (args.options) {
+      message.grpcWriteFlags = args.options.flags;
     }
     start_batch[grpc.opType.SEND_INITIAL_METADATA] =
         metadata._getCoreRepresentation();
@@ -592,12 +601,13 @@ function makeBidiStreamRequestFunction(method, serialize, deserialize) {
    */
   function makeBidiStreamRequest(metadata, options) {
     /* jshint validthis: true */
-    var call = getCall(this.$channel, method, options);
-    if (metadata === null || metadata === undefined) {
-      metadata = new Metadata();
-    } else {
-      metadata = metadata.clone();
-    }
+    /* While the arguments are listed in the function signature, those variables
+     * are not used directly. Instead, ArgueJS processes the arguments
+     * object. */
+    var args = arguejs({metadata: [Metadata, new Metadata()],
+                        options: [Object]}, arguments);
+    var call = getCall(this.$channel, method, args.options);
+    metadata = args.metadata.clone();
     var stream = new ClientDuplexStream(call, serialize, deserialize);
     var start_batch = {};
     start_batch[grpc.opType.SEND_INITIAL_METADATA] =
@@ -640,6 +650,40 @@ var requester_makers = {
   bidi: makeBidiStreamRequestFunction
 };
 
+function getDefaultValues(metadata, options) {
+  var res = {};
+  res.metadata = metadata || new Metadata();
+  res.options = options || {};
+  return res;
+}
+
+/**
+ * Map with wrappers for each type of requester function to make it use the old
+ * argument order with optional arguments after the callback.
+ */
+var deprecated_request_wrap = {
+  unary: function(makeUnaryRequest) {
+    return function makeWrappedUnaryRequest(argument, callback,
+                                            metadata, options) {
+      /* jshint validthis: true */
+      var opt_args = getDefaultValues(metadata, metadata);
+      return makeUnaryRequest.call(this, argument, opt_args.metadata,
+                                   opt_args.options, callback);
+    };
+  },
+  client_stream: function(makeServerStreamRequest) {
+    return function makeWrappedClientStreamRequest(callback, metadata,
+                                                   options) {
+      /* jshint validthis: true */
+      var opt_args = getDefaultValues(metadata, options);
+      return makeServerStreamRequest.call(this, opt_args.metadata,
+                                          opt_args.options, callback);
+    };
+  },
+  server_stream: _.identity,
+  bidi: _.identity
+};
+
 /**
  * Creates a constructor for a client with the given methods. The methods object
  * maps method name to an object with the following keys:
@@ -651,9 +695,19 @@ var requester_makers = {
  * responseDeserialize: function to deserialize response objects
  * @param {Object} methods An object mapping method names to method attributes
  * @param {string} serviceName The fully qualified name of the service
+ * @param {Object} class_options An options object. Currently only uses the key
+ *     deprecatedArgumentOrder, a boolean that Indicates that the old argument
+ *     order should be used for methods, with optional arguments at the end
+ *     instead of the callback at the end. Defaults to false. This option is
+ *     only a temporary stopgap measure to smooth an API breakage.
+ *     It is deprecated, and new code should not use it.
  * @return {function(string, Object)} New client constructor
  */
-exports.makeClientConstructor = function(methods, serviceName) {
+exports.makeClientConstructor = function(methods, serviceName,
+                                         class_options) {
+  if (!class_options) {
+    class_options = {};
+  }
   /**
    * Create a client with the given methods
    * @constructor
@@ -700,8 +754,13 @@ exports.makeClientConstructor = function(methods, serviceName) {
     }
     var serialize = attrs.requestSerialize;
     var deserialize = attrs.responseDeserialize;
-    Client.prototype[name] = requester_makers[method_type](
+    var method_func = requester_makers[method_type](
         attrs.path, serialize, deserialize);
+    if (class_options.deprecatedArgumentOrder) {
+      Client.prototype[name] = deprecated_request_wrap(method_func);
+    } else {
+      Client.prototype[name] = method_func;
+    }
     // Associate all provided attributes with the method
     _.assign(Client.prototype[name], attrs);
   });
@@ -758,8 +817,13 @@ exports.waitForClientReady = function(client, deadline, callback) {
 exports.makeProtobufClientConstructor =  function(service, options) {
   var method_attrs = common.getProtobufServiceAttrs(service, service.name,
                                                     options);
+  var deprecatedArgumentOrder = false;
+  if (options) {
+    deprecatedArgumentOrder = options.deprecatedArgumentOrder;
+  }
   var Client = exports.makeClientConstructor(
-      method_attrs, common.fullyQualifiedName(service));
+      method_attrs, common.fullyQualifiedName(service),
+      deprecatedArgumentOrder);
   Client.service = service;
   Client.service.grpc_options = options;
   return Client;
diff --git a/src/node/src/credentials.js b/src/node/src/credentials.js
index 1d73723cc0..97c4bd73ac 100644
--- a/src/node/src/credentials.js
+++ b/src/node/src/credentials.js
@@ -118,7 +118,6 @@ exports.createFromMetadataGenerator = function(metadata_generator) {
 exports.createFromGoogleCredential = function(google_credential) {
   return exports.createFromMetadataGenerator(function(auth_context, callback) {
     var service_url = auth_context.service_url;
-    console.log('Service URL:', service_url);
     google_credential.getRequestMetadata(service_url, function(err, header) {
       if (err) {
         console.log('Auth error:', err);
@@ -127,7 +126,6 @@ exports.createFromGoogleCredential = function(google_credential) {
       }
       var metadata = new Metadata();
       metadata.add('authorization', header.Authorization);
-      console.log(header.Authorization);
       callback(null, metadata);
     });
   });
diff --git a/src/node/src/server.js b/src/node/src/server.js
index 0cf7ba3424..dd0bc12bc9 100644
--- a/src/node/src/server.js
+++ b/src/node/src/server.js
@@ -339,7 +339,7 @@ function _read(size) {
     try {
       deserialized = self.deserialize(data);
     } catch (e) {
-      e.code = grpc.status.INVALID_ARGUMENT;
+      e.code = grpc.status.INTERNAL;
       self.emit('error', e);
       return;
     }
@@ -475,7 +475,7 @@ function handleUnary(call, handler, metadata) {
     try {
       emitter.request = handler.deserialize(result.read);
     } catch (e) {
-      e.code = grpc.status.INVALID_ARGUMENT;
+      e.code = grpc.status.INTERNAL;
       handleError(call, e);
       return;
     }
@@ -516,7 +516,7 @@ function handleServerStreaming(call, handler, metadata) {
     try {
       stream.request = handler.deserialize(result.read);
     } catch (e) {
-      e.code = grpc.status.INVALID_ARGUMENT;
+      e.code = grpc.status.INTERNAL;
       stream.emit('error', e);
       return;
     }
diff --git a/src/node/test/credentials_test.js b/src/node/test/credentials_test.js
index 294600c85a..73eadfab2c 100644
--- a/src/node/test/credentials_test.js
+++ b/src/node/test/credentials_test.js
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -398,18 +398,20 @@ describe('client credentials', function() {
           metadataUpdater);
     });
     it('Should update metadata on a unary call', function(done) {
-      var call = client.unary({}, function(err, data) {
-        assert.ifError(err);
-      }, null, {credentials: updater_creds});
+      var call = client.unary({}, {credentials: updater_creds},
+                              function(err, data) {
+                                assert.ifError(err);
+                              });
       call.on('metadata', function(metadata) {
         assert.deepEqual(metadata.get('plugin_key'), ['plugin_value']);
         done();
       });
     });
     it('should update metadata on a client streaming call', function(done) {
-      var call = client.clientStream(function(err, data) {
-        assert.ifError(err);
-      }, null, {credentials: updater_creds});
+      var call = client.clientStream({credentials: updater_creds},
+                                     function(err, data) {
+                                       assert.ifError(err);
+                                     });
       call.on('metadata', function(metadata) {
         assert.deepEqual(metadata.get('plugin_key'), ['plugin_value']);
         done();
@@ -417,7 +419,7 @@ describe('client credentials', function() {
       call.end();
     });
     it('should update metadata on a server streaming call', function(done) {
-      var call = client.serverStream({}, null, {credentials: updater_creds});
+      var call = client.serverStream({}, {credentials: updater_creds});
       call.on('data', function() {});
       call.on('metadata', function(metadata) {
         assert.deepEqual(metadata.get('plugin_key'), ['plugin_value']);
@@ -425,7 +427,7 @@ describe('client credentials', function() {
       });
     });
     it('should update metadata on a bidi streaming call', function(done) {
-      var call = client.bidiStream(null, {credentials: updater_creds});
+      var call = client.bidiStream({credentials: updater_creds});
       call.on('data', function() {});
       call.on('metadata', function(metadata) {
         assert.deepEqual(metadata.get('plugin_key'), ['plugin_value']);
@@ -443,9 +445,10 @@ describe('client credentials', function() {
           altMetadataUpdater);
       var combined_updater = grpc.credentials.combineCallCredentials(
           updater_creds, alt_updater_creds);
-      var call = client.unary({}, function(err, data) {
-        assert.ifError(err);
-      }, null, {credentials: combined_updater});
+      var call = client.unary({}, {credentials: combined_updater},
+                              function(err, data) {
+                                assert.ifError(err);
+                              });
       call.on('metadata', function(metadata) {
         assert.deepEqual(metadata.get('plugin_key'), ['plugin_value']);
         assert.deepEqual(metadata.get('other_plugin_key'),
diff --git a/src/node/test/surface_test.js b/src/node/test/surface_test.js
index 8a232d6fc4..5a704ee133 100644
--- a/src/node/test/surface_test.js
+++ b/src/node/test/surface_test.js
@@ -404,18 +404,18 @@ describe('Echo metadata', function() {
     server.forceShutdown();
   });
   it('with unary call', function(done) {
-    var call = client.unary({}, function(err, data) {
+    var call = client.unary({}, metadata, function(err, data) {
       assert.ifError(err);
-    }, metadata);
+    });
     call.on('metadata', function(metadata) {
       assert.deepEqual(metadata.get('key'), ['value']);
       done();
     });
   });
   it('with client stream call', function(done) {
-    var call = client.clientStream(function(err, data) {
+    var call = client.clientStream(metadata, function(err, data) {
       assert.ifError(err);
-    }, metadata);
+    });
     call.on('metadata', function(metadata) {
       assert.deepEqual(metadata.get('key'), ['value']);
       done();
@@ -441,8 +441,8 @@ describe('Echo metadata', function() {
   });
   it('shows the correct user-agent string', function(done) {
     var version = require('../../../package.json').version;
-    var call = client.unary({}, function(err, data) { assert.ifError(err); },
-                            metadata);
+    var call = client.unary({}, metadata,
+                            function(err, data) { assert.ifError(err); });
     call.on('metadata', function(metadata) {
       assert(_.startsWith(metadata.get('user-agent')[0],
                           'grpc-node/' + version));
@@ -452,8 +452,8 @@ describe('Echo metadata', function() {
   it('properly handles duplicate values', function(done) {
     var dup_metadata = metadata.clone();
     dup_metadata.add('key', 'value2');
-    var call = client.unary({}, function(err, data) {assert.ifError(err); },
-                            dup_metadata);
+    var call = client.unary({}, dup_metadata,
+                            function(err, data) {assert.ifError(err); });
     call.on('metadata', function(resp_metadata) {
       // Two arrays are equal iff their symmetric difference is empty
       assert.deepEqual(_.xor(dup_metadata.get('key'), resp_metadata.get('key')),
@@ -709,14 +709,14 @@ describe('Other conditions', function() {
     it('should respond correctly to a unary call', function(done) {
       misbehavingClient.unary(badArg, function(err, data) {
         assert(err);
-        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        assert.strictEqual(err.code, grpc.status.INTERNAL);
         done();
       });
     });
     it('should respond correctly to a client stream', function(done) {
       var call = misbehavingClient.clientStream(function(err, data) {
         assert(err);
-        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        assert.strictEqual(err.code, grpc.status.INTERNAL);
         done();
       });
       call.write(badArg);
@@ -729,7 +729,7 @@ describe('Other conditions', function() {
         assert.fail(data, null, 'Unexpected data', '===');
       });
       call.on('error', function(err) {
-        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        assert.strictEqual(err.code, grpc.status.INTERNAL);
         done();
       });
     });
@@ -739,7 +739,7 @@ describe('Other conditions', function() {
         assert.fail(data, null, 'Unexpected data', '===');
       });
       call.on('error', function(err) {
-        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        assert.strictEqual(err.code, grpc.status.INTERNAL);
         done();
       });
       call.write(badArg);
@@ -954,7 +954,7 @@ describe('Call propagation', function() {
       done = multiDone(done, 2);
       var call;
       proxy_impl.unary = function(parent, callback) {
-        client.unary(parent.request, function(err, value) {
+        client.unary(parent.request, {parent: parent}, function(err, value) {
           try {
             assert(err);
             assert.strictEqual(err.code, grpc.status.CANCELLED);
@@ -962,7 +962,7 @@ describe('Call propagation', function() {
             callback(err, value);
             done();
           }
-        }, null, {parent: parent});
+        });
         call.cancel();
       };
       proxy.addProtoService(test_service, proxy_impl);
@@ -976,7 +976,7 @@ describe('Call propagation', function() {
       done = multiDone(done, 2);
       var call;
       proxy_impl.clientStream = function(parent, callback) {
-        client.clientStream(function(err, value) {
+        client.clientStream({parent: parent}, function(err, value) {
           try {
             assert(err);
             assert.strictEqual(err.code, grpc.status.CANCELLED);
@@ -984,7 +984,7 @@ describe('Call propagation', function() {
             callback(err, value);
             done();
           }
-        }, null, {parent: parent});
+        });
         call.cancel();
       };
       proxy.addProtoService(test_service, proxy_impl);
@@ -998,8 +998,7 @@ describe('Call propagation', function() {
       done = multiDone(done, 2);
       var call;
       proxy_impl.serverStream = function(parent) {
-        var child = client.serverStream(parent.request, null,
-                                        {parent: parent});
+        var child = client.serverStream(parent.request, {parent: parent});
         child.on('data', function() {});
         child.on('error', function(err) {
           assert(err);
@@ -1023,7 +1022,7 @@ describe('Call propagation', function() {
       done = multiDone(done, 2);
       var call;
       proxy_impl.bidiStream = function(parent) {
-        var child = client.bidiStream(null, {parent: parent});
+        var child = client.bidiStream({parent: parent});
         child.on('data', function() {});
         child.on('error', function(err) {
           assert(err);
@@ -1051,7 +1050,8 @@ describe('Call propagation', function() {
     it('With a client stream call', function(done) {
       done = multiDone(done, 2);
       proxy_impl.clientStream = function(parent, callback) {
-        client.clientStream(function(err, value) {
+        var options = {parent: parent, propagate_flags: deadline_flags};
+        client.clientStream(options, function(err, value) {
           try {
             assert(err);
             assert(err.code === grpc.status.DEADLINE_EXCEEDED ||
@@ -1060,7 +1060,7 @@ describe('Call propagation', function() {
             callback(err, value);
             done();
           }
-        }, null, {parent: parent, propagate_flags: deadline_flags});
+        });
       };
       proxy.addProtoService(test_service, proxy_impl);
       var proxy_port = proxy.bind('localhost:0', server_insecure_creds);
@@ -1069,15 +1069,15 @@ describe('Call propagation', function() {
                                     grpc.credentials.createInsecure());
       var deadline = new Date();
       deadline.setSeconds(deadline.getSeconds() + 1);
-      proxy_client.clientStream(function(err, value) {
+      proxy_client.clientStream({deadline: deadline}, function(err, value) {
         done();
-      }, null, {deadline: deadline});
+      });
     });
     it('With a bidi stream call', function(done) {
       done = multiDone(done, 2);
       proxy_impl.bidiStream = function(parent) {
         var child = client.bidiStream(
-            null, {parent: parent, propagate_flags: deadline_flags});
+            {parent: parent, propagate_flags: deadline_flags});
         child.on('data', function() {});
         child.on('error', function(err) {
           assert(err);
@@ -1093,7 +1093,7 @@ describe('Call propagation', function() {
                                     grpc.credentials.createInsecure());
       var deadline = new Date();
       deadline.setSeconds(deadline.getSeconds() + 1);
-      var call = proxy_client.bidiStream(null, {deadline: deadline});
+      var call = proxy_client.bidiStream({deadline: deadline});
       call.on('data', function() {});
       call.on('error', function(err) {
         done();
diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m
index f79b7d0bc0..51263b073c 100644
--- a/src/objective-c/GRPCClient/GRPCCall.m
+++ b/src/objective-c/GRPCClient/GRPCCall.m
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,6 +37,8 @@
 #include <grpc/support/time.h>
 #import <RxLibrary/GRXConcurrentWriteable.h>
 
+#import "private/GRPCConnectivityMonitor.h"
+#import "private/GRPCHost.h"
 #import "private/GRPCRequestHeaders.h"
 #import "private/GRPCWrappedCall.h"
 #import "private/NSData+GRPC.h"
@@ -71,8 +73,11 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
 @implementation GRPCCall {
   dispatch_queue_t _callQueue;
 
+  NSString *_host;
+  NSString *_path;
   GRPCWrappedCall *_wrappedCall;
   dispatch_once_t _callAlreadyInvoked;
+  GRPCConnectivityMonitor *_connectivityMonitor;
 
   // The C gRPC library has less guarantees on the ordering of events than we
   // do. Particularly, in the face of errors, there's no ordering guarantee at
@@ -115,13 +120,11 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
                 format:@"The requests writer can't be already started."];
   }
   if ((self = [super init])) {
-    _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:host path:path];
-    if (!_wrappedCall) {
-      return nil;
-    }
+    _host = [host copy];
+    _path = [path copy];
 
     // Serial queue to invoke the non-reentrant methods of the grpc_call object.
-    _callQueue = dispatch_queue_create("org.grpc.call", NULL);
+    _callQueue = dispatch_queue_create("io.grpc.call", NULL);
 
     _requestWriter = requestWriter;
 
@@ -156,7 +159,7 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
 - (void)cancel {
   [self finishWithError:[NSError errorWithDomain:kGRPCErrorDomain
                                             code:GRPCErrorCodeCancelled
-                                        userInfo:nil]];
+                                        userInfo:@{NSLocalizedDescriptionKey: @"Canceled by app"}]];
   [self cancelCall];
 }
 
@@ -305,37 +308,30 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
 }
 
 - (void)invokeCall {
-  __weak GRPCCall *weakSelf = self;
   [self invokeCallWithHeadersHandler:^(NSDictionary *headers) {
     // Response headers received.
-    GRPCCall *strongSelf = weakSelf;
-    if (strongSelf) {
-      strongSelf.responseHeaders = headers;
-      [strongSelf startNextRead];
-    }
+    self.responseHeaders = headers;
+    [self startNextRead];
   } completionHandler:^(NSError *error, NSDictionary *trailers) {
-    GRPCCall *strongSelf = weakSelf;
-    if (strongSelf) {
-      strongSelf.responseTrailers = trailers;
-
-      if (error) {
-        NSMutableDictionary *userInfo = [NSMutableDictionary dictionary];
-        if (error.userInfo) {
-          [userInfo addEntriesFromDictionary:error.userInfo];
-        }
-        userInfo[kGRPCTrailersKey] = strongSelf.responseTrailers;
-        // TODO(jcanizales): The C gRPC library doesn't guarantee that the headers block will be
-        // called before this one, so an error might end up with trailers but no headers. We
-        // shouldn't call finishWithError until ater both blocks are called. It is also when this is
-        // done that we can provide a merged view of response headers and trailers in a thread-safe
-        // way.
-        if (strongSelf.responseHeaders) {
-          userInfo[kGRPCHeadersKey] = strongSelf.responseHeaders;
-        }
-        error = [NSError errorWithDomain:error.domain code:error.code userInfo:userInfo];
+    self.responseTrailers = trailers;
+
+    if (error) {
+      NSMutableDictionary *userInfo = [NSMutableDictionary dictionary];
+      if (error.userInfo) {
+        [userInfo addEntriesFromDictionary:error.userInfo];
       }
-      [strongSelf finishWithError:error];
+      userInfo[kGRPCTrailersKey] = self.responseTrailers;
+      // TODO(jcanizales): The C gRPC library doesn't guarantee that the headers block will be
+      // called before this one, so an error might end up with trailers but no headers. We
+      // shouldn't call finishWithError until ater both blocks are called. It is also when this is
+      // done that we can provide a merged view of response headers and trailers in a thread-safe
+      // way.
+      if (self.responseHeaders) {
+        userInfo[kGRPCHeadersKey] = self.responseHeaders;
+      }
+      error = [NSError errorWithDomain:error.domain code:error.code userInfo:userInfo];
     }
+    [self finishWithError:error];
   }];
   // Now that the RPC has been initiated, request writes can start.
   @synchronized(_requestWriter) {
@@ -354,8 +350,28 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
   _retainSelf = self;
 
   _responseWriteable = [[GRXConcurrentWriteable alloc] initWithWriteable:writeable];
+
+  _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host path:_path];
+  NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?");
+
   [self sendHeaders:_requestHeaders];
   [self invokeCall];
+  // TODO(jcanizales): Extract this logic somewhere common.
+  NSString *host = [NSURL URLWithString:[@"https://" stringByAppendingString:_host]].host;
+  if (!host) {
+    // TODO(jcanizales): Check this on init.
+    [NSException raise:NSInvalidArgumentException format:@"host of %@ is nil", _host];
+  }
+  __weak typeof(self) weakSelf = self;
+  _connectivityMonitor = [GRPCConnectivityMonitor monitorWithHost:host];
+  [_connectivityMonitor handleLossWithHandler:^{
+    typeof(self) strongSelf = weakSelf;
+    if (strongSelf) {
+      [strongSelf finishWithError:[NSError errorWithDomain:kGRPCErrorDomain
+                                                      code:GRPCErrorCodeUnavailable
+                                                  userInfo:@{NSLocalizedDescriptionKey: @"Connectivity lost."}]];
+    }
+  }];
 }
 
 - (void)setState:(GRXWriterState)newState {
@@ -385,4 +401,5 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
       return;
   }
 }
+
 @end
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h
index 8661ae6f97..e49a6aca29 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.h
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.h
@@ -35,6 +35,7 @@
 
 #include <grpc/grpc.h>
 
+@class GRPCCompletionQueue;
 struct grpc_channel_credentials;
 
 
@@ -80,4 +81,6 @@ struct grpc_channel_credentials;
 + (nonnull GRPCChannel *)insecureChannelWithHost:(nonnull NSString *)host
                                      channelArgs:(nullable NSDictionary *)channelArgs;
 
+- (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path
+                              completionQueue:(nonnull GRPCCompletionQueue *)queue;
 @end
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m
index 7e55a473d7..d7de025e21 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.m
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.m
@@ -38,6 +38,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#import "GRPCCompletionQueue.h"
+
 /**
  * Returns @c grpc_channel_credentials from the specified @c path. If the file at the path could not
  * be read then NULL is returned. If NULL is returned, @c errorPtr may not be NULL if there are
@@ -205,4 +207,16 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
                                channelArgs:channelArgs];
 }
 
+- (grpc_call *)unmanagedCallWithPath:(NSString *)path
+                     completionQueue:(GRPCCompletionQueue *)queue {
+  return grpc_channel_create_call(_unmanagedChannel,
+                                  NULL, GRPC_PROPAGATE_DEFAULTS,
+                                  queue.unmanagedQueue,
+                                  path.UTF8String,
+                                  // Get "host" from "host:port"
+                                  // TODO(jcanizales): Use NSURLs throughout, to clarify these.
+                                  [_host componentsSeparatedByString:@":"][0].UTF8String,
+                                  gpr_inf_future(GPR_CLOCK_REALTIME), NULL);
+}
+
 @end
diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m
index ff3031678c..d89602f7cb 100644
--- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m
+++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m
@@ -38,7 +38,12 @@
 @implementation GRPCCompletionQueue
 
 + (instancetype)completionQueue {
-  return [[self alloc] init];
+  static GRPCCompletionQueue *singleton = nil;
+  static dispatch_once_t onceToken;
+  dispatch_once(&onceToken, ^{
+    singleton = [[self alloc] init];
+  });
+  return singleton;
 }
 
 - (instancetype)init {
diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h
new file mode 100644
index 0000000000..2fae410331
--- /dev/null
+++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h
@@ -0,0 +1,77 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#import <Foundation/Foundation.h>
+#import <SystemConfiguration/SystemConfiguration.h>
+
+@interface GRPCReachabilityFlags : NSObject
+
++ (nonnull instancetype)flagsWithFlags:(SCNetworkReachabilityFlags)flags;
+
+/**
+ * One accessor method to query each of the different flags. Example:
+
+@property(nonatomic, readonly) BOOL isCell;
+
+ */
+#define GRPC_XMACRO_ITEM(methodName, FlagName) \
+@property(nonatomic, readonly) BOOL methodName;
+
+#include "GRPCReachabilityFlagNames.xmacro.h"
+#undef GRPC_XMACRO_ITEM
+
+@property(nonatomic, readonly) BOOL isHostReachable;
+@end
+
+
+@interface GRPCConnectivityMonitor : NSObject
+
++ (nullable instancetype)monitorWithHost:(nonnull NSString *)hostName;
+
+- (nonnull instancetype)init NS_UNAVAILABLE;
+
+/**
+ * Queue on which callbacks will be dispatched. Default is the main queue. Set it before calling
+ * handleLossWithHandler:.
+ */
+// TODO(jcanizales): Default to a serial background queue instead.
+@property(nonatomic, strong, null_resettable) dispatch_queue_t queue;
+
+/**
+ * Calls handler every time the connectivity to this instance's host is lost. If this instance is
+ * released before that happens, the handler won't be called.
+ * Only one handler is active at a time, so if this method is called again before the previous
+ * handler has been called, it might never be called at all (or yes, if it has already been queued).
+ */
+- (void)handleLossWithHandler:(nonnull void (^)())handler;
+@end
diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m
new file mode 100644
index 0000000000..b4061bd5ef
--- /dev/null
+++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m
@@ -0,0 +1,192 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#import "GRPCConnectivityMonitor.h"
+
+#pragma mark Flags
+
+@implementation GRPCReachabilityFlags {
+  SCNetworkReachabilityFlags _flags;
+}
+
++ (instancetype)flagsWithFlags:(SCNetworkReachabilityFlags)flags {
+  return [[self alloc] initWithFlags:flags];
+}
+
+- (instancetype)initWithFlags:(SCNetworkReachabilityFlags)flags {
+  if ((self = [super init])) {
+    _flags = flags;
+  }
+  return self;
+}
+
+/*
+ * One accessor method implementation per flag. Example:
+
+- (BOOL)isCell { \
+  return !!(_flags & kSCNetworkReachabilityFlagsIsWWAN); \
+}
+
+ */
+#define GRPC_XMACRO_ITEM(methodName, FlagName) \
+- (BOOL)methodName { \
+  return !!(_flags & kSCNetworkReachabilityFlags ## FlagName); \
+}
+#include "GRPCReachabilityFlagNames.xmacro.h"
+#undef GRPC_XMACRO_ITEM
+
+- (BOOL)isHostReachable {
+  // Note: connectionOnDemand means it'll be reachable only if using the CFSocketStream API or APIs
+  // on top of it.
+  // connectionRequired means we can't tell until a connection is attempted (e.g. for VPN on
+  // demand).
+  return self.reachable && !self.interventionRequired && !self.connectionOnDemand;
+}
+
+- (NSString *)description {
+  NSMutableArray *activeOptions = [NSMutableArray arrayWithCapacity:9];
+
+  /*
+   * For each flag, add its name to the array if it's ON. Example:
+
+  if (self.isCell) {
+    [activeOptions addObject:@"isCell"];
+  }
+
+   */
+#define GRPC_XMACRO_ITEM(methodName, FlagName) \
+  if (self.methodName) { \
+    [activeOptions addObject:@#methodName]; \
+  }
+#include "GRPCReachabilityFlagNames.xmacro.h"
+#undef GRPC_XMACRO_ITEM
+
+  return activeOptions.count == 0 ? @"(none)" : [activeOptions componentsJoinedByString:@", "];
+}
+
+- (BOOL)isEqual:(id)object {
+  return [object isKindOfClass:[GRPCReachabilityFlags class]] &&
+      _flags == ((GRPCReachabilityFlags *)object)->_flags;
+}
+
+- (NSUInteger)hash {
+  return _flags;
+}
+@end
+
+#pragma mark Connectivity Monitor
+
+// Assumes the third argument is a block that accepts a GRPCReachabilityFlags object, and passes the
+// received ones to it.
+static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target,
+                                        SCNetworkReachabilityFlags flags,
+                                        void *info) {
+  #pragma unused (target)
+  // This can be called many times with the same info. The info is retained by SCNetworkReachability
+  // while this function is being executed.
+  void (^handler)(GRPCReachabilityFlags *) = (__bridge void (^)(GRPCReachabilityFlags *))info;
+  handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]);
+}
+
+@implementation GRPCConnectivityMonitor {
+  SCNetworkReachabilityRef _reachabilityRef;
+}
+
+- (nullable instancetype)initWithReachability:(nullable SCNetworkReachabilityRef)reachability {
+  if (!reachability) {
+    return nil;
+  }
+  if ((self = [super init])) {
+    _reachabilityRef = CFRetain(reachability);
+    _queue = dispatch_get_main_queue();
+  }
+  return self;
+}
+
++ (nullable instancetype)monitorWithHost:(nonnull NSString *)host {
+  const char *hostName = host.UTF8String;
+  if (!hostName) {
+    [NSException raise:NSInvalidArgumentException
+                format:@"host.UTF8String returns NULL for %@", host];
+  }
+  SCNetworkReachabilityRef reachability =
+      SCNetworkReachabilityCreateWithName(NULL, hostName);
+
+  GRPCConnectivityMonitor *returnValue = [[self alloc] initWithReachability:reachability];
+  if (reachability) {
+    CFRelease(reachability);
+  }
+  return returnValue;
+}
+
+- (void)handleLossWithHandler:(void (^)())handler {
+  [self startListeningWithHandler:^(GRPCReachabilityFlags *flags) {
+    if (!flags.isHostReachable) {
+      handler();
+    }
+  }];
+}
+
+- (void)startListeningWithHandler:(void (^)(GRPCReachabilityFlags *))handler {
+  // Copy to ensure the handler block is in the heap (and so can't be deallocated when this method
+  // returns).
+  void (^copiedHandler)(GRPCReachabilityFlags *) = [handler copy];
+  SCNetworkReachabilityContext context = {
+    .version = 0,
+    .info = (__bridge void *)copiedHandler,
+    .retain = CFRetain,
+    .release = CFRelease,
+  };
+  // The following will retain context.info, and release it when the callback is set to NULL.
+  SCNetworkReachabilitySetCallback(_reachabilityRef, PassFlagsToContextInfoBlock, &context);
+  SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, _queue);
+}
+
+- (void)stopListening {
+  // This releases the block on context.info.
+  SCNetworkReachabilitySetCallback(_reachabilityRef, NULL, NULL);
+  SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, NULL);
+}
+
+- (void)setQueue:(dispatch_queue_t)queue {
+  _queue = queue ?: dispatch_get_main_queue();
+}
+
+- (void)dealloc {
+  if (_reachabilityRef) {
+    [self stopListening];
+    CFRelease(_reachabilityRef);
+  }
+}
+
+@end
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h
index 82c0ad6cf6..987d3e9f59 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.h
+++ b/src/objective-c/GRPCClient/private/GRPCHost.h
@@ -33,27 +33,39 @@
 
 #import <Foundation/Foundation.h>
 
+NS_ASSUME_NONNULL_BEGIN
+
 @class GRPCCompletionQueue;
 struct grpc_call;
 
 @interface GRPCHost : NSObject
 
 @property(nonatomic, readonly) NSString *address;
-@property(nonatomic, copy) NSString *userAgentPrefix;
+@property(nonatomic, copy, nullable) NSString *userAgentPrefix;
 
 /** The following properties should only be modified for testing: */
 
 @property(nonatomic, getter=isSecure) BOOL secure;
 
-@property(nonatomic, copy) NSString *pathToCertificates;
-@property(nonatomic, copy) NSString *hostNameOverride;
+@property(nonatomic, copy, nullable) NSString *pathToCertificates;
+@property(nonatomic, copy, nullable) NSString *hostNameOverride;
 
+- (nullable instancetype)init NS_UNAVAILABLE;
 /** Host objects initialized with the same address are the same. */
-+ (instancetype)hostWithAddress:(NSString *)address;
-- (instancetype)initWithAddress:(NSString *)address NS_DESIGNATED_INITIALIZER;
++ (nullable instancetype)hostWithAddress:(NSString *)address;
+- (nullable instancetype)initWithAddress:(NSString *)address NS_DESIGNATED_INITIALIZER;
 
 /** Create a grpc_call object to the provided path on this host. */
-- (struct grpc_call *)unmanagedCallWithPath:(NSString *)path
-                            completionQueue:(GRPCCompletionQueue *)queue;
+- (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path
+                                     completionQueue:(GRPCCompletionQueue *)queue;
 
+// TODO: There's a race when a new RPC is coming through just as an existing one is getting
+// notified that there's no connectivity. If connectivity comes back at that moment, the new RPC
+// will have its channel destroyed by the other RPC, and will never get notified of a problem, so
+// it'll hang (the C layer logs a timeout, with exponential back off). One solution could be to pass
+// the GRPCChannel to the GRPCCall, renaming this as "disconnectChannel:channel", which would only
+// act on that specific channel.
+- (void)disconnect;
 @end
+
+NS_ASSUME_NONNULL_END
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m
index eb1db899b7..508cb20644 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.m
+++ b/src/objective-c/GRPCClient/private/GRPCHost.m
@@ -34,33 +34,30 @@
 #import "GRPCHost.h"
 
 #include <grpc/grpc.h>
+#import <GRPCClient/GRPCCall.h>
 #import <GRPCClient/GRPCCall+ChannelArg.h>
 
 #import "GRPCChannel.h"
 #import "GRPCCompletionQueue.h"
 #import "NSDictionary+GRPC.h"
 
+NS_ASSUME_NONNULL_BEGIN
+
 // TODO(jcanizales): Generate the version in a standalone header, from templates. Like
 // templates/src/core/surface/version.c.template .
 #define GRPC_OBJC_VERSION_STRING @"0.13.0"
 
-@interface GRPCHost ()
-// TODO(mlumish): Investigate whether caching channels with strong links is a good idea.
-@property(nonatomic, strong) GRPCChannel *channel;
-@end
-
-@implementation GRPCHost
-
-+ (instancetype)hostWithAddress:(NSString *)address {
-  return [[self alloc] initWithAddress:address];
+@implementation GRPCHost {
+  // TODO(mlumish): Investigate whether caching channels with strong links is a good idea.
+  GRPCChannel *_channel;
 }
 
-- (instancetype)init {
-  return [self initWithAddress:nil];
++ (nullable instancetype)hostWithAddress:(NSString *)address {
+  return [[self alloc] initWithAddress:address];
 }
 
 // Default initializer.
-- (instancetype)initWithAddress:(NSString *)address {
+- (nullable instancetype)initWithAddress:(NSString *)address {
   if (!address) {
     return nil;
   }
@@ -95,46 +92,45 @@
   return self;
 }
 
-- (grpc_call *)unmanagedCallWithPath:(NSString *)path completionQueue:(GRPCCompletionQueue *)queue {
-  if (!queue || !path || !self.channel) {
-    return NULL;
+- (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path
+                              completionQueue:(GRPCCompletionQueue *)queue {
+  GRPCChannel *channel;
+  // This is racing -[GRPCHost disconnect].
+  @synchronized(self) {
+    if (!_channel) {
+      _channel = [self newChannel];
+    }
+    channel = _channel;
   }
-  return grpc_channel_create_call(self.channel.unmanagedChannel,
-                                  NULL, GRPC_PROPAGATE_DEFAULTS,
-                                  queue.unmanagedQueue,
-                                  path.UTF8String,
-                                  self.hostName.UTF8String,
-                                  gpr_inf_future(GPR_CLOCK_REALTIME), NULL);
+  return [channel unmanagedCallWithPath:path completionQueue:queue];
 }
 
-- (GRPCChannel *)channel {
-  // Create it lazily, because we don't want to open a connection just because someone is
-  // configuring a host.
+- (NSDictionary *)channelArgs {
+  NSMutableDictionary *args = [NSMutableDictionary dictionary];
 
-  if (!_channel) {
-    NSMutableDictionary *args = [NSMutableDictionary dictionary];
+  // TODO(jcanizales): Add OS and device information (see
+  // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#user-agents ).
+  NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING;
+  if (_userAgentPrefix) {
+    userAgent = [_userAgentPrefix stringByAppendingFormat:@" %@", userAgent];
+  }
+  args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent;
 
-    // TODO(jcanizales): Add OS and device information (see
-    // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#user-agents ).
-    NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING;
-    if (_userAgentPrefix) {
-      userAgent = [@[_userAgentPrefix, userAgent] componentsJoinedByString:@" "];
-    }
-    args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent;
-
-    if (_secure) {
-      if (_hostNameOverride) {
-        args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = _hostNameOverride;
-      }
-
-      _channel = [GRPCChannel secureChannelWithHost:_address
-                                 pathToCertificates:_pathToCertificates
-                                        channelArgs:args];
-    } else {
-      _channel = [GRPCChannel insecureChannelWithHost:_address channelArgs:args];
-    }
+  if (_secure && _hostNameOverride) {
+    args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = _hostNameOverride;
+  }
+  return args;
+}
+
+- (GRPCChannel *)newChannel {
+  NSDictionary *args = [self channelArgs];
+  if (_secure) {
+    return [GRPCChannel secureChannelWithHost:_address
+                           pathToCertificates:_pathToCertificates
+                                  channelArgs:args];
+  } else {
+    return [GRPCChannel insecureChannelWithHost:_address channelArgs:args];
   }
-  return _channel;
 }
 
 - (NSString *)hostName {
@@ -142,7 +138,16 @@
   return _hostNameOverride ?: _address;
 }
 
+- (void)disconnect {
+  // This is racing -[GRPCHost unmanagedCallWithPath:completionQueue:].
+  @synchronized(self) {
+    _channel = nil;
+  }
+}
+
 // TODO(jcanizales): Don't let set |secure| to |NO| if |pathToCertificates| or |hostNameOverride|
 // have been set. Don't let set either of the latter if |secure| has been set to |NO|.
 
 @end
+
+NS_ASSUME_NONNULL_END
diff --git a/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h b/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h
new file mode 100644
index 0000000000..4b92504b55
--- /dev/null
+++ b/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h
@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/**
+ * "X-macro" file that lists the flags names of Apple's Network Reachability API, along with a nice
+ * Objective-C method name used to query each of them.
+ *
+ * Example usage: To generate a dictionary from flag value to name, one can do:
+
+  NSDictionary *flagNames = @{
+#define GRPC_XMACRO_ITEM(methodName, FlagName) \
+    @(kSCNetworkReachabilityFlags ## FlagName): @#methodName,
+#include "GRXReachabilityFlagNames.xmacro.h"
+#undef GRPC_XMACRO_ITEM
+  };
+
+  XCTAssertEqualObjects(flagNames[@(kSCNetworkReachabilityFlagsIsWWAN)], @"isCell");
+
+ */
+
+#ifndef GRPC_XMACRO_ITEM
+#error This file is to be used with the "X-macro" pattern: Please #define \
+       GRPC_XMACRO_ITEM(methodName, FlagName), then #include this file, and then #undef \
+       GRPC_XMACRO_ITEM.
+#endif
+
+#if TARGET_OS_IPHONE
+GRPC_XMACRO_ITEM(isCell, IsWWAN)
+#endif
+GRPC_XMACRO_ITEM(reachable, Reachable)
+GRPC_XMACRO_ITEM(transientConnection, TransientConnection)
+GRPC_XMACRO_ITEM(connectionRequired, ConnectionRequired)
+GRPC_XMACRO_ITEM(connectionOnTraffic, ConnectionOnTraffic)
+GRPC_XMACRO_ITEM(interventionRequired, InterventionRequired)
+GRPC_XMACRO_ITEM(connectionOnDemand, ConnectionOnDemand)
+GRPC_XMACRO_ITEM(isLocalAddress, IsLocalAddress)
+GRPC_XMACRO_ITEM(isDirect, IsDirect)
diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h
index 71e7e0e54e..e37ed1b59f 100644
--- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h
+++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h
@@ -34,7 +34,6 @@
 #import <Foundation/Foundation.h>
 #include <grpc/grpc.h>
 
-#import "GRPCChannel.h"
 #import "GRPCRequestHeaders.h"
 
 @interface GRPCOperation : NSObject
@@ -94,4 +93,5 @@
 - (void)startBatchWithOperations:(NSArray *)ops;
 
 - (void)cancel;
+
 @end
diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
index fe3d51da53..f6527e283c 100644
--- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
+++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -54,7 +54,9 @@
 
 - (void)finish {
   if (_handler) {
-    _handler();
+    void(^handler)() = _handler;
+    _handler = nil;
+    handler();
   }
 }
 @end
diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m
index 2729d62b72..028ba9b551 100644
--- a/src/objective-c/RxLibrary/GRXWriteable.m
+++ b/src/objective-c/RxLibrary/GRXWriteable.m
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -42,11 +42,42 @@
   if (!handler) {
     return [[self alloc] init];
   }
-  return [[self alloc] initWithValueHandler:^(id value) {
-    handler(value, nil);
-  } completionHandler:^(NSError *errorOrNil) {
-    if (errorOrNil) {
-      handler(nil, errorOrNil);
+  // We nilify this variable when the block is invoked, so that handler is only invoked once even if
+  // the writer tries to write multiple values.
+  __block GRXEventHandler eventHandler = ^(BOOL done, id value, NSError *error) {
+    // Nillify eventHandler before invoking handler, in case the latter causes the former to be
+    // executed recursively. Because blocks can be deallocated even during execution, we have to
+    // first retain handler locally to guarantee it's valid.
+    // TODO(jcanizales): Just turn this craziness into a simple subclass of GRXWriteable.
+    GRXSingleHandler singleHandler = handler;
+    eventHandler = nil;
+
+    if (value) {
+      singleHandler(value, nil);
+    } else if (error) {
+      singleHandler(nil, error);
+    } else {
+      NSDictionary *userInfo = @{
+        NSLocalizedDescriptionKey: @"The writer finished without producing any value."
+      };
+      // Even though RxLibrary is independent of gRPC, the domain and code here are, for the moment,
+      // set to the values of kGRPCErrorDomain and GRPCErrorCodeInternal. This way, the error formed
+      // is the one user of gRPC would expect if the server failed to produce a response.
+      //
+      // TODO(jcanizales): Figure out a way to keep errors of RxLibrary generic without making users
+      // of gRPC take care of two different error domains and error code enums. A possibility is to
+      // add error handling to GRXWriters or GRXWriteables, and use them to translate errors between
+      // the two domains.
+      static NSString *kGRPCErrorDomain = @"io.grpc";
+      static NSUInteger kGRPCErrorCodeInternal = 13;
+      singleHandler(nil, [NSError errorWithDomain:kGRPCErrorDomain
+                                             code:kGRPCErrorCodeInternal
+                                         userInfo:userInfo]);
+    }
+  };
+  return [self writeableWithEventHandler:^(BOOL done, id value, NSError *error) {
+    if (eventHandler) {
+      eventHandler(done, value, error);
     }
   }];
 }
diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m
index 624958f4b9..7dd6873c80 100644
--- a/src/objective-c/tests/GRPCClientTests.m
+++ b/src/objective-c/tests/GRPCClientTests.m
@@ -273,10 +273,12 @@ static ProtoMethod *kUnaryCallMethod;
   id<GRXWriteable> responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) {
     XCTAssertNotNil(value, @"nil value received as response.");
     XCTAssertEqual([value length], 0, @"Non-empty response received: %@", value);
+    /* This test needs to be more clever in regards to changing the version of the core.
     XCTAssertEqualObjects(call.responseHeaders[@"x-grpc-test-echo-useragent"],
                           @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)",
                           @"Did not receive expected user agent %@",
                           call.responseHeaders[@"x-grpc-test-echo-useragent"]);
+    */
     [response fulfill];
   } completionHandler:^(NSError *errorOrNil) {
     XCTAssertNil(errorOrNil, @"Finished with unexpected error: %@", errorOrNil);
diff --git a/src/objective-c/tests/RxLibraryUnitTests.m b/src/objective-c/tests/RxLibraryUnitTests.m
index d342662814..ae9465f58c 100644
--- a/src/objective-c/tests/RxLibraryUnitTests.m
+++ b/src/objective-c/tests/RxLibraryUnitTests.m
@@ -64,6 +64,8 @@
 }
 @end
 
+// TODO(jcanizales): Split into one file per tested class.
+
 @interface RxLibraryUnitTests : XCTestCase
 @end
 
@@ -79,6 +81,7 @@
   // If:
   id<GRXWriteable> writeable = [GRXWriteable writeableWithSingleHandler:handler.block];
   [writeable writeValue:anyValue];
+  [writeable writesFinishedWithError:nil];
 
   // Then:
   XCTAssertEqual(handler.timesCalled, 1);
@@ -101,6 +104,54 @@
   XCTAssertEqualObjects(handler.errorOrNil, anyError);
 }
 
+- (void)testWriteableSingleHandlerIsCalledOnlyOnce_ValueThenError {
+  // Given:
+  CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler];
+  id anyValue = @7;
+  NSError *anyError = [NSError errorWithDomain:@"domain" code:7 userInfo:nil];
+
+  // If:
+  id<GRXWriteable> writeable = [GRXWriteable writeableWithSingleHandler:handler.block];
+  [writeable writeValue:anyValue];
+  [writeable writesFinishedWithError:anyError];
+
+  // Then:
+  XCTAssertEqual(handler.timesCalled, 1);
+  XCTAssertEqualObjects(handler.value, anyValue);
+  XCTAssertEqualObjects(handler.errorOrNil, nil);
+}
+
+- (void)testWriteableSingleHandlerIsCalledOnlyOnce_ValueThenValue {
+  // Given:
+  CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler];
+  id anyValue = @7;
+
+  // If:
+  id<GRXWriteable> writeable = [GRXWriteable writeableWithSingleHandler:handler.block];
+  [writeable writeValue:anyValue];
+  [writeable writeValue:anyValue];
+  [writeable writesFinishedWithError:nil];
+
+  // Then:
+  XCTAssertEqual(handler.timesCalled, 1);
+  XCTAssertEqualObjects(handler.value, anyValue);
+  XCTAssertEqualObjects(handler.errorOrNil, nil);
+}
+
+- (void)testWriteableSingleHandlerFailsOnEmptyWriter {
+  // Given:
+  CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler];
+
+  // If:
+  id<GRXWriteable> writeable = [GRXWriteable writeableWithSingleHandler:handler.block];
+  [writeable writesFinishedWithError:nil];
+
+  // Then:
+  XCTAssertEqual(handler.timesCalled, 1);
+  XCTAssertEqualObjects(handler.value, nil);
+  XCTAssertNotNil(handler.errorOrNil);
+}
+
 #pragma mark BufferedPipe
 
 - (void)testBufferedPipePropagatesValue {
diff --git a/src/php/composer.json b/src/php/composer.json
index 1d41f847ac..01674a25db 100644
--- a/src/php/composer.json
+++ b/src/php/composer.json
@@ -1,7 +1,9 @@
 {
   "name": "grpc/grpc",
+  "type": "library",
   "description": "gRPC library for PHP",
-  "version": "0.6.0",
+  "version": "0.14.0",
+  "keywords": ["rpc"],
   "homepage": "http://grpc.io",
   "license": "BSD-3-Clause",
   "repositories": [
@@ -13,7 +15,7 @@
   "require": {
     "php": ">=5.5.0",
     "datto/protobuf-php": "dev-master",
-    "google/auth": "dev-master"
+    "google/auth": "v0.7"
   },
   "autoload": {
     "psr-4": {
diff --git a/src/php/ext/grpc/README.md b/src/php/ext/grpc/README.md
deleted file mode 100644
index 6e1cb2002f..0000000000
--- a/src/php/ext/grpc/README.md
+++ /dev/null
@@ -1,67 +0,0 @@
-gRPC PHP Extension
-==================
-
-# Requirements
-
- * PHP 5.5+
- * [gRPC core library](https://github.com/grpc/grpc) 0.11.0
-
-# Installation
-
-## Install PHP 5
-
-```
-$ sudo apt-get install git php5 php5-dev php-pear unzip
-```
-
-## Compile gRPC Core Library
-
-Clone the gRPC source code repository
-
-```
-$ git clone https://github.com/grpc/grpc.git
-```
-
-Build and install the gRPC C core libraries
-
-```sh
-$ cd grpc
-$ git checkout --track origin/release-0_11
-$ git pull --recurse-submodules && git submodule update --init --recursive
-$ make
-$ sudo make install
-```
-
-Note: you may encounter a warning about the Protobuf compiler `protoc` 3.0.0+ not being installed. The following might help, and will be useful later on when we need to compile the `protoc-gen-php` tool.
-
-```sh
-$ cd grpc/third_party/protobuf
-$ sudo make install   # 'make' should have been run by core grpc
-```
-
-## Install the gRPC PHP extension
-
-Quick install
-
-```sh
-$ sudo pecl install grpc
-```
-
-Note: before a stable release, you may need to do
-
-```sh
-$ sudo pecl install grpc-beta
-```
-
-OR
-
-Compile from source
-
-```sh
-$ # from grpc
-$ cd src/php/ext/grpc
-$ phpize
-$ ./configure
-$ make
-$ sudo make install
-```
diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c
index f0bc7340ba..b7e7c26c10 100644
--- a/src/php/ext/grpc/channel.c
+++ b/src/php/ext/grpc/channel.c
@@ -110,9 +110,11 @@ void php_grpc_read_args_array(zval *args_array, grpc_channel_args *args) {
     switch (Z_TYPE_P(*data)) {
       case IS_LONG:
         args->args[args_index].value.integer = (int)Z_LVAL_P(*data);
+        args->args[args_index].type = GRPC_ARG_INTEGER;
         break;
       case IS_STRING:
         args->args[args_index].value.string = Z_STRVAL_P(*data);
+        args->args[args_index].type = GRPC_ARG_STRING;
         break;
       default:
         zend_throw_exception(spl_ce_InvalidArgumentException,
diff --git a/src/php/tests/generated_code/math_client.php b/src/php/tests/generated_code/math_client.php
index 76ccabc068..2085560d19 100644
--- a/src/php/tests/generated_code/math_client.php
+++ b/src/php/tests/generated_code/math_client.php
@@ -1,7 +1,7 @@
 <?php
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -43,7 +43,9 @@ function p($line)
 
 $host = 'localhost:50051';
 p("Connecting to host: $host");
-$client = new math\MathClient($host, []);
+$client = new math\MathClient($host, [
+    'credentials' => Grpc\ChannelCredentials::createInsecure()
+]);
 p('Client class: '.get_class($client));
 p('');
 
diff --git a/src/proto/grpc/testing/echo_messages.proto b/src/proto/grpc/testing/echo_messages.proto
index d05a35548d..5ce0a1fd64 100644
--- a/src/proto/grpc/testing/echo_messages.proto
+++ b/src/proto/grpc/testing/echo_messages.proto
@@ -42,6 +42,7 @@ message RequestParams {
   bool echo_peer = 7;
   string expected_client_identity = 8; // will force check_auth_context.
   bool skip_cancelled_check = 9;
+  string expected_transport_security_type = 10;
 }
 
 message EchoRequest {
diff --git a/src/proto/grpc/testing/metrics.proto b/src/proto/grpc/testing/metrics.proto
index 4485d3a53b..df719afd99 100644
--- a/src/proto/grpc/testing/metrics.proto
+++ b/src/proto/grpc/testing/metrics.proto
@@ -1,5 +1,4 @@
-
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -28,12 +27,17 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-// An integration test service that covers all the method signature permutations
-// of unary/streaming requests/responses.
+// Contains the definitions for a metrics service and the type of metrics
+// exposed by the service.
+//
+// Currently, 'Gauge' (i.e a metric that represents the measured value of
+// something at an instant of time) is the only metric type supported by the
+// service.
 syntax = "proto3";
 
 package grpc.testing;
 
+// Reponse message containing the gauge name and value
 message GaugeResponse {
   string name = 1;
   oneof value {
@@ -43,11 +47,18 @@ message GaugeResponse {
   }
 }
 
-message GaugeRequest { string name = 1; }
+// Request message containing the gauge name
+message GaugeRequest {
+  string name = 1;
+}
 
 message EmptyMessage {}
 
 service MetricsService {
+  // Returns the values of all the gauges that are currently being maintained by
+  // the service
   rpc GetAllGauges(EmptyMessage) returns (stream GaugeResponse);
+
+  // Returns the value of one gauge
   rpc GetGauge(GaugeRequest) returns (GaugeResponse);
 }
diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst
index f3e962c197..33a462b66f 100644
--- a/src/python/grpcio/README.rst
+++ b/src/python/grpcio/README.rst
@@ -6,7 +6,7 @@ Package for gRPC Python.
 Installation
 ------------
 
-gRPC Python is available for Linux and Mac OS X running Python 2.7.
+gRPC Python is available for Linux, Mac OS X, and Windows running Python 2.7.
 
 From PyPI
 ~~~~~~~~~
@@ -23,21 +23,26 @@ Else system wide (on Ubuntu)...
 
   $ sudo pip install grpcio
 
+n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip`
+to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest
+version!
+
 From Source
 ~~~~~~~~~~~
 
 Building from source requires that you have the Python headers (usually a
-package named `python-dev`).
+package named :code:`python-dev`).
 
 ::
 
-  $ export REPO_ROOT=grpc
+  $ export REPO_ROOT=grpc  # REPO_ROOT can be any directory of your choice
   $ git clone https://github.com/grpc/grpc.git $REPO_ROOT
   $ cd $REPO_ROOT
-  $ pip install .
 
-Note that `$REPO_ROOT` can be assigned to whatever directory name floats your
-fancy.
+  # For the next two commands do `sudo pip install` if you get permission-denied errors
+  $ pip install -rrequirements.txt
+  $ GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+
 
 Troubleshooting
 ~~~~~~~~~~~~~~~
@@ -45,10 +50,43 @@ Troubleshooting
 Help, I ...
 
 * **... see a** :code:`pkg_resources.VersionConflict` **when I try to install
-  grpc!**
+  grpc**
 
   This is likely because :code:`pip` doesn't own the offending dependency,
   which in turn is likely because your operating system's package manager owns
   it. You'll need to force the installation of the dependency:
 
   :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY`
+
+  For example, if you get an error like the following:
+
+  ::
+
+    Traceback (most recent call last):
+    File "<string>", line 17, in <module>
+     ...
+    File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find
+      raise VersionConflict(dist, req)
+    pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10'))
+
+  You can fix it by doing:
+
+  ::
+
+    sudo pip install --ignore-installed six
+
+* **... see the following error on some platforms**
+
+  ::
+
+    /tmp/pip-build-U8pSsr/cython/Cython/Plex/Scanners.c:4:20: fatal error: Python.h: No such file or directory
+    #include "Python.h"
+                    ^
+    compilation terminated.
+
+  You can fix it by installing `python-dev` package. i.e
+
+  ::
+
+    sudo apt-get install python-dev
+
diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py
index 3207ea3052..1d43547419 100644
--- a/src/python/grpcio/commands.py
+++ b/src/python/grpcio/commands.py
@@ -119,8 +119,7 @@ class SphinxDocumentation(setuptools.Command):
     import sphinx
     import sphinx.apidoc
     metadata = self.distribution.metadata
-    src_dir = os.path.join(
-        PYTHON_STEM, self.distribution.package_dir[''], 'grpc')
+    src_dir = os.path.join(PYTHON_STEM, 'grpc')
     sys.path.append(src_dir)
     sphinx.apidoc.main([
         '', '--force', '--full', '-H', metadata.name, '-A', metadata.author,
@@ -289,15 +288,9 @@ class TestLite(setuptools.Command):
       sys.exit('Test failure')
 
   def _add_eggs_to_path(self):
-    """Adds all egg files under .eggs to sys.path"""
-    # TODO(jtattemusch): there has to be a cleaner way to do this
-    import pkg_resources
-    eggs_dir = os.path.join(PYTHON_STEM, '../../../.eggs')
-    eggs = [os.path.join(eggs_dir, filename)
-            for filename in os.listdir(eggs_dir)
-            if filename.endswith('.egg')]
-    for egg in eggs:
-      sys.path.insert(0, pkg_resources.normalize_path(egg))
+    """Fetch install and test requirements"""
+    self.distribution.fetch_build_eggs(self.distribution.install_requires)
+    self.distribution.fetch_build_eggs(self.distribution.tests_require)
 
 
 class RunInterop(test.test):
diff --git a/src/python/grpcio/grpc/_adapter/_types.py b/src/python/grpcio/grpc/_adapter/_types.py
index 3d5ab33d00..f9e18f0bb3 100644
--- a/src/python/grpcio/grpc/_adapter/_types.py
+++ b/src/python/grpcio/grpc/_adapter/_types.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@ import abc
 import collections
 import enum
 
+import six
+
 from grpc._cython import cygrpc
 
 
@@ -247,8 +249,7 @@ class Event(collections.namedtuple(
   """
 
 
-class CompletionQueue:
-  __metaclass__ = abc.ABCMeta
+class CompletionQueue(six.with_metaclass(abc.ABCMeta)):
 
   @abc.abstractmethod
   def __init__(self):
@@ -262,6 +263,9 @@ class CompletionQueue:
     """
     return self
 
+  def __next__(self):
+    return self.next()
+
   @abc.abstractmethod
   def next(self, deadline=float('+inf')):
     """Get the next event on this completion queue.
@@ -285,8 +289,7 @@ class CompletionQueue:
     return None
 
 
-class Call:
-  __metaclass__ = abc.ABCMeta
+class Call(six.with_metaclass(abc.ABCMeta)):
 
   @abc.abstractmethod
   def start_batch(self, ops, tag):
@@ -334,8 +337,7 @@ class Call:
     return None
 
 
-class Channel:
-  __metaclass__ = abc.ABCMeta
+class Channel(six.with_metaclass(abc.ABCMeta)):
 
   @abc.abstractmethod
   def __init__(self, target, args, credentials=None):
@@ -399,8 +401,7 @@ class Channel:
     return None
 
 
-class Server:
-  __metaclass__ = abc.ABCMeta
+class Server(six.with_metaclass(abc.ABCMeta)):
 
   @abc.abstractmethod
   def __init__(self, completion_queue, args):
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
index 80f4da51e8..d1b9c98ffc 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -40,14 +40,17 @@ cdef class Call:
   def start_batch(self, operations, tag):
     if not self.is_valid:
       raise ValueError("invalid call object cannot be used from Python")
+    cdef grpc_call_error result
     cdef Operations cy_operations = Operations(operations)
     cdef OperationTag operation_tag = OperationTag(tag)
     operation_tag.operation_call = self
     operation_tag.batch_operations = cy_operations
     cpython.Py_INCREF(operation_tag)
-    return grpc_call_start_batch(
-        self.c_call, cy_operations.c_ops, cy_operations.c_nops,
-        <cpython.PyObject *>operation_tag, NULL)
+    with nogil:
+      result = grpc_call_start_batch(
+          self.c_call, cy_operations.c_ops, cy_operations.c_nops,
+          <cpython.PyObject *>operation_tag, NULL)
+    return result
 
   def cancel(
       self, grpc_status_code error_code=GRPC_STATUS__DO_NOT_USE,
@@ -57,6 +60,8 @@ cdef class Call:
     if (details is None) != (error_code == GRPC_STATUS__DO_NOT_USE):
       raise ValueError("if error_code is specified, so must details "
                        "(and vice-versa)")
+    cdef grpc_call_error result
+    cdef char *c_details = NULL
     if error_code != GRPC_STATUS__DO_NOT_USE:
       if isinstance(details, bytes):
         pass
@@ -65,25 +70,37 @@ cdef class Call:
       else:
         raise TypeError("expected details to be str or bytes")
       self.references.append(details)
-      return grpc_call_cancel_with_status(
-          self.c_call, error_code, details, NULL)
+      c_details = details
+      with nogil:
+        result = grpc_call_cancel_with_status(
+            self.c_call, error_code, c_details, NULL)
+      return result
     else:
-      return grpc_call_cancel(self.c_call, NULL)
+      with nogil:
+        result = grpc_call_cancel(self.c_call, NULL)
+      return result
 
   def set_credentials(
       self, CallCredentials call_credentials not None):
-    return grpc_call_set_credentials(
-        self.c_call, call_credentials.c_credentials)
+    cdef grpc_call_error result
+    with nogil:
+      result = grpc_call_set_credentials(
+          self.c_call, call_credentials.c_credentials)
+    return result
 
   def peer(self):
-    cdef char *peer = grpc_call_get_peer(self.c_call)
+    cdef char *peer = NULL
+    with nogil:
+      peer = grpc_call_get_peer(self.c_call)
     result = <bytes>peer
-    gpr_free(peer)
+    with nogil:
+      gpr_free(peer)
     return result
 
   def __dealloc__(self):
     if self.c_call != NULL:
-      grpc_call_destroy(self.c_call)
+      with nogil:
+        grpc_call_destroy(self.c_call)
 
   # The object *should* always be valid from Python. Used for debugging.
   @property
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
index 1f1833d5ec..d612c90791 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
@@ -35,6 +35,7 @@ cdef class Channel:
   def __cinit__(self, target, ChannelArgs arguments=None,
                 ChannelCredentials channel_credentials=None):
     cdef grpc_channel_args *c_arguments = NULL
+    cdef char *c_target = NULL
     self.c_channel = NULL
     self.references = []
     if arguments is not None:
@@ -45,12 +46,15 @@ cdef class Channel:
       target = target.encode()
     else:
       raise TypeError("expected target to be str or bytes")
+    c_target = target
     if channel_credentials is None:
-      self.c_channel = grpc_insecure_channel_create(target, c_arguments,
-                                                         NULL)
+      with nogil:
+        self.c_channel = grpc_insecure_channel_create(c_target, c_arguments,
+                                                      NULL)
     else:
-      self.c_channel = grpc_secure_channel_create(
-          channel_credentials.c_credentials, target, c_arguments, NULL)
+      with nogil:
+        self.c_channel = grpc_secure_channel_create(
+            channel_credentials.c_credentials, c_target, c_arguments, NULL)
       self.references.append(channel_credentials)
     self.references.append(target)
     self.references.append(arguments)
@@ -66,6 +70,7 @@ cdef class Channel:
       method = method.encode()
     else:
       raise TypeError("expected method to be str or bytes")
+    cdef char *method_c_string = method
     cdef char *host_c_string = NULL
     if host is None:
       pass
@@ -81,32 +86,40 @@ cdef class Channel:
     cdef grpc_call *parent_call = NULL
     if parent is not None:
       parent_call = parent.c_call
-    operation_call.c_call = grpc_channel_create_call(
-        self.c_channel, parent_call, flags,
-        queue.c_completion_queue, method, host_c_string, deadline.c_time,
-        NULL)
+    with nogil:
+      operation_call.c_call = grpc_channel_create_call(
+          self.c_channel, parent_call, flags,
+          queue.c_completion_queue, method_c_string, host_c_string,
+          deadline.c_time, NULL)
     return operation_call
 
   def check_connectivity_state(self, bint try_to_connect):
-    return grpc_channel_check_connectivity_state(self.c_channel,
-                                                 try_to_connect)
+    cdef grpc_connectivity_state result
+    with nogil:
+      result = grpc_channel_check_connectivity_state(self.c_channel,
+                                                     try_to_connect)
+    return result
 
   def watch_connectivity_state(
       self, grpc_connectivity_state last_observed_state,
       Timespec deadline not None, CompletionQueue queue not None, tag):
     cdef OperationTag operation_tag = OperationTag(tag)
-    operation_tag.references = [self, queue]
     cpython.Py_INCREF(operation_tag)
-    grpc_channel_watch_connectivity_state(
-        self.c_channel, last_observed_state, deadline.c_time,
-        queue.c_completion_queue, <cpython.PyObject *>operation_tag)
+    with nogil:
+      grpc_channel_watch_connectivity_state(
+          self.c_channel, last_observed_state, deadline.c_time,
+          queue.c_completion_queue, <cpython.PyObject *>operation_tag)
 
   def target(self):
-    cdef char * target = grpc_channel_get_target(self.c_channel)
+    cdef char *target = NULL
+    with nogil:
+      target = grpc_channel_get_target(self.c_channel)
     result = <bytes>target
-    gpr_free(target)
+    with nogil:
+      gpr_free(target)
     return result
 
   def __dealloc__(self):
     if self.c_channel != NULL:
-      grpc_channel_destroy(self.c_channel)
+      with nogil:
+        grpc_channel_destroy(self.c_channel)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
index c139147114..09e47d4222 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
@@ -36,7 +36,8 @@ import time
 cdef class CompletionQueue:
 
   def __cinit__(self):
-    self.c_completion_queue = grpc_completion_queue_create(NULL)
+    with nogil:
+      self.c_completion_queue = grpc_completion_queue_create(NULL)
     self.is_shutting_down = False
     self.is_shutdown = False
     self.pluck_condition = threading.Condition()
@@ -82,8 +83,9 @@ cdef class CompletionQueue:
   def poll(self, Timespec deadline=None):
     # We name this 'poll' to avoid problems with CPython's expectations for
     # 'special' methods (like next and __next__).
-    cdef gpr_timespec c_deadline = gpr_inf_future(
-        GPR_CLOCK_REALTIME)
+    cdef gpr_timespec c_deadline
+    with nogil:
+      c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME)
     if deadline is not None:
       c_deadline = deadline.c_time
     cdef grpc_event event
@@ -123,7 +125,8 @@ cdef class CompletionQueue:
     return self._interpret_event(event)
 
   def shutdown(self):
-    grpc_completion_queue_shutdown(self.c_completion_queue)
+    with nogil:
+      grpc_completion_queue_shutdown(self.c_completion_queue)
     self.is_shutting_down = True
 
   def clear(self):
@@ -133,14 +136,19 @@ cdef class CompletionQueue:
       pass
 
   def __dealloc__(self):
-    cdef gpr_timespec c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME)
+    cdef gpr_timespec c_deadline
+    with nogil:
+      c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME)
     if self.c_completion_queue != NULL:
       # Ensure shutdown
       if not self.is_shutting_down:
-        grpc_completion_queue_shutdown(self.c_completion_queue)
+        with nogil:
+          grpc_completion_queue_shutdown(self.c_completion_queue)
       # Pump the queue
       while not self.is_shutdown:
-        event = grpc_completion_queue_next(
-            self.c_completion_queue, c_deadline, NULL)
+        with nogil:
+          event = grpc_completion_queue_next(
+              self.c_completion_queue, c_deadline, NULL)
         self._interpret_event(event)
-      grpc_completion_queue_destroy(self.c_completion_queue)
+      with nogil:
+        grpc_completion_queue_destroy(self.c_completion_queue)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
index 3f439c8900..1d7adca23e 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -46,7 +46,8 @@ cdef class ChannelCredentials:
 
   def __dealloc__(self):
     if self.c_credentials != NULL:
-      grpc_channel_credentials_release(self.c_credentials)
+      with nogil:
+        grpc_channel_credentials_release(self.c_credentials)
 
 
 cdef class CallCredentials:
@@ -63,7 +64,8 @@ cdef class CallCredentials:
 
   def __dealloc__(self):
     if self.c_credentials != NULL:
-      grpc_call_credentials_release(self.c_credentials)
+      with nogil:
+        grpc_call_credentials_release(self.c_credentials)
 
 
 cdef class ServerCredentials:
@@ -74,7 +76,8 @@ cdef class ServerCredentials:
 
   def __dealloc__(self):
     if self.c_credentials != NULL:
-      grpc_server_credentials_release(self.c_credentials)
+      with nogil:
+        grpc_server_credentials_release(self.c_credentials)
 
 
 cdef class CredentialsMetadataPlugin:
@@ -139,7 +142,8 @@ cdef void plugin_destroy_c_plugin_state(void *state):
 
 def channel_credentials_google_default():
   cdef ChannelCredentials credentials = ChannelCredentials();
-  credentials.c_credentials = grpc_google_default_credentials_create()
+  with nogil:
+    credentials.c_credentials = grpc_google_default_credentials_create()
   return credentials
 
 def channel_credentials_ssl(pem_root_certificates,
@@ -158,12 +162,14 @@ def channel_credentials_ssl(pem_root_certificates,
     c_pem_root_certificates = pem_root_certificates
     credentials.references.append(pem_root_certificates)
   if ssl_pem_key_cert_pair is not None:
-    credentials.c_credentials = grpc_ssl_credentials_create(
-        c_pem_root_certificates, &ssl_pem_key_cert_pair.c_pair, NULL)
+    with nogil:
+      credentials.c_credentials = grpc_ssl_credentials_create(
+          c_pem_root_certificates, &ssl_pem_key_cert_pair.c_pair, NULL)
     credentials.references.append(ssl_pem_key_cert_pair)
   else:
-    credentials.c_credentials = grpc_ssl_credentials_create(
-      c_pem_root_certificates, NULL, NULL)
+    with nogil:
+      credentials.c_credentials = grpc_ssl_credentials_create(
+        c_pem_root_certificates, NULL, NULL)
   return credentials
 
 def channel_credentials_composite(
@@ -172,8 +178,9 @@ def channel_credentials_composite(
   if not credentials_1.is_valid or not credentials_2.is_valid:
     raise ValueError("passed credentials must both be valid")
   cdef ChannelCredentials credentials = ChannelCredentials()
-  credentials.c_credentials = grpc_composite_channel_credentials_create(
-      credentials_1.c_credentials, credentials_2.c_credentials, NULL)
+  with nogil:
+    credentials.c_credentials = grpc_composite_channel_credentials_create(
+        credentials_1.c_credentials, credentials_2.c_credentials, NULL)
   credentials.references.append(credentials_1)
   credentials.references.append(credentials_2)
   return credentials
@@ -184,16 +191,18 @@ def call_credentials_composite(
   if not credentials_1.is_valid or not credentials_2.is_valid:
     raise ValueError("passed credentials must both be valid")
   cdef CallCredentials credentials = CallCredentials()
-  credentials.c_credentials = grpc_composite_call_credentials_create(
-      credentials_1.c_credentials, credentials_2.c_credentials, NULL)
+  with nogil:
+    credentials.c_credentials = grpc_composite_call_credentials_create(
+        credentials_1.c_credentials, credentials_2.c_credentials, NULL)
   credentials.references.append(credentials_1)
   credentials.references.append(credentials_2)
   return credentials
 
 def call_credentials_google_compute_engine():
   cdef CallCredentials credentials = CallCredentials()
-  credentials.c_credentials = (
-      grpc_google_compute_engine_credentials_create(NULL))
+  with nogil:
+    credentials.c_credentials = (
+        grpc_google_compute_engine_credentials_create(NULL))
   return credentials
 
 def call_credentials_service_account_jwt_access(
@@ -205,9 +214,11 @@ def call_credentials_service_account_jwt_access(
   else:
     raise TypeError("expected json_key to be str or bytes")
   cdef CallCredentials credentials = CallCredentials()
-  credentials.c_credentials = (
-      grpc_service_account_jwt_access_credentials_create(
-          json_key, token_lifetime.c_time, NULL))
+  cdef char *json_key_c_string = json_key
+  with nogil:
+    credentials.c_credentials = (
+        grpc_service_account_jwt_access_credentials_create(
+            json_key_c_string, token_lifetime.c_time, NULL))
   credentials.references.append(json_key)
   return credentials
 
@@ -219,8 +230,10 @@ def call_credentials_google_refresh_token(json_refresh_token):
   else:
     raise TypeError("expected json_refresh_token to be str or bytes")
   cdef CallCredentials credentials = CallCredentials()
-  credentials.c_credentials = grpc_google_refresh_token_credentials_create(
-      json_refresh_token, NULL)
+  cdef char *json_refresh_token_c_string = json_refresh_token
+  with nogil:
+    credentials.c_credentials = grpc_google_refresh_token_credentials_create(
+        json_refresh_token_c_string, NULL)
   credentials.references.append(json_refresh_token)
   return credentials
 
@@ -238,17 +251,21 @@ def call_credentials_google_iam(authorization_token, authority_selector):
   else:
     raise TypeError("expected authority_selector to be str or bytes")
   cdef CallCredentials credentials = CallCredentials()
-  credentials.c_credentials = grpc_google_iam_credentials_create(
-      authorization_token, authority_selector, NULL)
+  cdef char *authorization_token_c_string = authorization_token
+  cdef char *authority_selector_c_string = authority_selector
+  with nogil:
+    credentials.c_credentials = grpc_google_iam_credentials_create(
+        authorization_token_c_string, authority_selector_c_string, NULL)
   credentials.references.append(authorization_token)
   credentials.references.append(authority_selector)
   return credentials
 
 def call_credentials_metadata_plugin(CredentialsMetadataPlugin plugin):
   cdef CallCredentials credentials = CallCredentials()
-  credentials.c_credentials = (
-      grpc_metadata_credentials_create_from_plugin(plugin.make_c_plugin(),
-                                                        NULL))
+  cdef grpc_metadata_credentials_plugin c_plugin = plugin.make_c_plugin()
+  with nogil:
+    credentials.c_credentials = (
+        grpc_metadata_credentials_create_from_plugin(c_plugin, NULL))
   # TODO(atash): the following held reference is *probably* never necessary
   credentials.references.append(plugin)
   return credentials
@@ -274,11 +291,12 @@ def server_credentials_ssl(pem_root_certs, pem_key_cert_pairs,
   credentials.references.append(pem_key_cert_pairs)
   credentials.references.append(pem_root_certs)
   credentials.c_ssl_pem_key_cert_pairs_count = len(pem_key_cert_pairs)
-  credentials.c_ssl_pem_key_cert_pairs = (
-      <grpc_ssl_pem_key_cert_pair *>gpr_malloc(
-          sizeof(grpc_ssl_pem_key_cert_pair) *
-              credentials.c_ssl_pem_key_cert_pairs_count
-      ))
+  with nogil:
+    credentials.c_ssl_pem_key_cert_pairs = (
+        <grpc_ssl_pem_key_cert_pair *>gpr_malloc(
+            sizeof(grpc_ssl_pem_key_cert_pair) *
+                credentials.c_ssl_pem_key_cert_pairs_count
+        ))
   for i in range(credentials.c_ssl_pem_key_cert_pairs_count):
     credentials.c_ssl_pem_key_cert_pairs[i] = (
         (<SslPemKeyCertPair>pem_key_cert_pairs[i]).c_pair)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index dbf0045710..61165cb021 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -38,27 +38,27 @@ cdef extern from "grpc/_cython/loader.h":
 
   int pygrpc_load_core(char*)
 
-  void *gpr_malloc(size_t size)
-  void gpr_free(void *ptr)
-  void *gpr_realloc(void *p, size_t size)
+  void *gpr_malloc(size_t size) nogil
+  void gpr_free(void *ptr) nogil
+  void *gpr_realloc(void *p, size_t size) nogil
 
   ctypedef struct gpr_slice:
     # don't worry about writing out the members of gpr_slice; we never access
     # them directly.
     pass
 
-  gpr_slice gpr_slice_ref(gpr_slice s)
-  void gpr_slice_unref(gpr_slice s)
-  gpr_slice gpr_slice_new(void *p, size_t len, void (*destroy)(void *))
+  gpr_slice gpr_slice_ref(gpr_slice s) nogil
+  void gpr_slice_unref(gpr_slice s) nogil
+  gpr_slice gpr_slice_new(void *p, size_t len, void (*destroy)(void *)) nogil
   gpr_slice gpr_slice_new_with_len(
-      void *p, size_t len, void (*destroy)(void *, size_t))
-  gpr_slice gpr_slice_malloc(size_t length)
-  gpr_slice gpr_slice_from_copied_string(const char *source)
-  gpr_slice gpr_slice_from_copied_buffer(const char *source, size_t len)
+      void *p, size_t len, void (*destroy)(void *, size_t)) nogil
+  gpr_slice gpr_slice_malloc(size_t length) nogil
+  gpr_slice gpr_slice_from_copied_string(const char *source) nogil
+  gpr_slice gpr_slice_from_copied_buffer(const char *source, size_t len) nogil
 
   # Declare functions for function-like macros (because Cython)...
-  void *gpr_slice_start_ptr "GPR_SLICE_START_PTR" (gpr_slice s)
-  size_t gpr_slice_length "GPR_SLICE_LENGTH" (gpr_slice s)
+  void *gpr_slice_start_ptr "GPR_SLICE_START_PTR" (gpr_slice s) nogil
+  size_t gpr_slice_length "GPR_SLICE_LENGTH" (gpr_slice s) nogil
 
   ctypedef enum gpr_clock_type:
     GPR_CLOCK_MONOTONIC
@@ -71,14 +71,14 @@ cdef extern from "grpc/_cython/loader.h":
     int32_t nanoseconds "tv_nsec"
     gpr_clock_type clock_type
 
-  gpr_timespec gpr_time_0(gpr_clock_type type)
-  gpr_timespec gpr_inf_future(gpr_clock_type type)
-  gpr_timespec gpr_inf_past(gpr_clock_type type)
+  gpr_timespec gpr_time_0(gpr_clock_type type) nogil
+  gpr_timespec gpr_inf_future(gpr_clock_type type) nogil
+  gpr_timespec gpr_inf_past(gpr_clock_type type) nogil
 
-  gpr_timespec gpr_now(gpr_clock_type clock)
+  gpr_timespec gpr_now(gpr_clock_type clock) nogil
 
   gpr_timespec gpr_convert_clock_type(gpr_timespec t,
-                                      gpr_clock_type target_clock)
+                                      gpr_clock_type target_clock) nogil
 
   ctypedef enum grpc_status_code:
     GRPC_STATUS_OK
@@ -114,15 +114,15 @@ cdef extern from "grpc/_cython/loader.h":
     pass
 
   grpc_byte_buffer *grpc_raw_byte_buffer_create(gpr_slice *slices,
-                                                size_t nslices)
-  size_t grpc_byte_buffer_length(grpc_byte_buffer *bb)
-  void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer)
+                                                size_t nslices) nogil
+  size_t grpc_byte_buffer_length(grpc_byte_buffer *bb) nogil
+  void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer) nogil
 
   void grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader,
-                                    grpc_byte_buffer *buffer)
+                                    grpc_byte_buffer *buffer) nogil
   int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader,
-                                   gpr_slice *slice)
-  void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader)
+                                   gpr_slice *slice) nogil
+  void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader) nogil
 
   const char *GRPC_ARG_PRIMARY_USER_AGENT_STRING
   const char *GRPC_ARG_ENABLE_CENSUS
@@ -221,8 +221,8 @@ cdef extern from "grpc/_cython/loader.h":
     size_t capacity
     grpc_metadata *metadata
 
-  void grpc_metadata_array_init(grpc_metadata_array *array)
-  void grpc_metadata_array_destroy(grpc_metadata_array *array)
+  void grpc_metadata_array_init(grpc_metadata_array *array) nogil
+  void grpc_metadata_array_destroy(grpc_metadata_array *array) nogil
 
   ctypedef struct grpc_call_details:
     char *method
@@ -231,8 +231,8 @@ cdef extern from "grpc/_cython/loader.h":
     size_t host_capacity
     gpr_timespec deadline
 
-  void grpc_call_details_init(grpc_call_details *details)
-  void grpc_call_details_destroy(grpc_call_details *details)
+  void grpc_call_details_init(grpc_call_details *details) nogil
+  void grpc_call_details_destroy(grpc_call_details *details) nogil
 
   ctypedef enum grpc_op_type:
     GRPC_OP_SEND_INITIAL_METADATA
@@ -277,61 +277,62 @@ cdef extern from "grpc/_cython/loader.h":
     uint32_t flags
     grpc_op_data data
 
-  void grpc_init()
-  void grpc_shutdown()
+  void grpc_init() nogil
+  void grpc_shutdown() nogil
 
-  grpc_completion_queue *grpc_completion_queue_create(void *reserved)
+  grpc_completion_queue *grpc_completion_queue_create(void *reserved) nogil
   grpc_event grpc_completion_queue_next(grpc_completion_queue *cq,
                                         gpr_timespec deadline,
                                         void *reserved) nogil
   grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cq, void *tag,
                                          gpr_timespec deadline,
                                          void *reserved) nogil
-  void grpc_completion_queue_shutdown(grpc_completion_queue *cq)
-  void grpc_completion_queue_destroy(grpc_completion_queue *cq)
+  void grpc_completion_queue_shutdown(grpc_completion_queue *cq) nogil
+  void grpc_completion_queue_destroy(grpc_completion_queue *cq) nogil
 
-  grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
-                                        size_t nops, void *tag, void *reserved)
-  grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved)
+  grpc_call_error grpc_call_start_batch(
+      grpc_call *call, const grpc_op *ops, size_t nops, void *tag,
+      void *reserved) nogil
+  grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) nogil
   grpc_call_error grpc_call_cancel_with_status(grpc_call *call,
                                                grpc_status_code status,
                                                const char *description,
-                                               void *reserved)
-  char *grpc_call_get_peer(grpc_call *call)
-  void grpc_call_destroy(grpc_call *call)
+                                               void *reserved) nogil
+  char *grpc_call_get_peer(grpc_call *call) nogil
+  void grpc_call_destroy(grpc_call *call) nogil
 
   grpc_channel *grpc_insecure_channel_create(const char *target,
                                              const grpc_channel_args *args,
-                                             void *reserved)
-  grpc_call *grpc_channel_create_call(grpc_channel *channel,
-                                      grpc_call *parent_call,
-                                      uint32_t propagation_mask,
-                                      grpc_completion_queue *completion_queue,
-                                      const char *method, const char *host,
-                                      gpr_timespec deadline, void *reserved)
+                                             void *reserved) nogil
+  grpc_call *grpc_channel_create_call(
+      grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+      grpc_completion_queue *completion_queue, const char *method,
+      const char *host, gpr_timespec deadline, void *reserved) nogil
   grpc_connectivity_state grpc_channel_check_connectivity_state(
-      grpc_channel *channel, int try_to_connect)
+      grpc_channel *channel, int try_to_connect) nogil
   void grpc_channel_watch_connectivity_state(
       grpc_channel *channel, grpc_connectivity_state last_observed_state,
-      gpr_timespec deadline, grpc_completion_queue *cq, void *tag)
-  char *grpc_channel_get_target(grpc_channel *channel)
-  void grpc_channel_destroy(grpc_channel *channel)
+      gpr_timespec deadline, grpc_completion_queue *cq, void *tag) nogil
+  char *grpc_channel_get_target(grpc_channel *channel) nogil
+  void grpc_channel_destroy(grpc_channel *channel) nogil
 
-  grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved)
+  grpc_server *grpc_server_create(
+      const grpc_channel_args *args, void *reserved) nogil
   grpc_call_error grpc_server_request_call(
       grpc_server *server, grpc_call **call, grpc_call_details *details,
       grpc_metadata_array *request_metadata, grpc_completion_queue
       *cq_bound_to_call, grpc_completion_queue *cq_for_notification, void
-      *tag_new)
+      *tag_new) nogil
   void grpc_server_register_completion_queue(grpc_server *server,
                                              grpc_completion_queue *cq,
-                                             void *reserved)
-  int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr)
-  void grpc_server_start(grpc_server *server)
+                                             void *reserved) nogil
+  int grpc_server_add_insecure_http2_port(
+      grpc_server *server, const char *addr) nogil
+  void grpc_server_start(grpc_server *server) nogil
   void grpc_server_shutdown_and_notify(
-      grpc_server *server, grpc_completion_queue *cq, void *tag)
-  void grpc_server_cancel_all_calls(grpc_server *server)
-  void grpc_server_destroy(grpc_server *server)
+      grpc_server *server, grpc_completion_queue *cq, void *tag) nogil
+  void grpc_server_cancel_all_calls(grpc_server *server) nogil
+  void grpc_server_destroy(grpc_server *server) nogil
 
   ctypedef struct grpc_ssl_pem_key_cert_pair:
     const char *private_key
@@ -347,35 +348,36 @@ cdef extern from "grpc/_cython/loader.h":
 
   ctypedef void (*grpc_ssl_roots_override_callback)(char **pem_root_certs)
 
-  void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb)
+  void grpc_set_ssl_roots_override_callback(
+      grpc_ssl_roots_override_callback cb) nogil
 
-  grpc_channel_credentials *grpc_google_default_credentials_create()
+  grpc_channel_credentials *grpc_google_default_credentials_create() nogil
   grpc_channel_credentials *grpc_ssl_credentials_create(
       const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-      void *reserved)
+      void *reserved) nogil
   grpc_channel_credentials *grpc_composite_channel_credentials_create(
       grpc_channel_credentials *creds1, grpc_call_credentials *creds2,
-      void *reserved)
-  void grpc_channel_credentials_release(grpc_channel_credentials *creds)
+      void *reserved) nogil
+  void grpc_channel_credentials_release(grpc_channel_credentials *creds) nogil
 
   grpc_call_credentials *grpc_composite_call_credentials_create(
       grpc_call_credentials *creds1, grpc_call_credentials *creds2,
-      void *reserved)
+      void *reserved) nogil
   grpc_call_credentials *grpc_google_compute_engine_credentials_create(
-      void *reserved)
+      void *reserved) nogil
   grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
       const char *json_key,
-      gpr_timespec token_lifetime, void *reserved)
+      gpr_timespec token_lifetime, void *reserved) nogil
   grpc_call_credentials *grpc_google_refresh_token_credentials_create(
-      const char *json_refresh_token, void *reserved)
+      const char *json_refresh_token, void *reserved) nogil
   grpc_call_credentials *grpc_google_iam_credentials_create(
       const char *authorization_token, const char *authority_selector,
-      void *reserved)
-  void grpc_call_credentials_release(grpc_call_credentials *creds)
+      void *reserved) nogil
+  void grpc_call_credentials_release(grpc_call_credentials *creds) nogil
 
   grpc_channel *grpc_secure_channel_create(
       grpc_channel_credentials *creds, const char *target,
-      const grpc_channel_args *args, void *reserved)
+      const grpc_channel_args *args, void *reserved) nogil
 
   ctypedef struct grpc_server_credentials:
     # We don't care about the internals (and in fact don't know them)
@@ -385,13 +387,13 @@ cdef extern from "grpc/_cython/loader.h":
       const char *pem_root_certs,
       grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
       size_t num_key_cert_pairs, int force_client_auth, void *reserved)
-  void grpc_server_credentials_release(grpc_server_credentials *creds)
+  void grpc_server_credentials_release(grpc_server_credentials *creds) nogil
 
   int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
-                                        grpc_server_credentials *creds)
+                                        grpc_server_credentials *creds) nogil
 
   grpc_call_error grpc_call_set_credentials(grpc_call *call,
-                                            grpc_call_credentials *creds)
+                                            grpc_call_credentials *creds) nogil
 
   ctypedef struct grpc_auth_context:
     # We don't care about the internals (and in fact don't know them)
@@ -415,4 +417,4 @@ cdef extern from "grpc/_cython/loader.h":
     const char *type
 
   grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
-      grpc_metadata_credentials_plugin plugin, void *reserved)
+      grpc_metadata_credentials_plugin plugin, void *reserved) nogil
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
index fa4ea99ea9..6ecdcf7222 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
@@ -107,15 +107,18 @@ cdef class Timespec:
 
   def __cinit__(self, time):
     if time is None:
-      self.c_time = gpr_now(GPR_CLOCK_REALTIME)
+      with nogil:
+        self.c_time = gpr_now(GPR_CLOCK_REALTIME)
       return
     if isinstance(time, int):
       time = float(time)
     if isinstance(time, float):
       if time == float("+inf"):
-        self.c_time = gpr_inf_future(GPR_CLOCK_REALTIME)
+        with nogil:
+          self.c_time = gpr_inf_future(GPR_CLOCK_REALTIME)
       elif time == float("-inf"):
-        self.c_time = gpr_inf_past(GPR_CLOCK_REALTIME)
+        with nogil:
+          self.c_time = gpr_inf_past(GPR_CLOCK_REALTIME)
       else:
         self.c_time.seconds = time
         self.c_time.nanoseconds = (time - float(self.c_time.seconds)) * 1e9
@@ -131,8 +134,10 @@ cdef class Timespec:
     # TODO(atash) ensure that everywhere a Timespec is created that it's
     # converted to GPR_CLOCK_REALTIME then and not every time someone wants to
     # read values off in Python.
-    cdef gpr_timespec real_time = (
-        gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME))
+    cdef gpr_timespec real_time
+    with nogil:
+      real_time = (
+          gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME))
     return real_time.seconds
 
   @property
@@ -158,10 +163,12 @@ cdef class Timespec:
 cdef class CallDetails:
 
   def __cinit__(self):
-    grpc_call_details_init(&self.c_details)
+    with nogil:
+      grpc_call_details_init(&self.c_details)
 
   def __dealloc__(self):
-    grpc_call_details_destroy(&self.c_details)
+    with nogil:
+      grpc_call_details_destroy(&self.c_details)
 
   @property
   def method(self):
@@ -229,10 +236,15 @@ cdef class ByteBuffer:
                       "ByteBuffer, not {}".format(type(data)))
 
     cdef char *c_data = data
-    data_slice = gpr_slice_from_copied_buffer(c_data, len(data))
-    self.c_byte_buffer = grpc_raw_byte_buffer_create(
-        &data_slice, 1)
-    gpr_slice_unref(data_slice)
+    cdef gpr_slice data_slice
+    cdef size_t data_length = len(data)
+    with nogil:
+      data_slice = gpr_slice_from_copied_buffer(c_data, data_length)
+    with nogil:
+      self.c_byte_buffer = grpc_raw_byte_buffer_create(
+          &data_slice, 1)
+    with nogil:
+      gpr_slice_unref(data_slice)
 
   def bytes(self):
     cdef grpc_byte_buffer_reader reader
@@ -240,20 +252,27 @@ cdef class ByteBuffer:
     cdef size_t data_slice_length
     cdef void *data_slice_pointer
     if self.c_byte_buffer != NULL:
-      grpc_byte_buffer_reader_init(&reader, self.c_byte_buffer)
-      result = b""
-      while grpc_byte_buffer_reader_next(&reader, &data_slice):
-        data_slice_pointer = gpr_slice_start_ptr(data_slice)
-        data_slice_length = gpr_slice_length(data_slice)
-        result += (<char *>data_slice_pointer)[:data_slice_length]
-      grpc_byte_buffer_reader_destroy(&reader)
-      return result
+      with nogil:
+        grpc_byte_buffer_reader_init(&reader, self.c_byte_buffer)
+      result = bytearray()
+      with nogil:
+        while grpc_byte_buffer_reader_next(&reader, &data_slice):
+          data_slice_pointer = gpr_slice_start_ptr(data_slice)
+          data_slice_length = gpr_slice_length(data_slice)
+          with gil:
+            result += (<char *>data_slice_pointer)[:data_slice_length]
+      with nogil:
+        grpc_byte_buffer_reader_destroy(&reader)
+      return bytes(result)
     else:
       return None
 
   def __len__(self):
+    cdef size_t result
     if self.c_byte_buffer != NULL:
-      return grpc_byte_buffer_length(self.c_byte_buffer)
+      with nogil:
+        result = grpc_byte_buffer_length(self.c_byte_buffer)
+      return result
     else:
       return 0
 
@@ -262,7 +281,8 @@ cdef class ByteBuffer:
 
   def __dealloc__(self):
     if self.c_byte_buffer != NULL:
-      grpc_byte_buffer_destroy(self.c_byte_buffer)
+      with nogil:
+        grpc_byte_buffer_destroy(self.c_byte_buffer)
 
 
 cdef class SslPemKeyCertPair:
@@ -319,14 +339,15 @@ cdef class ChannelArgs:
       if not isinstance(arg, ChannelArg):
         raise TypeError("expected list of ChannelArg")
     self.c_args.arguments_length = len(self.args)
-    self.c_args.arguments = <grpc_arg *>gpr_malloc(
-        self.c_args.arguments_length*sizeof(grpc_arg)
-    )
+    with nogil:
+      self.c_args.arguments = <grpc_arg *>gpr_malloc(
+          self.c_args.arguments_length*sizeof(grpc_arg))
     for i in range(self.c_args.arguments_length):
       self.c_args.arguments[i] = (<ChannelArg>self.args[i]).c_arg
 
   def __dealloc__(self):
-    gpr_free(self.c_args.arguments)
+    with nogil:
+      gpr_free(self.c_args.arguments)
 
   def __len__(self):
     # self.args is never stale; it's only updated from this file
@@ -407,21 +428,24 @@ cdef class Metadata:
     for metadatum in metadata:
       if not isinstance(metadatum, Metadatum):
         raise TypeError("expected list of Metadatum")
-    grpc_metadata_array_init(&self.c_metadata_array)
+    with nogil:
+      grpc_metadata_array_init(&self.c_metadata_array)
     self.c_metadata_array.count = len(self.metadata)
     self.c_metadata_array.capacity = len(self.metadata)
-    self.c_metadata_array.metadata = <grpc_metadata *>gpr_malloc(
-        self.c_metadata_array.count*sizeof(grpc_metadata)
-    )
+    with nogil:
+      self.c_metadata_array.metadata = <grpc_metadata *>gpr_malloc(
+          self.c_metadata_array.count*sizeof(grpc_metadata)
+      )
     for i in range(self.c_metadata_array.count):
       self.c_metadata_array.metadata[i] = (
           (<Metadatum>self.metadata[i]).c_metadata)
 
   def __dealloc__(self):
     # this frees the allocated memory for the grpc_metadata_array (although
-    # it'd be nice if that were documented somewhere...) TODO(atash): document
-    # this in the C core
-    grpc_metadata_array_destroy(&self.c_metadata_array)
+    # it'd be nice if that were documented somewhere...)
+    # TODO(atash): document this in the C core
+    with nogil:
+      grpc_metadata_array_destroy(&self.c_metadata_array)
 
   def __len__(self):
     return self.c_metadata_array.count
@@ -526,7 +550,8 @@ cdef class Operation:
     # Python. The remaining one(s) are primitive fields filled in by GRPC core.
     # This means that we need to clean up after receive_status_on_client.
     if self.c_op.type == GRPC_OP_RECV_STATUS_ON_CLIENT:
-      gpr_free(self._received_status_details)
+      with nogil:
+        gpr_free(self._received_status_details)
 
 def operation_send_initial_metadata(Metadata metadata):
   cdef Operation op = Operation()
@@ -648,8 +673,8 @@ cdef class Operations:
       if not isinstance(operation, Operation):
         raise TypeError("expected operations to be iterable of Operation")
     self.c_nops = len(self.operations)
-    self.c_ops = <grpc_op *>gpr_malloc(
-        sizeof(grpc_op)*self.c_nops)
+    with nogil:
+      self.c_ops = <grpc_op *>gpr_malloc(sizeof(grpc_op)*self.c_nops)
     for i in range(self.c_nops):
       self.c_ops[i] = (<Operation>(self.operations[i])).c_op
 
@@ -661,7 +686,8 @@ cdef class Operations:
     return self.operations[i]
 
   def __dealloc__(self):
-    gpr_free(self.c_ops)
+    with nogil:
+      gpr_free(self.c_ops)
 
   def __iter__(self):
     return _OperationsIterator(self)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
index fe93da6c12..a098f11da2 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
@@ -41,7 +41,8 @@ cdef class Server:
     if arguments is not None:
       c_arguments = &arguments.c_args
       self.references.append(arguments)
-    self.c_server = grpc_server_create(c_arguments, NULL)
+    with nogil:
+      self.c_server = grpc_server_create(c_arguments, NULL)
     self.is_started = False
     self.is_shutting_down = False
     self.is_shutdown = False
@@ -53,6 +54,7 @@ cdef class Server:
       raise ValueError("server must be started and not shutting down")
     if server_queue not in self.registered_completion_queues:
       raise ValueError("server_queue must be a registered completion queue")
+    cdef grpc_call_error result
     cdef OperationTag operation_tag = OperationTag(tag)
     operation_tag.operation_call = Call()
     operation_tag.request_call_details = CallDetails()
@@ -61,19 +63,22 @@ cdef class Server:
     operation_tag.is_new_request = True
     operation_tag.batch_operations = Operations([])
     cpython.Py_INCREF(operation_tag)
-    return grpc_server_request_call(
-        self.c_server, &operation_tag.operation_call.c_call,
-        &operation_tag.request_call_details.c_details,
-        &operation_tag.request_metadata.c_metadata_array,
-        call_queue.c_completion_queue, server_queue.c_completion_queue,
-        <cpython.PyObject *>operation_tag)
+    with nogil:
+      result = grpc_server_request_call(
+          self.c_server, &operation_tag.operation_call.c_call,
+          &operation_tag.request_call_details.c_details,
+          &operation_tag.request_metadata.c_metadata_array,
+          call_queue.c_completion_queue, server_queue.c_completion_queue,
+          <cpython.PyObject *>operation_tag)
+    return result
 
   def register_completion_queue(
       self, CompletionQueue queue not None):
     if self.is_started:
       raise ValueError("cannot register completion queues after start")
-    grpc_server_register_completion_queue(
-        self.c_server, queue.c_completion_queue, NULL)
+    with nogil:
+      grpc_server_register_completion_queue(
+          self.c_server, queue.c_completion_queue, NULL)
     self.registered_completion_queues.append(queue)
 
   def start(self):
@@ -82,7 +87,8 @@ cdef class Server:
     self.backup_shutdown_queue = CompletionQueue()
     self.register_completion_queue(self.backup_shutdown_queue)
     self.is_started = True
-    grpc_server_start(self.c_server)
+    with nogil:
+      grpc_server_start(self.c_server)
     # Ensure the core has gotten a chance to do the start-up work
     self.backup_shutdown_queue.pluck(None, Timespec(None))
 
@@ -95,22 +101,28 @@ cdef class Server:
     else:
       raise TypeError("expected address to be a str or bytes")
     self.references.append(address)
+    cdef int result
+    cdef char *address_c_string = address
     if server_credentials is not None:
       self.references.append(server_credentials)
-      return grpc_server_add_secure_http2_port(
-          self.c_server, address, server_credentials.c_credentials)
+      with nogil:
+        result = grpc_server_add_secure_http2_port(
+            self.c_server, address_c_string, server_credentials.c_credentials)
     else:
-      return grpc_server_add_insecure_http2_port(self.c_server, address)
+      with nogil:
+        result = grpc_server_add_insecure_http2_port(self.c_server,
+                                                     address_c_string)
+    return result
 
   cdef _c_shutdown(self, CompletionQueue queue, tag):
     self.is_shutting_down = True
     operation_tag = OperationTag(tag)
     operation_tag.shutting_down_server = self
-    operation_tag.references.extend([self, queue])
     cpython.Py_INCREF(operation_tag)
-    grpc_server_shutdown_and_notify(
-        self.c_server, queue.c_completion_queue,
-        <cpython.PyObject *>operation_tag)
+    with nogil:
+      grpc_server_shutdown_and_notify(
+          self.c_server, queue.c_completion_queue,
+          <cpython.PyObject *>operation_tag)
 
   def shutdown(self, CompletionQueue queue not None, tag):
     cdef OperationTag operation_tag
@@ -135,7 +147,8 @@ cdef class Server:
     elif self.is_shutdown:
       return
     else:
-      grpc_server_cancel_all_calls(self.c_server)
+      with nogil:
+        grpc_server_cancel_all_calls(self.c_server)
 
   def __dealloc__(self):
     if self.c_server != NULL:
@@ -154,5 +167,6 @@ cdef class Server:
         # much but repeatedly release the GIL and wait
         while not self.is_shutdown:
           time.sleep(0)
-      grpc_server_destroy(self.c_server)
+      with nogil:
+        grpc_server_destroy(self.c_server)
 
diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx
index 30cc7a132b..8a0f171ee7 100644
--- a/src/python/grpcio/grpc/_cython/cygrpc.pyx
+++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx
@@ -57,14 +57,17 @@ cdef class _ModuleState:
           'grpc._cython', '_windows/grpc_c.64.python')
       if not pygrpc_load_core(filename):
         raise ImportError('failed to load core gRPC library')
-    grpc_init()
+    with nogil:
+      grpc_init()
     self.is_loaded = True
-    grpc_set_ssl_roots_override_callback(
-        <grpc_ssl_roots_override_callback>ssl_roots_override_callback)
+    with nogil:
+      grpc_set_ssl_roots_override_callback(
+          <grpc_ssl_roots_override_callback>ssl_roots_override_callback)
 
   def __dealloc__(self):
     if self.is_loaded:
-      grpc_shutdown()
+      with nogil:
+        grpc_shutdown()
 
 _module_state = _ModuleState()
 
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
index b70dcccd17..adcd8e954a 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.h
+++ b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -166,7 +166,7 @@ extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_im
 typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name);
 extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import;
 #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import
-typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level);
+typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings);
 extern grpc_compression_algorithm_for_level_type grpc_compression_algorithm_for_level_import;
 #define grpc_compression_algorithm_for_level grpc_compression_algorithm_for_level_import
 typedef void(*grpc_compression_options_init_type)(grpc_compression_options *opts);
@@ -283,7 +283,7 @@ extern grpc_call_destroy_type grpc_call_destroy_import;
 typedef grpc_call_error(*grpc_server_request_call_type)(grpc_server *server, grpc_call **call, grpc_call_details *details, grpc_metadata_array *request_metadata, grpc_completion_queue *cq_bound_to_call, grpc_completion_queue *cq_for_notification, void *tag_new);
 extern grpc_server_request_call_type grpc_server_request_call_import;
 #define grpc_server_request_call grpc_server_request_call_import
-typedef void *(*grpc_server_register_method_type)(grpc_server *server, const char *method, const char *host);
+typedef void *(*grpc_server_register_method_type)(grpc_server *server, const char *method, const char *host, uint32_t flags);
 extern grpc_server_register_method_type grpc_server_register_method_import;
 #define grpc_server_register_method grpc_server_register_method_import
 typedef grpc_call_error(*grpc_server_request_registered_call_type)(grpc_server *server, void *registered_method, grpc_call **call, gpr_timespec *deadline, grpc_metadata_array *request_metadata, grpc_byte_buffer **optional_payload, grpc_completion_queue *cq_bound_to_call, grpc_completion_queue *cq_for_notification, void *tag_new);
diff --git a/src/python/grpcio/grpc/_links/invocation.py b/src/python/grpcio/grpc/_links/invocation.py
index 5ca0a0ee60..672e3e4cc8 100644
--- a/src/python/grpcio/grpc/_links/invocation.py
+++ b/src/python/grpcio/grpc/_links/invocation.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -35,6 +35,8 @@ import logging
 import threading
 import time
 
+import six
+
 from grpc._adapter import _intermediary_low
 from grpc._links import _constants
 from grpc.beta import interfaces as beta_interfaces
@@ -372,12 +374,11 @@ class _Kernel(object):
     pool.shutdown(wait=True)
 
 
-class InvocationLink(links.Link, activated.Activated):
+class InvocationLink(six.with_metaclass(abc.ABCMeta, links.Link, activated.Activated)):
   """A links.Link for use on the invocation-side of a gRPC connection.
 
   Implementations of this interface are only valid for use when activated.
   """
-  __metaclass__ = abc.ABCMeta
 
 
 class _InvocationLink(InvocationLink):
diff --git a/src/python/grpcio/grpc/_links/service.py b/src/python/grpcio/grpc/_links/service.py
index 01edee6896..e0f26a5b0f 100644
--- a/src/python/grpcio/grpc/_links/service.py
+++ b/src/python/grpcio/grpc/_links/service.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -177,7 +177,7 @@ class _Kernel(object):
     call = service_acceptance.call
     call.accept(self._completion_queue, call)
     try:
-      group, method = service_acceptance.method.split('/')[1:3]
+      group, method = service_acceptance.method.split(b'/')[1:3]
     except ValueError:
       logging.info('Illegal path "%s"!', service_acceptance.method)
       return
diff --git a/src/python/grpcio/grpc/beta/_server.py b/src/python/grpcio/grpc/beta/_server.py
index 2b520cc7e5..12d16e6c18 100644
--- a/src/python/grpcio/grpc/beta/_server.py
+++ b/src/python/grpcio/grpc/beta/_server.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -62,7 +62,7 @@ class _GRPCServicer(base.Servicer):
       if e.code is None and e.details is None:
         raise base.NoSuchMethodError(
             interfaces.StatusCode.UNIMPLEMENTED,
-            b'Method "%s" of service "%s" not implemented!' % (method, group))
+            'Method "%s" of service "%s" not implemented!' % (method, group))
       else:
         raise
 
diff --git a/src/python/grpcio/grpc/beta/interfaces.py b/src/python/grpcio/grpc/beta/interfaces.py
index 0663119163..e29a5b3379 100644
--- a/src/python/grpcio/grpc/beta/interfaces.py
+++ b/src/python/grpcio/grpc/beta/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import abc
 import enum
 
+import six
+
 from grpc._adapter import _types
 
 
@@ -105,19 +107,17 @@ def grpc_call_options(disable_compression=False, credentials=None):
   return GRPCCallOptions(disable_compression, None, credentials)
 
 
-class GRPCAuthMetadataContext(object):
+class GRPCAuthMetadataContext(six.with_metaclass(abc.ABCMeta)):
   """Provides information to call credentials metadata plugins.
 
   Attributes:
     service_url: A string URL of the service being called into.
     method_name: A string of the fully qualified method name being called.
   """
-  __metaclass__ = abc.ABCMeta
 
 
-class GRPCAuthMetadataPluginCallback(object):
+class GRPCAuthMetadataPluginCallback(six.with_metaclass(abc.ABCMeta)):
   """Callback object received by a metadata plugin."""
-  __metaclass__ = abc.ABCMeta
 
   def __call__(self, metadata, error):
     """Inform the gRPC runtime of the metadata to construct a CallCredentials.
@@ -130,10 +130,9 @@ class GRPCAuthMetadataPluginCallback(object):
     raise NotImplementedError()
 
 
-class GRPCAuthMetadataPlugin(object):
+class GRPCAuthMetadataPlugin(six.with_metaclass(abc.ABCMeta)):
   """
   """
-  __metaclass__ = abc.ABCMeta
 
   def __call__(self, context, callback):
     """Invoke the plugin.
@@ -149,9 +148,8 @@ class GRPCAuthMetadataPlugin(object):
     raise NotImplementedError()
 
 
-class GRPCServicerContext(object):
+class GRPCServicerContext(six.with_metaclass(abc.ABCMeta)):
   """Exposes gRPC-specific options and behaviors to code servicing RPCs."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def peer(self):
@@ -168,9 +166,8 @@ class GRPCServicerContext(object):
     raise NotImplementedError()
 
 
-class GRPCInvocationContext(object):
+class GRPCInvocationContext(six.with_metaclass(abc.ABCMeta)):
   """Exposes gRPC-specific options and behaviors to code invoking RPCs."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def disable_next_request_compression(self):
@@ -178,9 +175,8 @@ class GRPCInvocationContext(object):
     raise NotImplementedError()
 
 
-class Server(object):
+class Server(six.with_metaclass(abc.ABCMeta)):
   """Services RPCs."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def add_insecure_port(self, address):
diff --git a/src/python/grpcio/grpc/framework/alpha/_face_utilities.py b/src/python/grpcio/grpc/framework/alpha/_face_utilities.py
index fb0cfe426d..b5e4133cad 100644
--- a/src/python/grpcio/grpc/framework/alpha/_face_utilities.py
+++ b/src/python/grpcio/grpc/framework/alpha/_face_utilities.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -30,6 +30,8 @@
 import abc
 import collections
 
+import six
+
 # face_interfaces is referenced from specification in this module.
 from grpc.framework.common import cardinality
 from grpc.framework.face import interfaces as face_interfaces  # pylint: disable=unused-import
@@ -45,7 +47,7 @@ def _qualified_name(service_name, method_name):
 # TODO(nathaniel): This structure is getting bloated; it could be shrunk if
 # implementations._Stub used a generic rather than a dynamic underlying
 # face-layer stub.
-class InvocationBreakdown(object):
+class InvocationBreakdown(six.with_metaclass(abc.ABCMeta)):
   """An intermediate representation of invocation-side views of RPC methods.
 
   Attributes:
@@ -61,7 +63,6 @@ class InvocationBreakdown(object):
       to callable behavior to be used deserializing response values for the
       RPC.
   """
-  __metaclass__ = abc.ABCMeta
 
 
 class _EasyInvocationBreakdown(
@@ -73,7 +74,7 @@ class _EasyInvocationBreakdown(
   pass
 
 
-class ServiceBreakdown(object):
+class ServiceBreakdown(six.with_metaclass(abc.ABCMeta)):
   """An intermediate representation of service-side views of RPC methods.
 
   Attributes:
@@ -84,7 +85,6 @@ class ServiceBreakdown(object):
     response_serializers: A dictionary from service-qualified RPC method name
       to callable behavior to be used serializing response values for the RPC.
   """
-  __metaclass__ = abc.ABCMeta
 
 
 class _EasyServiceBreakdown(
@@ -111,7 +111,7 @@ def break_down_invocation(service_name, method_descriptions):
   face_cardinalities = {}
   request_serializers = {}
   response_deserializers = {}
-  for name, method_description in method_descriptions.iteritems():
+  for name, method_description in six.iteritems(method_descriptions):
     qualified_name = _qualified_name(service_name, name)
     method_cardinality = method_description.cardinality()
     cardinalities[name] = method_description.cardinality()
@@ -139,7 +139,7 @@ def break_down_service(service_name, method_descriptions):
   implementations = {}
   request_deserializers = {}
   response_serializers = {}
-  for name, method_description in method_descriptions.iteritems():
+  for name, method_description in six.iteritems(method_descriptions):
     qualified_name = _qualified_name(service_name, name)
     method_cardinality = method_description.cardinality()
     if method_cardinality is interfaces.Cardinality.UNARY_UNARY:
diff --git a/src/python/grpcio/grpc/framework/alpha/_reexport.py b/src/python/grpcio/grpc/framework/alpha/_reexport.py
index 198cb95ad5..4ea0e94d80 100644
--- a/src/python/grpcio/grpc/framework/alpha/_reexport.py
+++ b/src/python/grpcio/grpc/framework/alpha/_reexport.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,6 +27,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import six
+
 from grpc.framework.common import cardinality
 from grpc.framework.face import exceptions as face_exceptions
 from grpc.framework.face import interfaces as face_interfaces
@@ -181,7 +183,7 @@ def common_cardinality(early_adopter_cardinality):
 
 def common_cardinalities(early_adopter_cardinalities):
   common_cardinalities = {}
-  for name, early_adopter_cardinality in early_adopter_cardinalities.iteritems():
+  for name, early_adopter_cardinality in six.iteritems(early_adopter_cardinalities):
     common_cardinalities[name] = _EARLY_ADOPTER_CARDINALITY_TO_COMMON_CARDINALITY[
         early_adopter_cardinality]
   return common_cardinalities
diff --git a/src/python/grpcio/grpc/framework/alpha/exceptions.py b/src/python/grpcio/grpc/framework/alpha/exceptions.py
index 5234d3b91c..8ec260488e 100644
--- a/src/python/grpcio/grpc/framework/alpha/exceptions.py
+++ b/src/python/grpcio/grpc/framework/alpha/exceptions.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,12 @@
 
 Only GRPC should instantiate and raise these exceptions.
 """
-
 import abc
 
+import six
 
-class RpcError(Exception):
+class RpcError(six.with_metaclass(abc.ABCMeta, Exception)):
   """Common super type for all exceptions raised by GRPC."""
-  __metaclass__ = abc.ABCMeta
 
 
 class CancellationError(RpcError):
diff --git a/src/python/grpcio/grpc/framework/alpha/interfaces.py b/src/python/grpcio/grpc/framework/alpha/interfaces.py
index 8380567c97..cb6d58bb2e 100644
--- a/src/python/grpcio/grpc/framework/alpha/interfaces.py
+++ b/src/python/grpcio/grpc/framework/alpha/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import abc
 import enum
 
+import six
+
 # exceptions is referenced from specification in this module.
 from grpc.framework.alpha import exceptions  # pylint: disable=unused-import
 from grpc.framework.foundation import activated
@@ -59,15 +61,17 @@ class Abortion(enum.Enum):
   SERVICER_FAILURE = 'servicer failure'
 
 
-class CancellableIterator(object):
+class CancellableIterator(six.with_metaclass(abc.ABCMeta)):
   """Implements the Iterator protocol and affords a cancel method."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __iter__(self):
     """Returns the self object in accordance with the Iterator protocol."""
     raise NotImplementedError()
 
+  def __next__(self):
+    return self.next()
+
   @abc.abstractmethod
   def next(self):
     """Returns a value or raises StopIteration per the Iterator protocol."""
@@ -79,9 +83,8 @@ class CancellableIterator(object):
     raise NotImplementedError()
 
 
-class RpcContext(object):
+class RpcContext(six.with_metaclass(abc.ABCMeta)):
   """Provides RPC-related information and action."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def is_active(self):
@@ -108,7 +111,7 @@ class RpcContext(object):
     raise NotImplementedError()
 
 
-class UnaryUnarySyncAsync(object):
+class UnaryUnarySyncAsync(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a unary-unary RPC synchronously or asynchronously.
   Values implementing this interface are directly callable and present an
   "async" method. Both calls take a request value and a numeric timeout.
@@ -117,7 +120,6 @@ class UnaryUnarySyncAsync(object):
   of a value of this type invokes its associated RPC and immediately returns a
   future.Future bound to the asynchronous execution of the RPC.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request, timeout):
@@ -147,7 +149,7 @@ class UnaryUnarySyncAsync(object):
     raise NotImplementedError()
 
 
-class StreamUnarySyncAsync(object):
+class StreamUnarySyncAsync(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a stream-unary RPC synchronously or asynchronously.
   Values implementing this interface are directly callable and present an
   "async" method. Both calls take an iterator of request values and a numeric
@@ -156,7 +158,6 @@ class StreamUnarySyncAsync(object):
   of a value of this type invokes its associated RPC and immediately returns a
   future.Future bound to the asynchronous execution of the RPC.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request_iterator, timeout):
@@ -191,9 +192,8 @@ class StreamUnarySyncAsync(object):
     raise NotImplementedError()
 
 
-class RpcMethodDescription(object):
+class RpcMethodDescription(six.with_metaclass(abc.ABCMeta)):
   """A type for the common aspects of RPC method descriptions."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def cardinality(self):
@@ -207,9 +207,8 @@ class RpcMethodDescription(object):
     raise NotImplementedError()
 
 
-class RpcMethodInvocationDescription(RpcMethodDescription):
+class RpcMethodInvocationDescription(six.with_metaclass(abc.ABCMeta, RpcMethodDescription)):
   """Invocation-side description of an RPC method."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def serialize_request(self, request):
@@ -240,9 +239,8 @@ class RpcMethodInvocationDescription(RpcMethodDescription):
     raise NotImplementedError()
 
 
-class RpcMethodServiceDescription(RpcMethodDescription):
+class RpcMethodServiceDescription(six.with_metaclass(abc.ABCMeta, RpcMethodDescription)):
   """Service-side description of an RPC method."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def deserialize_request(self, serialized_request):
@@ -345,7 +343,7 @@ class RpcMethodServiceDescription(RpcMethodDescription):
     raise NotImplementedError()
 
 
-class Stub(object):
+class Stub(six.with_metaclass(abc.ABCMeta)):
   """A stub with callable RPC method names for attributes.
 
   Instances of this type are context managers and only afford RPC invocation
@@ -369,12 +367,10 @@ class Stub(object):
   exceptions.RpcError, exceptions.CancellationError,
   and exceptions.ExpirationError.
   """
-  __metaclass__ = abc.ABCMeta
 
 
-class Server(activated.Activated):
+class Server(six.with_metaclass(abc.ABCMeta, activated.Activated)):
   """A GRPC Server."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def port(self):
diff --git a/src/python/grpcio/grpc/framework/base/_ingestion.py b/src/python/grpcio/grpc/framework/base/_ingestion.py
index 06d5b92f0b..090cb158c9 100644
--- a/src/python/grpcio/grpc/framework/base/_ingestion.py
+++ b/src/python/grpcio/grpc/framework/base/_ingestion.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import abc
 import collections
 
+import six
+
 from grpc.framework.base import _constants
 from grpc.framework.base import _interfaces
 from grpc.framework.base import exceptions
@@ -72,9 +74,8 @@ class _EmptyConsumer(stream.Consumer):
     """See stream.Consumer.consume_and_terminate for specification."""
 
 
-class _ConsumerCreator(object):
+class _ConsumerCreator(six.with_metaclass(abc.ABCMeta)):
   """Common specification of different consumer-creating behavior."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def create_consumer(self, requirement):
diff --git a/src/python/grpcio/grpc/framework/base/_interfaces.py b/src/python/grpcio/grpc/framework/base/_interfaces.py
index d88cf76590..c0cc866cad 100644
--- a/src/python/grpcio/grpc/framework/base/_interfaces.py
+++ b/src/python/grpcio/grpc/framework/base/_interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,15 @@
 
 import abc
 
+import six
+
 # interfaces is referenced from specification in this module.
 from grpc.framework.base import interfaces  # pylint: disable=unused-import
 from grpc.framework.foundation import stream
 
 
-class TerminationManager(object):
+class TerminationManager(six.with_metaclass(abc.ABCMeta)):
   """An object responsible for handling the termination of an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_expiration_manager(self, expiration_manager):
@@ -91,9 +92,8 @@ class TerminationManager(object):
     raise NotImplementedError()
 
 
-class TransmissionManager(object):
+class TransmissionManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for transmitting to the other end of an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def inmit(self, emission, complete):
@@ -117,9 +117,8 @@ class TransmissionManager(object):
     raise NotImplementedError()
 
 
-class EmissionManager(stream.Consumer):
+class EmissionManager(six.with_metaclass(abc.ABCMeta, stream.Consumer)):
   """A manager of values emitted by customer code."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_ingestion_manager_and_expiration_manager(
@@ -166,9 +165,8 @@ class EmissionManager(stream.Consumer):
     raise NotImplementedError()
 
 
-class IngestionManager(stream.Consumer):
+class IngestionManager(six.with_metaclass(abc.ABCMeta, stream.Consumer)):
   """A manager responsible for executing customer code."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_expiration_manager(self, expiration_manager):
@@ -214,9 +212,8 @@ class IngestionManager(stream.Consumer):
     raise NotImplementedError()
 
 
-class ExpirationManager(object):
+class ExpirationManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for aborting the operation if it runs out of time."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def change_timeout(self, timeout):
@@ -246,9 +243,8 @@ class ExpirationManager(object):
     raise NotImplementedError()
 
 
-class ReceptionManager(object):
+class ReceptionManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for receiving tickets from the other end."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def receive_ticket(self, ticket):
@@ -261,9 +257,8 @@ class ReceptionManager(object):
     raise NotImplementedError()
 
 
-class CancellationManager(object):
+class CancellationManager(six.with_metaclass(abc.ABCMeta)):
   """A manager of operation cancellation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def cancel(self):
diff --git a/src/python/grpcio/grpc/framework/base/_reception.py b/src/python/grpcio/grpc/framework/base/_reception.py
index dd428964f1..2bee3947f0 100644
--- a/src/python/grpcio/grpc/framework/base/_reception.py
+++ b/src/python/grpcio/grpc/framework/base/_reception.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
 
 import abc
 
+import six
+
 from grpc.framework.base import interfaces
 from grpc.framework.base import _interfaces
 
@@ -40,9 +42,8 @@ _INITIAL_FRONT_TO_BACK_TICKET_KINDS = (
 )
 
 
-class _Receiver(object):
+class _Receiver(six.with_metaclass(abc.ABCMeta)):
   """Common specification of different ticket-handling behavior."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def abort_if_abortive(self, ticket):
diff --git a/src/python/grpcio/grpc/framework/base/_transmission.py b/src/python/grpcio/grpc/framework/base/_transmission.py
index 6845129234..398faaf314 100644
--- a/src/python/grpcio/grpc/framework/base/_transmission.py
+++ b/src/python/grpcio/grpc/framework/base/_transmission.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
 
 import abc
 
+import six
+
 from grpc.framework.base import _constants
 from grpc.framework.base import _interfaces
 from grpc.framework.base import interfaces
@@ -77,9 +79,8 @@ _ABORTION_OUTCOME_TO_BACK_TO_FRONT_TICKET_KIND = {
 }
 
 
-class _Ticketizer(object):
+class _Ticketizer(six.with_metaclass(abc.ABCMeta)):
   """Common specification of different ticket-creating behavior."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def ticketize(self, operation_id, sequence_number, payload, complete):
@@ -187,9 +188,8 @@ class _BackTicketizer(_Ticketizer):
           operation_id, sequence_number, kind, None)
 
 
-class TransmissionManager(_interfaces.TransmissionManager):
+class TransmissionManager(six.with_metaclass(abc.ABCMeta, _interfaces.TransmissionManager)):
   """A _interfaces.TransmissionManager on which other managers may be set."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_ingestion_and_expiration_managers(
diff --git a/src/python/grpcio/grpc/framework/base/interfaces.py b/src/python/grpcio/grpc/framework/base/interfaces.py
index e22c10d975..7c58a23ce0 100644
--- a/src/python/grpcio/grpc/framework/base/interfaces.py
+++ b/src/python/grpcio/grpc/framework/base/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@ import abc
 import collections
 import enum
 
+import six
+
 # stream is referenced from specification in this module.
 from grpc.framework.foundation import stream  # pylint: disable=unused-import
 
@@ -50,13 +52,12 @@ class Outcome(enum.Enum):
   SERVICED_FAILURE = 'serviced failure'
 
 
-class OperationContext(object):
+class OperationContext(six.with_metaclass(abc.ABCMeta)):
   """Provides operation-related information and action.
 
   Attributes:
     trace_id: A uuid.UUID identifying a particular set of related operations.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def is_active(self):
@@ -93,9 +94,8 @@ class OperationContext(object):
     raise NotImplementedError()
 
 
-class Servicer(object):
+class Servicer(six.with_metaclass(abc.ABCMeta)):
   """Interface for service implementations."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def service(self, name, context, output_consumer):
@@ -120,7 +120,7 @@ class Servicer(object):
     raise NotImplementedError()
 
 
-class Operation(object):
+class Operation(six.with_metaclass(abc.ABCMeta)):
   """Representation of an in-progress operation.
 
   Attributes:
@@ -129,7 +129,6 @@ class Operation(object):
     context: An OperationContext affording information and action about the
       operation.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def cancel(self):
@@ -137,9 +136,8 @@ class Operation(object):
     raise NotImplementedError()
 
 
-class ServicedIngestor(object):
+class ServicedIngestor(six.with_metaclass(abc.ABCMeta)):
   """Responsible for accepting the result of an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def consumer(self, operation_context):
@@ -159,7 +157,7 @@ class ServicedIngestor(object):
     raise NotImplementedError()
 
 
-class ServicedSubscription(object):
+class ServicedSubscription(six.with_metaclass(abc.ABCMeta)):
   """A sum type representing a serviced's interest in an operation.
 
   Attributes:
@@ -167,7 +165,6 @@ class ServicedSubscription(object):
     ingestor: A ServicedIngestor. Must be present if kind is Kind.FULL. Must
       be None if kind is Kind.TERMINATION_ONLY or Kind.NONE.
   """
-  __metaclass__ = abc.ABCMeta
 
   @enum.unique
   class Kind(enum.Enum):
@@ -178,9 +175,8 @@ class ServicedSubscription(object):
     NONE = 'none'
 
 
-class End(object):
+class End(six.with_metaclass(abc.ABCMeta)):
   """Common type for entry-point objects on both sides of an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def operation_stats(self):
@@ -202,9 +198,8 @@ class End(object):
     raise NotImplementedError()
 
 
-class Front(End):
+class Front(six.with_metaclass(abc.ABCMeta, End)):
   """Clientish objects that afford the invocation of operations."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def operate(
@@ -228,9 +223,8 @@ class Front(End):
     raise NotImplementedError()
 
 
-class Back(End):
+class Back(six.with_metaclass(abc.ABCMeta, End)):
   """Serverish objects that perform the work of operations."""
-  __metaclass__ = abc.ABCMeta
 
 
 class FrontToBackTicket(
@@ -315,9 +309,8 @@ class BackToFrontTicket(
     TRANSMISSION_FAILURE = 'transmission failure'
 
 
-class ForeLink(object):
+class ForeLink(six.with_metaclass(abc.ABCMeta)):
   """Accepts back-to-front tickets and emits front-to-back tickets."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def accept_back_to_front_ticket(self, ticket):
@@ -334,9 +327,8 @@ class ForeLink(object):
     raise NotImplementedError()
 
 
-class RearLink(object):
+class RearLink(six.with_metaclass(abc.ABCMeta)):
   """Accepts front-to-back tickets and emits back-to-front tickets."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def accept_front_to_back_ticket(self, ticket):
@@ -353,11 +345,9 @@ class RearLink(object):
     raise NotImplementedError()
 
 
-class FrontLink(Front, ForeLink):
+class FrontLink(six.with_metaclass(abc.ABCMeta, Front, ForeLink)):
   """Clientish objects that operate by sending and receiving tickets."""
-  __metaclass__ = abc.ABCMeta
 
 
-class BackLink(Back, RearLink):
+class BackLink(six.with_metaclass(abc.ABCMeta, Back, RearLink)):
   """Serverish objects that operate by sending and receiving tickets."""
-  __metaclass__ = abc.ABCMeta
diff --git a/src/python/grpcio/grpc/framework/core/_end.py b/src/python/grpcio/grpc/framework/core/_end.py
index 9c615672aa..dc2f48589a 100644
--- a/src/python/grpcio/grpc/framework/core/_end.py
+++ b/src/python/grpcio/grpc/framework/core/_end.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@ import abc
 import threading
 import uuid
 
+import six
+
 from grpc.framework.core import _operation
 from grpc.framework.core import _utilities
 from grpc.framework.foundation import callable_util
@@ -45,7 +47,7 @@ from grpc.framework.interfaces.links import utilities
 _IDLE_ACTION_EXCEPTION_LOG_MESSAGE = 'Exception calling idle action!'
 
 
-class End(base.End, links.Link):
+class End(six.with_metaclass(abc.ABCMeta, base.End, links.Link)):
   """A bridge between base.End and links.Link.
 
   Implementations of this interface translate arriving tickets into
@@ -53,7 +55,6 @@ class End(base.End, links.Link):
   translate calls from application objects implementing base interfaces
   into tickets sent to a joined link.
   """
-  __metaclass__ = abc.ABCMeta
 
 
 class _Cycle(object):
diff --git a/src/python/grpcio/grpc/framework/core/_ingestion.py b/src/python/grpcio/grpc/framework/core/_ingestion.py
index 4129a8ce43..1e1fd73ce4 100644
--- a/src/python/grpcio/grpc/framework/core/_ingestion.py
+++ b/src/python/grpcio/grpc/framework/core/_ingestion.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@ import abc
 import collections
 import enum
 
+import six
+
 from grpc.framework.core import _constants
 from grpc.framework.core import _interfaces
 from grpc.framework.core import _utilities
@@ -70,9 +72,8 @@ class _SubscriptionCreation(
     ABANDONED = 'abandoned'
 
 
-class _SubscriptionCreator(object):
+class _SubscriptionCreator(six.with_metaclass(abc.ABCMeta)):
   """Common specification of subscription-creating behavior."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def create(self, group, method):
diff --git a/src/python/grpcio/grpc/framework/core/_interfaces.py b/src/python/grpcio/grpc/framework/core/_interfaces.py
index ffa686b2b7..985e5e8550 100644
--- a/src/python/grpcio/grpc/framework/core/_interfaces.py
+++ b/src/python/grpcio/grpc/framework/core/_interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,17 +31,18 @@
 
 import abc
 
+import six
+
 from grpc.framework.interfaces.base import base
 
 
-class TerminationManager(object):
+class TerminationManager(six.with_metaclass(abc.ABCMeta)):
   """An object responsible for handling the termination of an operation.
 
   Attributes:
     outcome: None if the operation is active or a base.Outcome value if it has
       terminated.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def add_callback(self, callback):
@@ -105,9 +106,8 @@ class TerminationManager(object):
     raise NotImplementedError()
 
 
-class TransmissionManager(object):
+class TransmissionManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for transmitting to the other end of an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def kick_off(
@@ -171,9 +171,8 @@ class TransmissionManager(object):
     raise NotImplementedError()
 
 
-class ExpirationManager(object):
+class ExpirationManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for aborting the operation if it runs out of time."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def change_timeout(self, timeout):
@@ -203,9 +202,8 @@ class ExpirationManager(object):
     raise NotImplementedError()
 
 
-class ProtocolManager(object):
+class ProtocolManager(six.with_metaclass(abc.ABCMeta)):
   """A manager of protocol-specific values passing through an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_protocol_receiver(self, protocol_receiver):
@@ -228,9 +226,8 @@ class ProtocolManager(object):
     raise NotImplementedError()
 
 
-class EmissionManager(base.Operator):
+class EmissionManager(six.with_metaclass(abc.ABCMeta, base.Operator)):
   """A manager of values emitted by customer code."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def advance(
@@ -254,14 +251,13 @@ class EmissionManager(base.Operator):
     raise NotImplementedError()
 
 
-class IngestionManager(object):
+class IngestionManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for executing customer code.
 
   This name of this manager comes from its responsibility to pass successive
   values from the other side of the operation into the code of the local
   customer.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_group_and_method(self, group, method):
@@ -294,9 +290,8 @@ class IngestionManager(object):
     raise NotImplementedError()
 
 
-class ReceptionManager(object):
+class ReceptionManager(six.with_metaclass(abc.ABCMeta)):
   """A manager responsible for receiving tickets from the other end."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def receive_ticket(self, ticket):
@@ -308,7 +303,7 @@ class ReceptionManager(object):
     raise NotImplementedError()
 
 
-class Operation(object):
+class Operation(six.with_metaclass(abc.ABCMeta)):
   """An ongoing operation.
 
   Attributes:
@@ -316,7 +311,6 @@ class Operation(object):
     operator: A base.Operator object for the operation for use by the customer
       of the operation.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def handle_ticket(self, ticket):
diff --git a/src/python/grpcio/grpc/framework/core/_termination.py b/src/python/grpcio/grpc/framework/core/_termination.py
index 364158b2b8..e8c4ec60a3 100644
--- a/src/python/grpcio/grpc/framework/core/_termination.py
+++ b/src/python/grpcio/grpc/framework/core/_termination.py
@@ -31,6 +31,8 @@
 
 import abc
 
+import six
+
 from grpc.framework.core import _constants
 from grpc.framework.core import _interfaces
 from grpc.framework.core import _utilities
@@ -50,9 +52,8 @@ def _service_completion_predicate(
   return transmission_complete and ingestion_complete
 
 
-class TerminationManager(_interfaces.TerminationManager):
+class TerminationManager(six.with_metaclass(abc.ABCMeta, _interfaces.TerminationManager)):
   """A _interfaces.TransmissionManager on which another manager may be set."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_expiration_manager(self, expiration_manager):
diff --git a/src/python/grpcio/grpc/framework/crust/_control.py b/src/python/grpcio/grpc/framework/crust/_control.py
index 5e9efdf732..c27fc9106d 100644
--- a/src/python/grpcio/grpc/framework/crust/_control.py
+++ b/src/python/grpcio/grpc/framework/crust/_control.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -405,6 +405,9 @@ class Rendezvous(base.Operator, future.Future, stream.Consumer, face.Call):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while True:
diff --git a/src/python/grpcio/grpc/framework/crust/implementations.py b/src/python/grpcio/grpc/framework/crust/implementations.py
index 4ebc4e9ae8..d0ecafcaf6 100644
--- a/src/python/grpcio/grpc/framework/crust/implementations.py
+++ b/src/python/grpcio/grpc/framework/crust/implementations.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,6 +29,8 @@
 
 """Entry points into the Crust layer of RPC Framework."""
 
+import six
+
 from grpc.framework.common import cardinality
 from grpc.framework.common import style
 from grpc.framework.crust import _calls
@@ -271,7 +273,7 @@ class _DynamicStub(face.DynamicStub):
 
 def _adapt_method_implementations(method_implementations, pool):
   adapted_implementations = {}
-  for name, method_implementation in method_implementations.iteritems():
+  for name, method_implementation in six.iteritems(method_implementations):
     if method_implementation.style is style.Service.INLINE:
       if method_implementation.cardinality is cardinality.Cardinality.UNARY_UNARY:
         adapted_implementations[name] = _service.adapt_inline_unary_unary(
diff --git a/src/python/grpcio/grpc/framework/face/_control.py b/src/python/grpcio/grpc/framework/face/_control.py
index e918907b74..ec43203a25 100644
--- a/src/python/grpcio/grpc/framework/face/_control.py
+++ b/src/python/grpcio/grpc/framework/face/_control.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -120,6 +120,9 @@ class Rendezvous(stream.Consumer):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while ((self._abortion is None) and
diff --git a/src/python/grpcio/grpc/framework/face/exceptions.py b/src/python/grpcio/grpc/framework/face/exceptions.py
index f112df70bc..c272ac75ab 100644
--- a/src/python/grpcio/grpc/framework/face/exceptions.py
+++ b/src/python/grpcio/grpc/framework/face/exceptions.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
 
 import abc
 
+import six
+
 
 class NoSuchMethodError(Exception):
   """Raised by customer code to indicate an unrecognized RPC method name.
@@ -49,12 +51,11 @@ class NoSuchMethodError(Exception):
     self.name = name
 
 
-class RpcError(Exception):
+class RpcError(six.with_metaclass(abc.ABCMeta, Exception)):
   """Common super type for all exceptions raised by the Face layer.
 
   Only RPC Framework should instantiate and raise these exceptions.
   """
-  __metaclass__ = abc.ABCMeta
 
 
 class CancellationError(RpcError):
diff --git a/src/python/grpcio/grpc/framework/face/implementations.py b/src/python/grpcio/grpc/framework/face/implementations.py
index 4a6de52974..9c75a5faf4 100644
--- a/src/python/grpcio/grpc/framework/face/implementations.py
+++ b/src/python/grpcio/grpc/framework/face/implementations.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,6 +29,8 @@
 
 """Entry points into the Face layer of RPC Framework."""
 
+import six
+
 from grpc.framework.common import cardinality
 from grpc.framework.common import style
 from grpc.framework.base import exceptions as _base_exceptions
@@ -228,7 +230,7 @@ class _DynamicStub(interfaces.DynamicStub):
 
 def _adapt_method_implementations(method_implementations, pool):
   adapted_implementations = {}
-  for name, method_implementation in method_implementations.iteritems():
+  for name, method_implementation in six.iteritems(method_implementations):
     if method_implementation.style is style.Service.INLINE:
       if method_implementation.cardinality is cardinality.Cardinality.UNARY_UNARY:
         adapted_implementations[name] = _service.adapt_inline_value_in_value_out(
diff --git a/src/python/grpcio/grpc/framework/face/interfaces.py b/src/python/grpcio/grpc/framework/face/interfaces.py
index b7cc4c1169..9fc18d73bc 100644
--- a/src/python/grpcio/grpc/framework/face/interfaces.py
+++ b/src/python/grpcio/grpc/framework/face/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import abc
 import enum
 
+import six
+
 # cardinality, style, exceptions, abandonment, future, and stream are
 # referenced from specification in this module.
 from grpc.framework.common import cardinality  # pylint: disable=unused-import
@@ -52,15 +54,17 @@ class Abortion(enum.Enum):
   SERVICER_FAILURE = 'servicer failure'
 
 
-class CancellableIterator(object):
+class CancellableIterator(six.with_metaclass(abc.ABCMeta)):
   """Implements the Iterator protocol and affords a cancel method."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __iter__(self):
     """Returns the self object in accordance with the Iterator protocol."""
     raise NotImplementedError()
 
+  def __next__(self):
+    return self.next()
+
   @abc.abstractmethod
   def next(self):
     """Returns a value or raises StopIteration per the Iterator protocol."""
@@ -72,9 +76,8 @@ class CancellableIterator(object):
     raise NotImplementedError()
 
 
-class RpcContext(object):
+class RpcContext(six.with_metaclass(abc.ABCMeta)):
   """Provides RPC-related information and action."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def is_active(self):
@@ -103,13 +106,12 @@ class RpcContext(object):
     raise NotImplementedError()
 
 
-class Call(object):
+class Call(six.with_metaclass(abc.ABCMeta)):
   """Invocation-side representation of an RPC.
 
   Attributes:
     context: An RpcContext affording information about the RPC.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def cancel(self):
@@ -117,9 +119,8 @@ class Call(object):
     raise NotImplementedError()
 
 
-class UnaryUnaryMultiCallable(object):
+class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a unary-unary RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request, timeout):
@@ -171,9 +172,8 @@ class UnaryUnaryMultiCallable(object):
     raise NotImplementedError()
 
 
-class UnaryStreamMultiCallable(object):
+class UnaryStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a unary-stream RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request, timeout):
@@ -209,9 +209,8 @@ class UnaryStreamMultiCallable(object):
     raise NotImplementedError()
 
 
-class StreamUnaryMultiCallable(object):
+class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a stream-unary RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request_iterator, timeout):
@@ -264,9 +263,8 @@ class StreamUnaryMultiCallable(object):
     raise NotImplementedError()
 
 
-class StreamStreamMultiCallable(object):
+class StreamStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a stream-stream RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request_iterator, timeout):
@@ -302,7 +300,7 @@ l    Args:
     raise NotImplementedError()
 
 
-class MethodImplementation(object):
+class MethodImplementation(six.with_metaclass(abc.ABCMeta)):
   """A sum type that describes an RPC method implementation.
 
   Attributes:
@@ -347,12 +345,10 @@ class MethodImplementation(object):
       is cardinality.Cardinality.STREAM_STREAM and style is
       style.Service.EVENT.
   """
-  __metaclass__ = abc.ABCMeta
 
 
-class MultiMethodImplementation(object):
+class MultiMethodImplementation(six.with_metaclass(abc.ABCMeta)):
   """A general type able to service many RPC methods."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def service(self, name, response_consumer, context):
@@ -381,9 +377,8 @@ class MultiMethodImplementation(object):
     raise NotImplementedError()
 
 
-class GenericStub(object):
+class GenericStub(six.with_metaclass(abc.ABCMeta)):
   """Affords RPC methods to callers."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def blocking_value_in_value_out(self, name, request, timeout):
@@ -622,7 +617,7 @@ class GenericStub(object):
     raise NotImplementedError()
 
 
-class DynamicStub(object):
+class DynamicStub(six.with_metaclass(abc.ABCMeta)):
   """A stub with RPC-method-bound multi-callable attributes.
 
   Instances of this type responsd to attribute access as follows: if the
@@ -637,4 +632,3 @@ class DynamicStub(object):
   the attribute will be a StreamStreamMultiCallable with which to invoke the
   RPC method.
   """
-  __metaclass__ = abc.ABCMeta
diff --git a/src/python/grpcio/grpc/framework/foundation/activated.py b/src/python/grpcio/grpc/framework/foundation/activated.py
index 426a71c705..9b49b6363c 100644
--- a/src/python/grpcio/grpc/framework/foundation/activated.py
+++ b/src/python/grpcio/grpc/framework/foundation/activated.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,14 +31,14 @@
 
 import abc
 
+import six
 
-class Activated(object):
+class Activated(six.with_metaclass(abc.ABCMeta)):
   """Interface for objects that may be started and stopped.
 
   Values implementing this type must also implement the context manager
   protocol.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __enter__(self):
diff --git a/src/python/grpcio/grpc/framework/foundation/callable_util.py b/src/python/grpcio/grpc/framework/foundation/callable_util.py
index 32b0751a01..e0a4cab738 100644
--- a/src/python/grpcio/grpc/framework/foundation/callable_util.py
+++ b/src/python/grpcio/grpc/framework/foundation/callable_util.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -35,8 +35,10 @@ import enum
 import functools
 import logging
 
+import six
 
-class Outcome(object):
+
+class Outcome(six.with_metaclass(abc.ABCMeta)):
   """A sum type describing the outcome of some call.
 
   Attributes:
@@ -47,7 +49,6 @@ class Outcome(object):
     exception: The exception raised by the call. Must be present if kind is
       Kind.RAISED.
   """
-  __metaclass__ = abc.ABCMeta
 
   @enum.unique
   class Kind(enum.Enum):
diff --git a/src/python/grpcio/grpc/framework/foundation/future.py b/src/python/grpcio/grpc/framework/foundation/future.py
index bfc16fc1ea..bb8ee3ad87 100644
--- a/src/python/grpcio/grpc/framework/foundation/future.py
+++ b/src/python/grpcio/grpc/framework/foundation/future.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -49,6 +49,8 @@ built-in-but-only-in-3.3-and-later TimeoutError.
 
 import abc
 
+import six
+
 
 class TimeoutError(Exception):
   """Indicates that a particular call timed out."""
@@ -58,13 +60,12 @@ class CancelledError(Exception):
   """Indicates that the computation underlying a Future was cancelled."""
 
 
-class Future(object):
+class Future(six.with_metaclass(abc.ABCMeta)):
   """A representation of a computation in another control flow.
 
   Computations represented by a Future may be yet to be begun, may be ongoing,
   or may have already completed.
   """
-  __metaclass__ = abc.ABCMeta
 
   # NOTE(nathaniel): This isn't the return type that I would want to have if it
   # were up to me. Were this interface being written from scratch, the return
diff --git a/src/python/grpcio/grpc/framework/foundation/relay.py b/src/python/grpcio/grpc/framework/foundation/relay.py
index 9c23946552..ff4e2275ae 100644
--- a/src/python/grpcio/grpc/framework/foundation/relay.py
+++ b/src/python/grpcio/grpc/framework/foundation/relay.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -46,7 +46,6 @@ class Relay(object):
   would be no reason to use an implementation of this interface instead of a
   thread pool.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def add_value(self, value):
diff --git a/src/python/grpcio/grpc/framework/foundation/stream.py b/src/python/grpcio/grpc/framework/foundation/stream.py
index 75c0cf145b..32a2e52aed 100644
--- a/src/python/grpcio/grpc/framework/foundation/stream.py
+++ b/src/python/grpcio/grpc/framework/foundation/stream.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,10 @@
 
 import abc
 
+import six
 
-class Consumer(object):
+class Consumer(six.with_metaclass(abc.ABCMeta)):
   """Interface for consumers of finite streams of values or objects."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def consume(self, value):
diff --git a/src/python/grpcio/grpc/framework/foundation/stream_util.py b/src/python/grpcio/grpc/framework/foundation/stream_util.py
index 2210e4efcf..7d5977fbbd 100644
--- a/src/python/grpcio/grpc/framework/foundation/stream_util.py
+++ b/src/python/grpcio/grpc/framework/foundation/stream_util.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -83,6 +83,9 @@ class IterableConsumer(stream.Consumer):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while self._active and not self._values:
diff --git a/src/python/grpcio/grpc/framework/interfaces/base/base.py b/src/python/grpcio/grpc/framework/interfaces/base/base.py
index a1e70be5e8..69be37e7ab 100644
--- a/src/python/grpcio/grpc/framework/interfaces/base/base.py
+++ b/src/python/grpcio/grpc/framework/interfaces/base/base.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -42,6 +42,8 @@ import abc
 import enum
 import threading  # pylint: disable=unused-import
 
+import six
+
 # abandonment is referenced from specification in this module.
 from grpc.framework.foundation import abandonment  # pylint: disable=unused-import
 
@@ -95,7 +97,7 @@ class Outcome(object):
     REMOTE_FAILURE = 'remote failure'
 
 
-class Completion(object):
+class Completion(six.with_metaclass(abc.ABCMeta)):
   """An aggregate of the values exchanged upon operation completion.
 
   Attributes:
@@ -103,12 +105,10 @@ class Completion(object):
     code: A code value for the operation.
     message: A message value for the operation.
   """
-  __metaclass__ = abc.ABCMeta
 
 
-class OperationContext(object):
+class OperationContext(six.with_metaclass(abc.ABCMeta)):
   """Provides operation-related information and action."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def outcome(self):
@@ -162,9 +162,8 @@ class OperationContext(object):
     raise NotImplementedError()
 
 
-class Operator(object):
+class Operator(six.with_metaclass(abc.ABCMeta)):
   """An interface through which to participate in an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def advance(
@@ -184,9 +183,8 @@ class Operator(object):
     """
     raise NotImplementedError()
 
-class ProtocolReceiver(object):
+class ProtocolReceiver(six.with_metaclass(abc.ABCMeta)):
   """A means of receiving protocol values during an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def context(self, protocol_context):
@@ -198,7 +196,7 @@ class ProtocolReceiver(object):
     raise NotImplementedError()
 
 
-class Subscription(object):
+class Subscription(six.with_metaclass(abc.ABCMeta)):
   """Describes customer code's interest in values from the other side.
 
   Attributes:
@@ -216,7 +214,6 @@ class Subscription(object):
       become available during the operation. Must be non-None if kind is
       Kind.FULL.
   """
-  __metaclass__ = abc.ABCMeta
 
   @enum.unique
   class Kind(enum.Enum):
@@ -226,9 +223,8 @@ class Subscription(object):
     FULL = 'full'
 
 
-class Servicer(object):
+class Servicer(six.with_metaclass(abc.ABCMeta)):
   """Interface for service implementations."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def service(self, group, method, context, output_operator):
@@ -255,9 +251,8 @@ class Servicer(object):
     raise NotImplementedError()
 
 
-class End(object):
+class End(six.with_metaclass(abc.ABCMeta)):
   """Common type for entry-point objects on both sides of an operation."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def start(self):
diff --git a/src/python/grpcio/grpc/framework/interfaces/face/face.py b/src/python/grpcio/grpc/framework/interfaces/face/face.py
index 404c3a7937..b994acecac 100644
--- a/src/python/grpcio/grpc/framework/interfaces/face/face.py
+++ b/src/python/grpcio/grpc/framework/interfaces/face/face.py
@@ -33,6 +33,8 @@ import abc
 import collections
 import enum
 
+import six
+
 # cardinality, style, abandonment, future, and stream are
 # referenced from specification in this module.
 from grpc.framework.common import cardinality  # pylint: disable=unused-import
@@ -96,7 +98,7 @@ class Abortion(
     REMOTE_FAILURE = 'remote failure'
 
 
-class AbortionError(Exception):
+class AbortionError(six.with_metaclass(abc.ABCMeta, Exception)):
   """Common super type for exceptions indicating RPC abortion.
 
     initial_metadata: The initial metadata from the other side of the RPC or
@@ -108,7 +110,6 @@ class AbortionError(Exception):
     details: The details value from the other side of the RPC or None if no
       details value was received.
   """
-  __metaclass__ = abc.ABCMeta
 
   def __init__(self, initial_metadata, terminal_metadata, code, details):
     super(AbortionError, self).__init__()
@@ -150,9 +151,8 @@ class RemoteError(AbortionError):
   """Indicates that an RPC has terminated due to a remote defect."""
 
 
-class RpcContext(object):
+class RpcContext(six.with_metaclass(abc.ABCMeta)):
   """Provides RPC-related information and action."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def is_active(self):
@@ -199,9 +199,8 @@ class RpcContext(object):
     raise NotImplementedError()
 
 
-class Call(RpcContext):
+class Call(six.with_metaclass(abc.ABCMeta, RpcContext)):
   """Invocation-side utility object for an RPC."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def initial_metadata(self):
@@ -256,9 +255,8 @@ class Call(RpcContext):
     raise NotImplementedError()
 
 
-class ServicerContext(RpcContext):
+class ServicerContext(six.with_metaclass(abc.ABCMeta, RpcContext)):
   """A context object passed to method implementations."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def invocation_metadata(self):
@@ -326,9 +324,8 @@ class ServicerContext(RpcContext):
     raise NotImplementedError()
 
 
-class ResponseReceiver(object):
+class ResponseReceiver(six.with_metaclass(abc.ABCMeta)):
   """Invocation-side object used to accept the output of an RPC."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def initial_metadata(self, initial_metadata):
@@ -362,9 +359,8 @@ class ResponseReceiver(object):
     raise NotImplementedError()
 
 
-class UnaryUnaryMultiCallable(object):
+class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a unary-unary RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(
@@ -434,9 +430,8 @@ class UnaryUnaryMultiCallable(object):
     raise NotImplementedError()
 
 
-class UnaryStreamMultiCallable(object):
+class UnaryStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a unary-stream RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(self, request, timeout, metadata=None, protocol_options=None):
@@ -480,9 +475,8 @@ class UnaryStreamMultiCallable(object):
     raise NotImplementedError()
 
 
-class StreamUnaryMultiCallable(object):
+class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a stream-unary RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(
@@ -553,9 +547,8 @@ class StreamUnaryMultiCallable(object):
     raise NotImplementedError()
 
 
-class StreamStreamMultiCallable(object):
+class StreamStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
   """Affords invoking a stream-stream RPC in any call style."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def __call__(
@@ -600,7 +593,7 @@ class StreamStreamMultiCallable(object):
     raise NotImplementedError()
 
 
-class MethodImplementation(object):
+class MethodImplementation(six.with_metaclass(abc.ABCMeta)):
   """A sum type that describes a method implementation.
 
   Attributes:
@@ -643,12 +636,10 @@ class MethodImplementation(object):
       is cardinality.Cardinality.STREAM_STREAM and style is
       style.Service.EVENT.
   """
-  __metaclass__ = abc.ABCMeta
 
 
-class MultiMethodImplementation(object):
+class MultiMethodImplementation(six.with_metaclass(abc.ABCMeta)):
   """A general type able to service many methods."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def service(self, group, method, response_consumer, context):
@@ -678,9 +669,8 @@ class MultiMethodImplementation(object):
     raise NotImplementedError()
 
 
-class GenericStub(object):
+class GenericStub(six.with_metaclass(abc.ABCMeta)):
   """Affords RPC invocation via generic methods."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def blocking_unary_unary(
@@ -977,7 +967,7 @@ class GenericStub(object):
     raise NotImplementedError()
 
 
-class DynamicStub(object):
+class DynamicStub(six.with_metaclass(abc.ABCMeta)):
   """Affords RPC invocation via attributes corresponding to afforded methods.
 
   Instances of this type may be scoped to a single group so that attribute
@@ -993,4 +983,3 @@ class DynamicStub(object):
   if the requested attribute is the name of a stream-stream method, the value of
   the attribute will be a StreamStreamMultiCallable with which to invoke an RPC.
   """
-  __metaclass__ = abc.ABCMeta
diff --git a/src/python/grpcio/grpc/framework/interfaces/links/links.py b/src/python/grpcio/grpc/framework/interfaces/links/links.py
index 24f0e3b354..808167935f 100644
--- a/src/python/grpcio/grpc/framework/interfaces/links/links.py
+++ b/src/python/grpcio/grpc/framework/interfaces/links/links.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@ import abc
 import collections
 import enum
 
+import six
+
 
 class Protocol(collections.namedtuple('Protocol', ('kind', 'value',))):
   """A sum type for handles to a system that transmits tickets.
@@ -123,9 +125,8 @@ class Ticket(
     REMOTE_FAILURE = 'remote failure'
 
 
-class Link(object):
+class Link(six.with_metaclass(abc.ABCMeta)):
   """Accepts and emits tickets."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def accept_ticket(self, ticket):
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index a543791f5c..3c57ad71da 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -30,206 +30,211 @@
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_core_dependencies.py.template`!!!
 
 CORE_SOURCE_FILES = [
-  'src/core/profiling/basic_timers.c',
-  'src/core/profiling/stap_timers.c',
-  'src/core/support/alloc.c',
-  'src/core/support/avl.c',
-  'src/core/support/cmdline.c',
-  'src/core/support/cpu_iphone.c',
-  'src/core/support/cpu_linux.c',
-  'src/core/support/cpu_posix.c',
-  'src/core/support/cpu_windows.c',
-  'src/core/support/env_linux.c',
-  'src/core/support/env_posix.c',
-  'src/core/support/env_win32.c',
-  'src/core/support/histogram.c',
-  'src/core/support/host_port.c',
-  'src/core/support/load_file.c',
-  'src/core/support/log.c',
-  'src/core/support/log_android.c',
-  'src/core/support/log_linux.c',
-  'src/core/support/log_posix.c',
-  'src/core/support/log_win32.c',
-  'src/core/support/murmur_hash.c',
-  'src/core/support/slice.c',
-  'src/core/support/slice_buffer.c',
-  'src/core/support/stack_lockfree.c',
-  'src/core/support/string.c',
-  'src/core/support/string_posix.c',
-  'src/core/support/string_win32.c',
-  'src/core/support/subprocess_posix.c',
-  'src/core/support/subprocess_windows.c',
-  'src/core/support/sync.c',
-  'src/core/support/sync_posix.c',
-  'src/core/support/sync_win32.c',
-  'src/core/support/thd.c',
-  'src/core/support/thd_posix.c',
-  'src/core/support/thd_win32.c',
-  'src/core/support/time.c',
-  'src/core/support/time_posix.c',
-  'src/core/support/time_precise.c',
-  'src/core/support/time_win32.c',
-  'src/core/support/tls_pthread.c',
-  'src/core/support/tmpfile_posix.c',
-  'src/core/support/tmpfile_win32.c',
-  'src/core/support/wrap_memcpy.c',
-  'src/core/census/grpc_context.c',
-  'src/core/census/grpc_filter.c',
-  'src/core/channel/channel_args.c',
-  'src/core/channel/channel_stack.c',
-  'src/core/channel/client_channel.c',
-  'src/core/channel/client_uchannel.c',
-  'src/core/channel/compress_filter.c',
-  'src/core/channel/connected_channel.c',
-  'src/core/channel/http_client_filter.c',
-  'src/core/channel/http_server_filter.c',
-  'src/core/channel/subchannel_call_holder.c',
-  'src/core/client_config/client_config.c',
-  'src/core/client_config/connector.c',
-  'src/core/client_config/default_initial_connect_string.c',
-  'src/core/client_config/initial_connect_string.c',
-  'src/core/client_config/lb_policies/load_balancer_api.c',
-  'src/core/client_config/lb_policies/pick_first.c',
-  'src/core/client_config/lb_policies/round_robin.c',
-  'src/core/client_config/lb_policy.c',
-  'src/core/client_config/lb_policy_factory.c',
-  'src/core/client_config/lb_policy_registry.c',
-  'src/core/client_config/resolver.c',
-  'src/core/client_config/resolver_factory.c',
-  'src/core/client_config/resolver_registry.c',
-  'src/core/client_config/resolvers/dns_resolver.c',
-  'src/core/client_config/resolvers/sockaddr_resolver.c',
-  'src/core/client_config/subchannel.c',
-  'src/core/client_config/subchannel_factory.c',
-  'src/core/client_config/subchannel_index.c',
-  'src/core/client_config/uri_parser.c',
-  'src/core/compression/compression_algorithm.c',
-  'src/core/compression/message_compress.c',
-  'src/core/debug/trace.c',
-  'src/core/httpcli/format_request.c',
-  'src/core/httpcli/httpcli.c',
-  'src/core/httpcli/parser.c',
-  'src/core/iomgr/closure.c',
-  'src/core/iomgr/endpoint.c',
-  'src/core/iomgr/endpoint_pair_posix.c',
-  'src/core/iomgr/endpoint_pair_windows.c',
-  'src/core/iomgr/exec_ctx.c',
-  'src/core/iomgr/executor.c',
-  'src/core/iomgr/fd_posix.c',
-  'src/core/iomgr/iocp_windows.c',
-  'src/core/iomgr/iomgr.c',
-  'src/core/iomgr/iomgr_posix.c',
-  'src/core/iomgr/iomgr_windows.c',
-  'src/core/iomgr/pollset_multipoller_with_epoll.c',
-  'src/core/iomgr/pollset_multipoller_with_poll_posix.c',
-  'src/core/iomgr/pollset_posix.c',
-  'src/core/iomgr/pollset_set_posix.c',
-  'src/core/iomgr/pollset_set_windows.c',
-  'src/core/iomgr/pollset_windows.c',
-  'src/core/iomgr/resolve_address_posix.c',
-  'src/core/iomgr/resolve_address_windows.c',
-  'src/core/iomgr/sockaddr_utils.c',
-  'src/core/iomgr/socket_utils_common_posix.c',
-  'src/core/iomgr/socket_utils_linux.c',
-  'src/core/iomgr/socket_utils_posix.c',
-  'src/core/iomgr/socket_windows.c',
-  'src/core/iomgr/tcp_client_posix.c',
-  'src/core/iomgr/tcp_client_windows.c',
-  'src/core/iomgr/tcp_posix.c',
-  'src/core/iomgr/tcp_server_posix.c',
-  'src/core/iomgr/tcp_server_windows.c',
-  'src/core/iomgr/tcp_windows.c',
-  'src/core/iomgr/time_averaged_stats.c',
-  'src/core/iomgr/timer.c',
-  'src/core/iomgr/timer_heap.c',
-  'src/core/iomgr/udp_server.c',
-  'src/core/iomgr/wakeup_fd_eventfd.c',
-  'src/core/iomgr/wakeup_fd_nospecial.c',
-  'src/core/iomgr/wakeup_fd_pipe.c',
-  'src/core/iomgr/wakeup_fd_posix.c',
-  'src/core/iomgr/workqueue_posix.c',
-  'src/core/iomgr/workqueue_windows.c',
-  'src/core/json/json.c',
-  'src/core/json/json_reader.c',
-  'src/core/json/json_string.c',
-  'src/core/json/json_writer.c',
-  'src/core/proto/grpc/lb/v0/load_balancer.pb.c',
-  'src/core/surface/alarm.c',
-  'src/core/surface/api_trace.c',
-  'src/core/surface/byte_buffer.c',
-  'src/core/surface/byte_buffer_reader.c',
-  'src/core/surface/call.c',
-  'src/core/surface/call_details.c',
-  'src/core/surface/call_log_batch.c',
-  'src/core/surface/channel.c',
-  'src/core/surface/channel_connectivity.c',
-  'src/core/surface/channel_create.c',
-  'src/core/surface/channel_ping.c',
-  'src/core/surface/completion_queue.c',
-  'src/core/surface/event_string.c',
-  'src/core/surface/init.c',
-  'src/core/surface/lame_client.c',
-  'src/core/surface/metadata_array.c',
-  'src/core/surface/server.c',
-  'src/core/surface/server_chttp2.c',
-  'src/core/surface/server_create.c',
-  'src/core/surface/validate_metadata.c',
-  'src/core/surface/version.c',
-  'src/core/transport/byte_stream.c',
-  'src/core/transport/chttp2/alpn.c',
-  'src/core/transport/chttp2/bin_encoder.c',
-  'src/core/transport/chttp2/frame_data.c',
-  'src/core/transport/chttp2/frame_goaway.c',
-  'src/core/transport/chttp2/frame_ping.c',
-  'src/core/transport/chttp2/frame_rst_stream.c',
-  'src/core/transport/chttp2/frame_settings.c',
-  'src/core/transport/chttp2/frame_window_update.c',
-  'src/core/transport/chttp2/hpack_encoder.c',
-  'src/core/transport/chttp2/hpack_parser.c',
-  'src/core/transport/chttp2/hpack_table.c',
-  'src/core/transport/chttp2/huffsyms.c',
-  'src/core/transport/chttp2/incoming_metadata.c',
-  'src/core/transport/chttp2/parsing.c',
-  'src/core/transport/chttp2/status_conversion.c',
-  'src/core/transport/chttp2/stream_lists.c',
-  'src/core/transport/chttp2/stream_map.c',
-  'src/core/transport/chttp2/timeout_encoding.c',
-  'src/core/transport/chttp2/varint.c',
-  'src/core/transport/chttp2/writing.c',
-  'src/core/transport/chttp2_transport.c',
-  'src/core/transport/connectivity_state.c',
-  'src/core/transport/metadata.c',
-  'src/core/transport/metadata_batch.c',
-  'src/core/transport/static_metadata.c',
-  'src/core/transport/transport.c',
-  'src/core/transport/transport_op_string.c',
-  'src/core/httpcli/httpcli_security_connector.c',
-  'src/core/security/b64.c',
-  'src/core/security/client_auth_filter.c',
-  'src/core/security/credentials.c',
-  'src/core/security/credentials_metadata.c',
-  'src/core/security/credentials_posix.c',
-  'src/core/security/credentials_win32.c',
-  'src/core/security/google_default_credentials.c',
-  'src/core/security/handshake.c',
-  'src/core/security/json_token.c',
-  'src/core/security/jwt_verifier.c',
-  'src/core/security/secure_endpoint.c',
-  'src/core/security/security_connector.c',
-  'src/core/security/security_context.c',
-  'src/core/security/server_auth_filter.c',
-  'src/core/security/server_secure_chttp2.c',
-  'src/core/surface/init_secure.c',
-  'src/core/surface/secure_channel_create.c',
-  'src/core/tsi/fake_transport_security.c',
-  'src/core/tsi/ssl_transport_security.c',
-  'src/core/tsi/transport_security.c',
-  'src/core/census/context.c',
-  'src/core/census/initialize.c',
-  'src/core/census/mlog.c',
-  'src/core/census/operation.c',
-  'src/core/census/placeholders.c',
-  'src/core/census/tracing.c',
+  'src/core/lib/profiling/basic_timers.c',
+  'src/core/lib/profiling/stap_timers.c',
+  'src/core/lib/support/alloc.c',
+  'src/core/lib/support/avl.c',
+  'src/core/lib/support/backoff.c',
+  'src/core/lib/support/cmdline.c',
+  'src/core/lib/support/cpu_iphone.c',
+  'src/core/lib/support/cpu_linux.c',
+  'src/core/lib/support/cpu_posix.c',
+  'src/core/lib/support/cpu_windows.c',
+  'src/core/lib/support/env_linux.c',
+  'src/core/lib/support/env_posix.c',
+  'src/core/lib/support/env_win32.c',
+  'src/core/lib/support/histogram.c',
+  'src/core/lib/support/host_port.c',
+  'src/core/lib/support/load_file.c',
+  'src/core/lib/support/log.c',
+  'src/core/lib/support/log_android.c',
+  'src/core/lib/support/log_linux.c',
+  'src/core/lib/support/log_posix.c',
+  'src/core/lib/support/log_win32.c',
+  'src/core/lib/support/murmur_hash.c',
+  'src/core/lib/support/slice.c',
+  'src/core/lib/support/slice_buffer.c',
+  'src/core/lib/support/stack_lockfree.c',
+  'src/core/lib/support/string.c',
+  'src/core/lib/support/string_posix.c',
+  'src/core/lib/support/string_win32.c',
+  'src/core/lib/support/subprocess_posix.c',
+  'src/core/lib/support/subprocess_windows.c',
+  'src/core/lib/support/sync.c',
+  'src/core/lib/support/sync_posix.c',
+  'src/core/lib/support/sync_win32.c',
+  'src/core/lib/support/thd.c',
+  'src/core/lib/support/thd_posix.c',
+  'src/core/lib/support/thd_win32.c',
+  'src/core/lib/support/time.c',
+  'src/core/lib/support/time_posix.c',
+  'src/core/lib/support/time_precise.c',
+  'src/core/lib/support/time_win32.c',
+  'src/core/lib/support/tls_pthread.c',
+  'src/core/lib/support/tmpfile_posix.c',
+  'src/core/lib/support/tmpfile_win32.c',
+  'src/core/lib/support/wrap_memcpy.c',
+  'src/core/ext/transport/chttp2/client/insecure/channel_create.c',
+  'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c',
+  'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
+  'src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c',
+  'src/core/ext/transport/chttp2/transport/alpn.c',
+  'src/core/ext/transport/chttp2/transport/bin_encoder.c',
+  'src/core/ext/transport/chttp2/transport/chttp2_transport.c',
+  'src/core/ext/transport/chttp2/transport/frame_data.c',
+  'src/core/ext/transport/chttp2/transport/frame_goaway.c',
+  'src/core/ext/transport/chttp2/transport/frame_ping.c',
+  'src/core/ext/transport/chttp2/transport/frame_rst_stream.c',
+  'src/core/ext/transport/chttp2/transport/frame_settings.c',
+  'src/core/ext/transport/chttp2/transport/frame_window_update.c',
+  'src/core/ext/transport/chttp2/transport/hpack_encoder.c',
+  'src/core/ext/transport/chttp2/transport/hpack_parser.c',
+  'src/core/ext/transport/chttp2/transport/hpack_table.c',
+  'src/core/ext/transport/chttp2/transport/huffsyms.c',
+  'src/core/ext/transport/chttp2/transport/incoming_metadata.c',
+  'src/core/ext/transport/chttp2/transport/parsing.c',
+  'src/core/ext/transport/chttp2/transport/status_conversion.c',
+  'src/core/ext/transport/chttp2/transport/stream_lists.c',
+  'src/core/ext/transport/chttp2/transport/stream_map.c',
+  'src/core/ext/transport/chttp2/transport/timeout_encoding.c',
+  'src/core/ext/transport/chttp2/transport/varint.c',
+  'src/core/ext/transport/chttp2/transport/writing.c',
+  'src/core/lib/census/context.c',
+  'src/core/lib/census/grpc_context.c',
+  'src/core/lib/census/grpc_filter.c',
+  'src/core/lib/census/grpc_plugin.c',
+  'src/core/lib/census/initialize.c',
+  'src/core/lib/census/mlog.c',
+  'src/core/lib/census/operation.c',
+  'src/core/lib/census/placeholders.c',
+  'src/core/lib/census/tracing.c',
+  'src/core/lib/channel/channel_args.c',
+  'src/core/lib/channel/channel_stack.c',
+  'src/core/lib/channel/channel_stack_builder.c',
+  'src/core/lib/channel/client_channel.c',
+  'src/core/lib/channel/compress_filter.c',
+  'src/core/lib/channel/connected_channel.c',
+  'src/core/lib/channel/http_client_filter.c',
+  'src/core/lib/channel/http_server_filter.c',
+  'src/core/lib/channel/subchannel_call_holder.c',
+  'src/core/lib/client_config/client_config.c',
+  'src/core/lib/client_config/connector.c',
+  'src/core/lib/client_config/default_initial_connect_string.c',
+  'src/core/lib/client_config/initial_connect_string.c',
+  'src/core/lib/client_config/lb_policies/load_balancer_api.c',
+  'src/core/lib/client_config/lb_policies/pick_first.c',
+  'src/core/lib/client_config/lb_policies/round_robin.c',
+  'src/core/lib/client_config/lb_policy.c',
+  'src/core/lib/client_config/lb_policy_factory.c',
+  'src/core/lib/client_config/lb_policy_registry.c',
+  'src/core/lib/client_config/resolver.c',
+  'src/core/lib/client_config/resolver_factory.c',
+  'src/core/lib/client_config/resolver_registry.c',
+  'src/core/lib/client_config/resolvers/dns_resolver.c',
+  'src/core/lib/client_config/resolvers/sockaddr_resolver.c',
+  'src/core/lib/client_config/subchannel.c',
+  'src/core/lib/client_config/subchannel_factory.c',
+  'src/core/lib/client_config/subchannel_index.c',
+  'src/core/lib/client_config/uri_parser.c',
+  'src/core/lib/compression/compression_algorithm.c',
+  'src/core/lib/compression/message_compress.c',
+  'src/core/lib/debug/trace.c',
+  'src/core/lib/http/format_request.c',
+  'src/core/lib/http/httpcli.c',
+  'src/core/lib/http/httpcli_security_connector.c',
+  'src/core/lib/http/parser.c',
+  'src/core/lib/iomgr/closure.c',
+  'src/core/lib/iomgr/endpoint.c',
+  'src/core/lib/iomgr/endpoint_pair_posix.c',
+  'src/core/lib/iomgr/endpoint_pair_windows.c',
+  'src/core/lib/iomgr/exec_ctx.c',
+  'src/core/lib/iomgr/executor.c',
+  'src/core/lib/iomgr/fd_posix.c',
+  'src/core/lib/iomgr/iocp_windows.c',
+  'src/core/lib/iomgr/iomgr.c',
+  'src/core/lib/iomgr/iomgr_posix.c',
+  'src/core/lib/iomgr/iomgr_windows.c',
+  'src/core/lib/iomgr/pollset_multipoller_with_epoll.c',
+  'src/core/lib/iomgr/pollset_multipoller_with_poll_posix.c',
+  'src/core/lib/iomgr/pollset_posix.c',
+  'src/core/lib/iomgr/pollset_set_posix.c',
+  'src/core/lib/iomgr/pollset_set_windows.c',
+  'src/core/lib/iomgr/pollset_windows.c',
+  'src/core/lib/iomgr/resolve_address_posix.c',
+  'src/core/lib/iomgr/resolve_address_windows.c',
+  'src/core/lib/iomgr/sockaddr_utils.c',
+  'src/core/lib/iomgr/socket_utils_common_posix.c',
+  'src/core/lib/iomgr/socket_utils_linux.c',
+  'src/core/lib/iomgr/socket_utils_posix.c',
+  'src/core/lib/iomgr/socket_windows.c',
+  'src/core/lib/iomgr/tcp_client_posix.c',
+  'src/core/lib/iomgr/tcp_client_windows.c',
+  'src/core/lib/iomgr/tcp_posix.c',
+  'src/core/lib/iomgr/tcp_server_posix.c',
+  'src/core/lib/iomgr/tcp_server_windows.c',
+  'src/core/lib/iomgr/tcp_windows.c',
+  'src/core/lib/iomgr/time_averaged_stats.c',
+  'src/core/lib/iomgr/timer.c',
+  'src/core/lib/iomgr/timer_heap.c',
+  'src/core/lib/iomgr/udp_server.c',
+  'src/core/lib/iomgr/unix_sockets_posix.c',
+  'src/core/lib/iomgr/unix_sockets_posix_noop.c',
+  'src/core/lib/iomgr/wakeup_fd_eventfd.c',
+  'src/core/lib/iomgr/wakeup_fd_nospecial.c',
+  'src/core/lib/iomgr/wakeup_fd_pipe.c',
+  'src/core/lib/iomgr/wakeup_fd_posix.c',
+  'src/core/lib/iomgr/workqueue_posix.c',
+  'src/core/lib/iomgr/workqueue_windows.c',
+  'src/core/lib/json/json.c',
+  'src/core/lib/json/json_reader.c',
+  'src/core/lib/json/json_string.c',
+  'src/core/lib/json/json_writer.c',
+  'src/core/lib/proto/grpc/lb/v0/load_balancer.pb.c',
+  'src/core/lib/security/b64.c',
+  'src/core/lib/security/client_auth_filter.c',
+  'src/core/lib/security/credentials.c',
+  'src/core/lib/security/credentials_metadata.c',
+  'src/core/lib/security/credentials_posix.c',
+  'src/core/lib/security/credentials_win32.c',
+  'src/core/lib/security/google_default_credentials.c',
+  'src/core/lib/security/handshake.c',
+  'src/core/lib/security/json_token.c',
+  'src/core/lib/security/jwt_verifier.c',
+  'src/core/lib/security/secure_endpoint.c',
+  'src/core/lib/security/security_connector.c',
+  'src/core/lib/security/security_context.c',
+  'src/core/lib/security/server_auth_filter.c',
+  'src/core/lib/surface/alarm.c',
+  'src/core/lib/surface/api_trace.c',
+  'src/core/lib/surface/byte_buffer.c',
+  'src/core/lib/surface/byte_buffer_reader.c',
+  'src/core/lib/surface/call.c',
+  'src/core/lib/surface/call_details.c',
+  'src/core/lib/surface/call_log_batch.c',
+  'src/core/lib/surface/channel.c',
+  'src/core/lib/surface/channel_connectivity.c',
+  'src/core/lib/surface/channel_init.c',
+  'src/core/lib/surface/channel_ping.c',
+  'src/core/lib/surface/channel_stack_type.c',
+  'src/core/lib/surface/completion_queue.c',
+  'src/core/lib/surface/event_string.c',
+  'src/core/lib/surface/init.c',
+  'src/core/lib/surface/init_secure.c',
+  'src/core/lib/surface/lame_client.c',
+  'src/core/lib/surface/metadata_array.c',
+  'src/core/lib/surface/server.c',
+  'src/core/lib/surface/validate_metadata.c',
+  'src/core/lib/surface/version.c',
+  'src/core/lib/transport/byte_stream.c',
+  'src/core/lib/transport/connectivity_state.c',
+  'src/core/lib/transport/metadata.c',
+  'src/core/lib/transport/metadata_batch.c',
+  'src/core/lib/transport/static_metadata.c',
+  'src/core/lib/transport/transport.c',
+  'src/core/lib/transport/transport_op_string.c',
+  'src/core/lib/tsi/fake_transport_security.c',
+  'src/core/lib/tsi/ssl_transport_security.c',
+  'src/core/lib/tsi/transport_security.c',
   'third_party/nanopb/pb_common.c',
   'third_party/nanopb/pb_decode.c',
   'third_party/nanopb/pb_encode.c',
diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py
index ae2a0c835a..aeeb2754ea 100644
--- a/src/python/grpcio/precompiled.py
+++ b/src/python/grpcio/precompiled.py
@@ -31,6 +31,7 @@ import os
 import platform
 import shutil
 import sys
+import sysconfig
 
 import setuptools
 
@@ -51,9 +52,15 @@ USE_PRECOMPILED_BINARIES = bool(int(os.environ.get(
 
 def _tagged_ext_name(base):
   uname = platform.uname()
-  tags = '-'.join((grpc_version.VERSION, uname[0], uname[4]))
-  flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
-  return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor)
+  tags = (
+      grpc_version.VERSION,
+      'py{}'.format(sysconfig.get_python_version()),
+      uname[0],
+      uname[4],
+  )
+  ucs = 'ucs{}'.format(sysconfig.get_config_var('Py_UNICODE_SIZE'))
+  return '{base}-{tags}-{ucs}'.format(
+      base=base, tags='-'.join(tags), ucs=ucs)
 
 
 class BuildTaggedExt(setuptools.Command):
@@ -84,6 +91,9 @@ class BuildTaggedExt(setuptools.Command):
 
 
 def update_setup_arguments(setup_arguments):
+  if not USE_PRECOMPILED_BINARIES:
+    sys.stderr.write('not using precompiled extension')
+    return
   url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc'))
   target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so')
   try:
diff --git a/src/python/grpcio/tests/__init__.py b/src/python/grpcio/tests/__init__.py
index b76b3985a1..c3b80d766d 100644
--- a/src/python/grpcio/tests/__init__.py
+++ b/src/python/grpcio/tests/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,6 +27,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import absolute_import
+
 from tests import _loader
 from tests import _runner
 
diff --git a/src/python/grpcio/tests/_loader.py b/src/python/grpcio/tests/_loader.py
index 6992029b5e..2f9e5c660e 100644
--- a/src/python/grpcio/tests/_loader.py
+++ b/src/python/grpcio/tests/_loader.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,6 +27,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import absolute_import
+
 import importlib
 import pkgutil
 import re
diff --git a/src/python/grpcio/tests/_result.py b/src/python/grpcio/tests/_result.py
index 0670be921f..18b0f43963 100644
--- a/src/python/grpcio/tests/_result.py
+++ b/src/python/grpcio/tests/_result.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,7 +27,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import cStringIO as StringIO
+from __future__ import absolute_import
+
 import collections
 import itertools
 import traceback
@@ -35,6 +36,7 @@ import unittest
 from xml.etree import ElementTree
 
 import coverage
+from six import moves
 
 from tests import _loader
 
@@ -202,7 +204,7 @@ class AugmentedResult(unittest.TestResult):
     """
     case_id = self.id_map(test)
     self.cases[case_id] = self.cases[case_id].updated(
-        stdout=stdout, stderr=stderr)
+        stdout=stdout.decode(), stderr=stderr.decode())
 
   def augmented_results(self, filter):
     """Convenience method to retrieve filtered case results.
@@ -356,7 +358,7 @@ def _traceback_string(type, value, trace):
   Returns:
     str: Formatted exception descriptive string.
   """
-  buffer = StringIO.StringIO()
+  buffer = moves.cStringIO()
   traceback.print_exception(type, value, trace, file=buffer)
   return buffer.getvalue()
 
diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py
index 38a5432e79..173a170409 100644
--- a/src/python/grpcio/tests/_runner.py
+++ b/src/python/grpcio/tests/_runner.py
@@ -27,7 +27,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import cStringIO as StringIO
+from __future__ import absolute_import
+
 import collections
 import fcntl
 import multiprocessing
@@ -35,69 +36,59 @@ import os
 import select
 import signal
 import sys
+import tempfile
 import threading
 import time
 import unittest
 import uuid
 
+import six
+from six import moves
+
 from tests import _loader
 from tests import _result
 
 
-class CapturePipe(object):
-  """A context-manager pipe to redirect output to a byte array.
+class CaptureFile(object):
+  """A context-managed file to redirect output to a byte array.
+
+  Use by invoking `start` (`__enter__`) and at some point invoking `stop`
+  (`__exit__`). At any point after the initial call to `start` call `output` to
+  get the current redirected output. Note that we don't currently use file
+  locking, so calling `output` between calls to `start` and `stop` may muddle
+  the result (you should only be doing this during a Python-handled interrupt as
+  a last ditch effort to provide output to the user).
 
   Attributes:
-    _redirect_fd (int): File descriptor of file to redirect writes from.
+    _redirected_fd (int): File descriptor of file to redirect writes from.
     _saved_fd (int): A copy of the original value of the redirected file
       descriptor.
-    _read_thread (threading.Thread or None): Thread upon which reads through the
-      pipe are performed. Only non-None when self is started.
-    _read_fd (int or None): File descriptor of the read end of the redirect
-      pipe. Only non-None when self is started.
-    _write_fd (int or None): File descriptor of the write end of the redirect
-      pipe. Only non-None when self is started.
-    output (bytearray or None): Redirected output from writes to the redirected
-      file descriptor. Only valid during and after self has started.
+    _into_file (TemporaryFile or None): File to which writes are redirected.
+      Only non-None when self is started.
   """
 
   def __init__(self, fd):
-    self._redirect_fd = fd
-    self._saved_fd = os.dup(self._redirect_fd)
-    self._read_thread = None
-    self._read_fd = None
-    self._write_fd = None
-    self.output = None
+    self._redirected_fd = fd
+    self._saved_fd = os.dup(self._redirected_fd)
+    self._into_file = None
+
+  def output(self):
+    """Get all output from the redirected-to file if it exists."""
+    if self._into_file:
+      self._into_file.seek(0)
+      return bytes(self._into_file.read())
+    else:
+      return bytes()
 
   def start(self):
     """Start redirection of writes to the file descriptor."""
-    self._read_fd, self._write_fd = os.pipe()
-    os.dup2(self._write_fd, self._redirect_fd)
-    flags = fcntl.fcntl(self._read_fd, fcntl.F_GETFL)
-    fcntl.fcntl(self._read_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-    self._read_thread = threading.Thread(target=self._read)
-    self._read_thread.start()
+    self._into_file = tempfile.TemporaryFile()
+    os.dup2(self._into_file.fileno(), self._redirected_fd)
 
   def stop(self):
     """Stop redirection of writes to the file descriptor."""
-    os.close(self._write_fd)
-    os.dup2(self._saved_fd, self._redirect_fd)  # auto-close self._redirect_fd
-    self._read_thread.join()
-    self._read_thread = None
-    # we waited for the read thread to finish, so _read_fd has been read and we
-    # can close it.
-    os.close(self._read_fd)
-
-  def _read(self):
-    """Read-thread target for self."""
-    self.output = bytearray()
-    while True:
-      select.select([self._read_fd], [], [])
-      read_bytes = os.read(self._read_fd, 1024)
-      if read_bytes:
-        self.output.extend(read_bytes)
-      else:
-        break
+    # n.b. this dup2 call auto-closes self._redirected_fd
+    os.dup2(self._saved_fd, self._redirected_fd)
 
   def write_bypass(self, value):
     """Bypass the redirection and write directly to the original file.
@@ -105,6 +96,8 @@ class CapturePipe(object):
     Arguments:
       value (str): What to write to the original file.
     """
+    if six.PY3 and not isinstance(value, six.binary_type):
+      value = bytes(value, 'ascii')
     if self._saved_fd is None:
       os.write(self._redirect_fd, value)
     else:
@@ -144,7 +137,7 @@ class Runner(object):
   def run(self, suite):
     """See setuptools' test_runner setup argument for information."""
     # only run test cases with id starting with given prefix
-    testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER')
+    testcase_filter = os.getenv('GRPC_PYTHON_TESTRUNNER_FILTER')
     filtered_cases = []
     for case in _loader.iterate_suite_cases(suite):
       if not testcase_filter or case.id().startswith(testcase_filter):
@@ -156,11 +149,11 @@ class Runner(object):
                        for case in filtered_cases]
     case_id_by_case = dict((augmented_case.case, augmented_case.id)
                            for augmented_case in augmented_cases)
-    result_out = StringIO.StringIO()
+    result_out = moves.cStringIO()
     result = _result.TerminalResult(
         result_out, id_map=lambda case: case_id_by_case[case])
-    stdout_pipe = CapturePipe(sys.stdout.fileno())
-    stderr_pipe = CapturePipe(sys.stderr.fileno())
+    stdout_pipe = CaptureFile(sys.stdout.fileno())
+    stderr_pipe = CaptureFile(sys.stderr.fileno())
     kill_flag = [False]
 
     def sigint_handler(signal_number, frame):
@@ -171,7 +164,8 @@ class Runner(object):
     def fault_handler(signal_number, frame):
       stdout_pipe.write_bypass(
           'Received fault signal {}\nstdout:\n{}\n\nstderr:{}\n'
-          .format(signal_number, stdout_pipe.output, stderr_pipe.output))
+          .format(signal_number, stdout_pipe.output(),
+                  stderr_pipe.output()))
       os._exit(1)
 
     def check_kill_self():
@@ -180,9 +174,9 @@ class Runner(object):
         result.stopTestRun()
         stdout_pipe.write_bypass(result_out.getvalue())
         stdout_pipe.write_bypass(
-            '\ninterrupted stdout:\n{}\n'.format(stdout_pipe.output))
+            '\ninterrupted stdout:\n{}\n'.format(stdout_pipe.output().decode()))
         stderr_pipe.write_bypass(
-            '\ninterrupted stderr:\n{}\n'.format(stderr_pipe.output))
+            '\ninterrupted stderr:\n{}\n'.format(stderr_pipe.output().decode()))
         os._exit(1)
     signal.signal(signal.SIGINT, sigint_handler)
     signal.signal(signal.SIGSEGV, fault_handler)
@@ -212,7 +206,7 @@ class Runner(object):
         # re-raise the exception after forcing the with-block to end
         raise
       result.set_output(
-          augmented_case.case, stdout_pipe.output, stderr_pipe.output)
+          augmented_case.case, stdout_pipe.output(), stderr_pipe.output())
       sys.stdout.write(result_out.getvalue())
       sys.stdout.flush()
       result_out.truncate(0)
@@ -225,7 +219,7 @@ class Runner(object):
     sys.stdout.write(result_out.getvalue())
     sys.stdout.flush()
     signal.signal(signal.SIGINT, signal.SIG_DFL)
-    with open('report.xml', 'w') as report_xml_file:
+    with open('report.xml', 'wb') as report_xml_file:
       _result.jenkins_junit_xml(result).write(report_xml_file)
     return result
 
diff --git a/src/python/grpcio/tests/interop/methods.py b/src/python/grpcio/tests/interop/methods.py
index b3591aef7b..7f42b4a005 100644
--- a/src/python/grpcio/tests/interop/methods.py
+++ b/src/python/grpcio/tests/interop/methods.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,6 +29,8 @@
 
 """Implementations of interoperability test methods."""
 
+from __future__ import print_function
+
 import enum
 import json
 import os
@@ -173,6 +175,9 @@ class _Pipe(object):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while not self._values and self._open:
@@ -205,7 +210,7 @@ def _ping_pong(stub):
 
   with stub, _Pipe() as pipe:
     response_iterator = stub.FullDuplexCall(pipe, _TIMEOUT)
-    print 'Starting ping-pong with response iterator %s' % response_iterator
+    print('Starting ping-pong with response iterator %s' % response_iterator)
     for response_size, payload_size in zip(
         request_response_sizes, request_payload_sizes):
       request = messages_pb2.StreamingOutputCallRequest(
diff --git a/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py b/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py
index ba5b219a88..230ec6487d 100644
--- a/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py
+++ b/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -42,6 +42,8 @@ import threading
 import time
 import unittest
 
+from six import moves
+
 from grpc.beta import implementations
 from grpc.framework.foundation import future
 from grpc.framework.interfaces.face import face
@@ -250,7 +252,7 @@ class PythonPluginTest(unittest.TestCase):
   def testImportAttributes(self):
     # check that we can access the generated module and its members.
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     self.assertIsNotNone(getattr(test_pb2, SERVICER_IDENTIFIER, None))
     self.assertIsNotNone(getattr(test_pb2, STUB_IDENTIFIER, None))
     self.assertIsNotNone(getattr(test_pb2, SERVER_FACTORY_IDENTIFIER, None))
@@ -258,13 +260,13 @@ class PythonPluginTest(unittest.TestCase):
 
   def testUpDown(self):
     import protoc_plugin_test_pb2 as test_pb2
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (servicer, stub):
       request = test_pb2.SimpleRequest(response_size=13)
 
   def testUnaryCall(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       request = test_pb2.SimpleRequest(response_size=13)
       response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT)
@@ -273,7 +275,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testUnaryCallFuture(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = test_pb2.SimpleRequest(response_size=13)
     with _CreateService(test_pb2) as (methods, stub):
       # Check that the call does not block waiting for the server to respond.
@@ -286,7 +288,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testUnaryCallFutureExpired(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       request = test_pb2.SimpleRequest(response_size=13)
       with methods.pause():
@@ -297,7 +299,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testUnaryCallFutureCancelled(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = test_pb2.SimpleRequest(response_size=13)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.pause():
@@ -307,7 +309,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testUnaryCallFutureFailed(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = test_pb2.SimpleRequest(response_size=13)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.fail():
@@ -317,20 +319,20 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingOutputCall(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = _streaming_output_request(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       responses = stub.StreamingOutputCall(
           request, test_constants.LONG_TIMEOUT)
       expected_responses = methods.StreamingOutputCall(
           request, 'not a real RpcContext!')
-      for expected_response, response in itertools.izip_longest(
+      for expected_response, response in moves.zip_longest(
           expected_responses, responses):
         self.assertEqual(expected_response, response)
 
   def testStreamingOutputCallExpired(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = _streaming_output_request(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.pause():
@@ -341,7 +343,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingOutputCallCancelled(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = _streaming_output_request(test_pb2)
     with _CreateService(test_pb2) as (unused_methods, stub):
       responses = stub.StreamingOutputCall(
@@ -353,7 +355,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingOutputCallFailed(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request = _streaming_output_request(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.fail():
@@ -364,7 +366,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingInputCall(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       response = stub.StreamingInputCall(
           _streaming_input_request_iterator(test_pb2),
@@ -375,7 +377,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingInputCallFuture(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
@@ -388,7 +390,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingInputCallFutureExpired(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
@@ -401,7 +403,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingInputCallFutureCancelled(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
@@ -414,7 +416,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testStreamingInputCallFutureFailed(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.fail():
         response_future = stub.StreamingInputCall.future(
@@ -424,19 +426,19 @@ class PythonPluginTest(unittest.TestCase):
 
   def testFullDuplexCall(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       responses = stub.FullDuplexCall(
           _full_duplex_request_iterator(test_pb2), test_constants.LONG_TIMEOUT)
       expected_responses = methods.FullDuplexCall(
           _full_duplex_request_iterator(test_pb2), 'not a real RpcContext!')
-      for expected_response, response in itertools.izip_longest(
+      for expected_response, response in moves.zip_longest(
           expected_responses, responses):
         self.assertEqual(expected_response, response)
 
   def testFullDuplexCallExpired(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request_iterator = _full_duplex_request_iterator(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.pause():
@@ -447,7 +449,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testFullDuplexCallCancelled(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       request_iterator = _full_duplex_request_iterator(test_pb2)
       responses = stub.FullDuplexCall(
@@ -459,7 +461,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testFullDuplexCallFailed(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     request_iterator = _full_duplex_request_iterator(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       with methods.fail():
@@ -471,7 +473,7 @@ class PythonPluginTest(unittest.TestCase):
 
   def testHalfDuplexCall(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     with _CreateService(test_pb2) as (methods, stub):
       def half_duplex_request_iterator():
         request = test_pb2.StreamingOutputCallRequest()
@@ -485,13 +487,13 @@ class PythonPluginTest(unittest.TestCase):
           half_duplex_request_iterator(), test_constants.LONG_TIMEOUT)
       expected_responses = methods.HalfDuplexCall(
           half_duplex_request_iterator(), 'not a real RpcContext!')
-      for check in itertools.izip_longest(expected_responses, responses):
+      for check in moves.zip_longest(expected_responses, responses):
         expected_response, response = check
         self.assertEqual(expected_response, response)
 
   def testHalfDuplexCallWedged(self):
     import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    reload(test_pb2)
+    moves.reload_module(test_pb2)
     condition = threading.Condition()
     wait_cell = [False]
     @contextlib.contextmanager
diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json
index 388d040d5c..84870aaa5c 100644
--- a/src/python/grpcio/tests/tests.json
+++ b/src/python/grpcio/tests/tests.json
@@ -12,31 +12,22 @@
   "_core_over_links_base_interface_test.SyncEasyTest", 
   "_core_over_links_base_interface_test.SyncPeasyTest", 
   "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
   "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
   "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
   "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
   "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
   "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
   "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
-  "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
   "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
-  "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
   "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
-  "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
   "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_implementations_test.ChannelCredentialsTest", 
   "_insecure_interop_test.InsecureInteropTest", 
diff --git a/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py b/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py
index a6fd82388c..06bfc34977 100644
--- a/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py
+++ b/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,11 +29,13 @@
 
 """Tests for the old '_low'."""
 
-import Queue
 import threading
 import time
 import unittest
 
+import six
+from six.moves import queue
+
 from grpc._adapter import _intermediary_low as _low
 
 _STREAM_LENGTH = 300
@@ -67,7 +69,7 @@ class LonelyClientTest(unittest.TestCase):
     second_event = completion_queue.get(after_deadline)
     self.assertIsNotNone(second_event)
     kinds = [event.kind for event in (first_event, second_event)]
-    self.assertItemsEqual(
+    six.assertCountEqual(self,
         (_low.Event.Kind.METADATA_ACCEPTED, _low.Event.Kind.FINISH),
         kinds)
 
@@ -99,7 +101,7 @@ class EchoTest(unittest.TestCase):
     self.server = _low.Server(self.server_completion_queue)
     port = self.server.add_http2_addr('[::]:0')
     self.server.start()
-    self.server_events = Queue.Queue()
+    self.server_events = queue.Queue()
     self.server_completion_queue_thread = threading.Thread(
         target=_drive_completion_queue,
         args=(self.server_completion_queue, self.server_events))
@@ -107,7 +109,7 @@ class EchoTest(unittest.TestCase):
 
     self.client_completion_queue = _low.CompletionQueue()
     self.channel = _low.Channel('%s:%d' % (self.host, port), None)
-    self.client_events = Queue.Queue()
+    self.client_events = queue.Queue()
     self.client_completion_queue_thread = threading.Thread(
         target=_drive_completion_queue,
         args=(self.client_completion_queue, self.client_events))
@@ -315,7 +317,7 @@ class CancellationTest(unittest.TestCase):
     self.server = _low.Server(self.server_completion_queue)
     port = self.server.add_http2_addr('[::]:0')
     self.server.start()
-    self.server_events = Queue.Queue()
+    self.server_events = queue.Queue()
     self.server_completion_queue_thread = threading.Thread(
         target=_drive_completion_queue,
         args=(self.server_completion_queue, self.server_events))
@@ -323,7 +325,7 @@ class CancellationTest(unittest.TestCase):
 
     self.client_completion_queue = _low.CompletionQueue()
     self.channel = _low.Channel('%s:%d' % (self.host, port), None)
-    self.client_events = Queue.Queue()
+    self.client_events = queue.Queue()
     self.client_completion_queue_thread = threading.Thread(
         target=_drive_completion_queue,
         args=(self.client_completion_queue, self.client_events))
diff --git a/src/python/grpcio/tests/unit/_adapter/_proto_scenarios.py b/src/python/grpcio/tests/unit/_adapter/_proto_scenarios.py
index f55a7a23ea..c9f36636b5 100644
--- a/src/python/grpcio/tests/unit/_adapter/_proto_scenarios.py
+++ b/src/python/grpcio/tests/unit/_adapter/_proto_scenarios.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,12 +32,13 @@
 import abc
 import threading
 
+import six
+
 from tests.unit._junkdrawer import math_pb2
 
 
-class ProtoScenario(object):
+class ProtoScenario(six.with_metaclass(abc.ABCMeta)):
   """An RPC test scenario using protocol buffers."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def method(self):
diff --git a/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py b/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py
index efc990421a..881633754c 100644
--- a/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py
+++ b/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -35,6 +35,8 @@ import random
 import time
 import unittest
 
+import six
+
 from grpc._adapter import _intermediary_low
 from grpc._links import invocation
 from grpc._links import service
@@ -68,7 +70,7 @@ def _serialization_behaviors_from_serializations(serializations):
   request_deserializers = {}
   response_serializers = {}
   response_deserializers = {}
-  for (group, method), serialization in serializations.iteritems():
+  for (group, method), serialization in six.iteritems(serializations):
     request_serializers[group, method] = serialization.serialize_request
     request_deserializers[group, method] = serialization.deserialize_request
     response_serializers[group, method] = serialization.serialize_response
diff --git a/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py b/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py
index 4faaaadc2b..3be3b051fb 100644
--- a/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py
+++ b/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import collections
 import unittest
 
+import six
+
 from grpc._adapter import _intermediary_low
 from grpc._links import invocation
 from grpc._links import service
@@ -59,7 +61,7 @@ def _serialization_behaviors_from_test_methods(test_methods):
   request_deserializers = {}
   response_serializers = {}
   response_deserializers = {}
-  for (group, method), test_method in test_methods.iteritems():
+  for (group, method), test_method in six.iteritems(test_methods):
     request_serializers[group, method] = test_method.serialize_request
     request_deserializers[group, method] = test_method.deserialize_request
     response_serializers[group, method] = test_method.serialize_response
@@ -108,7 +110,7 @@ class _Implementation(test_interfaces.Implementation):
     # _digest.TestServiceDigest.
     cardinalities = {
         method: method_object.cardinality()
-        for (group, method), method_object in methods.iteritems()}
+        for (group, method), method_object in six.iteritems(methods)}
     dynamic_stub = crust_implementations.dynamic_stub(
         invocation_end_link, group, cardinalities, pool)
 
diff --git a/src/python/grpcio/tests/unit/_links/_proto_scenarios.py b/src/python/grpcio/tests/unit/_links/_proto_scenarios.py
index f69ff51b16..acd4891390 100644
--- a/src/python/grpcio/tests/unit/_links/_proto_scenarios.py
+++ b/src/python/grpcio/tests/unit/_links/_proto_scenarios.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,13 +32,14 @@
 import abc
 import threading
 
+import six
+
 from tests.unit._junkdrawer import math_pb2
 from tests.unit.framework.common import test_constants
 
 
-class ProtoScenario(object):
+class ProtoScenario(six.with_metaclass(abc.ABCMeta)):
   """An RPC test scenario using protocol buffers."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def group_and_method(self):
diff --git a/src/python/grpcio/tests/unit/beta/_beta_features_test.py b/src/python/grpcio/tests/unit/beta/_beta_features_test.py
index ea44177b49..ebdedcc11e 100644
--- a/src/python/grpcio/tests/unit/beta/_beta_features_test.py
+++ b/src/python/grpcio/tests/unit/beta/_beta_features_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -125,6 +125,9 @@ class _BlockingIterator(object):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while True:
diff --git a/src/python/grpcio/tests/unit/beta/_face_interface_test.py b/src/python/grpcio/tests/unit/beta/_face_interface_test.py
index 1c21dfd03d..cb302bbf68 100644
--- a/src/python/grpcio/tests/unit/beta/_face_interface_test.py
+++ b/src/python/grpcio/tests/unit/beta/_face_interface_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import collections
 import unittest
 
+import six
+
 from grpc.beta import implementations
 from grpc.beta import interfaces
 from tests.unit import resources
@@ -57,7 +59,7 @@ def _serialization_behaviors_from_test_methods(test_methods):
   request_deserializers = {}
   response_serializers = {}
   response_deserializers = {}
-  for (group, method), test_method in test_methods.iteritems():
+  for (group, method), test_method in six.iteritems(test_methods):
     request_serializers[group, method] = test_method.serialize_request
     request_deserializers[group, method] = test_method.deserialize_request
     response_serializers[group, method] = test_method.serialize_response
@@ -79,7 +81,7 @@ class _Implementation(test_interfaces.Implementation):
     # _digest.TestServiceDigest.
     cardinalities = {
         method: method_object.cardinality()
-        for (group, method), method_object in methods.iteritems()}
+        for (group, method), method_object in six.iteritems(methods)}
 
     server_options = implementations.server_options(
         request_deserializers=serialization_behaviors.request_deserializers,
diff --git a/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py b/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py
index 360ecc95d5..fd2d4298f9 100644
--- a/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py
+++ b/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import collections
 import unittest
 
+import six
+
 from grpc.framework.core import implementations as core_implementations
 from grpc.framework.crust import implementations as crust_implementations
 from grpc.framework.foundation import logging_pool
@@ -66,7 +68,7 @@ class _Implementation(test_interfaces.Implementation):
     # _digest.TestServiceDigest.
     cardinalities = {
         method: method_object.cardinality()
-        for (group, method), method_object in methods.iteritems()}
+        for (group, method), method_object in six.iteritems(methods)}
     dynamic_stub = crust_implementations.dynamic_stub(
         invocation_end_link, group, cardinalities, pool)
 
diff --git a/src/python/grpcio/tests/unit/framework/common/test_control.py b/src/python/grpcio/tests/unit/framework/common/test_control.py
index 8d6eba5c2c..0387668b11 100644
--- a/src/python/grpcio/tests/unit/framework/common/test_control.py
+++ b/src/python/grpcio/tests/unit/framework/common/test_control.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@ import abc
 import contextlib
 import threading
 
+import six
+
 
 class Defect(Exception):
   """Simulates a programming defect raised into in a system under test.
@@ -42,7 +44,7 @@ class Defect(Exception):
   """
 
 
-class Control(object):
+class Control(six.with_metaclass(abc.ABCMeta)):
   """An object that accepts program control from a system under test.
 
   Systems under test passed a Control should call its control() method
@@ -51,8 +53,6 @@ class Control(object):
   the system under test to simulate hanging, failing, or functioning.
   """
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def control(self):
     """Potentially does anything."""
diff --git a/src/python/grpcio/tests/unit/framework/common/test_coverage.py b/src/python/grpcio/tests/unit/framework/common/test_coverage.py
index a7ed3582c4..184621fb5c 100644
--- a/src/python/grpcio/tests/unit/framework/common/test_coverage.py
+++ b/src/python/grpcio/tests/unit/framework/common/test_coverage.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,14 @@
 
 import abc
 
+import six
+
 # This code is designed for use with the unittest module.
 # pylint: disable=invalid-name
 
 
-class Coverage(object):
+class Coverage(six.with_metaclass(abc.ABCMeta)):
   """Specification of test coverage."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def testSuccessfulUnaryRequestUnaryResponse(self):
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/base_util.py b/src/python/grpcio/tests/unit/framework/face/testing/base_util.py
index 1df1529b27..60ab5bc0fe 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/base_util.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/base_util.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
 
 import abc
 
+import six
+
 # interfaces is referenced from specification in this module.
 from grpc.framework.base import util as _base_util
 from grpc.framework.base import implementations
@@ -43,7 +45,7 @@ _POOL_SIZE_LIMIT = 5
 _MAXIMUM_TIMEOUT = 90
 
 
-class LinkedPair(object):
+class LinkedPair(six.with_metaclass(abc.ABCMeta)):
   """A Front and Back that are linked to one another.
 
   Attributes:
@@ -51,8 +53,6 @@ class LinkedPair(object):
     back: An interfaces.Back.
   """
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def shut_down(self):
     """Shuts down this object and releases its resources."""
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/blocking_invocation_inline_service_test_case.py b/src/python/grpcio/tests/unit/framework/face/testing/blocking_invocation_inline_service_test_case.py
index 0613516421..b0b00bfa81 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/blocking_invocation_inline_service_test_case.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/blocking_invocation_inline_service_test_case.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@
 import abc
 import unittest  # pylint: disable=unused-import
 
+import six
+
 from grpc.framework.face import exceptions
 from tests.unit.framework.common import test_constants
 from tests.unit.framework.face.testing import control
@@ -43,12 +45,12 @@ from tests.unit.framework.face.testing import test_case
 
 
 class BlockingInvocationInlineServiceTestCase(
-    test_case.FaceTestCase, coverage.BlockingCoverage):
+    six.with_metaclass(abc.ABCMeta,
+    test_case.FaceTestCase, coverage.BlockingCoverage)):
   """A test of the Face layer of RPC Framework.
 
   Concrete subclasses must also extend unittest.TestCase.
   """
-  __metaclass__ = abc.ABCMeta
 
   def setUp(self):
     """See unittest.TestCase.setUp for full specification.
@@ -72,7 +74,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testSuccessfulUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -83,7 +85,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testSuccessfulUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -95,7 +97,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testSuccessfulStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -106,7 +108,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testSuccessfulStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -118,7 +120,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testSequentialInvocations(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -135,7 +137,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testExpiredUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -146,7 +148,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testExpiredUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -158,7 +160,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testExpiredStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -169,7 +171,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testExpiredStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -181,7 +183,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testFailedUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -191,7 +193,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testFailedUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -202,7 +204,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testFailedStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -212,7 +214,7 @@ class BlockingInvocationInlineServiceTestCase(
 
   def testFailedStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/control.py b/src/python/grpcio/tests/unit/framework/face/testing/control.py
index 3960c4e649..0d40331e19 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/control.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/control.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,8 +33,10 @@ import abc
 import contextlib
 import threading
 
+import six
 
-class Control(object):
+
+class Control(six.with_metaclass(abc.ABCMeta)):
   """An object that accepts program control from a system under test.
 
   Systems under test passed a Control should call its control() method
@@ -43,8 +45,6 @@ class Control(object):
   the system under test to simulate hanging, failing, or functioning.
   """
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def control(self):
     """Potentially does anything."""
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/coverage.py b/src/python/grpcio/tests/unit/framework/face/testing/coverage.py
index f3aca113fe..9f5381069d 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/coverage.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/coverage.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,15 +31,15 @@
 
 import abc
 
+import six
+
 # These classes are only valid when inherited by unittest.TestCases.
 # pylint: disable=invalid-name
 
 
-class BlockingCoverage(object):
+class BlockingCoverage(six.with_metaclass(abc.ABCMeta)):
   """Specification of test coverage for blocking behaviors."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def testSuccessfulUnaryRequestUnaryResponse(self):
     raise NotImplementedError()
@@ -93,11 +93,9 @@ class BlockingCoverage(object):
     raise NotImplementedError()
 
 
-class FullCoverage(BlockingCoverage):
+class FullCoverage(six.with_metaclass(abc.ABCMeta, BlockingCoverage)):
   """Specification of test coverage for non-blocking behaviors."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def testParallelInvocations(self):
     raise NotImplementedError()
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/digest.py b/src/python/grpcio/tests/unit/framework/face/testing/digest.py
index 39f28b9657..100067cc83 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/digest.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/digest.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import collections
 import threading
 
+import six
+
 # testing_control, interfaces, and testing_service are referenced from
 # specification in this module.
 from grpc.framework.common import cardinality
@@ -368,7 +370,7 @@ def _assemble(
   events = {}
   adaptations = {}
   messages = {}
-  for name, scenario in scenarios.iteritems():
+  for name, scenario in six.iteritems(scenarios):
     if name in names:
       raise ValueError('Repeated name "%s"!' % name)
 
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/event_invocation_synchronous_event_service_test_case.py b/src/python/grpcio/tests/unit/framework/face/testing/event_invocation_synchronous_event_service_test_case.py
index 179f3a2f67..db901cfe4e 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/event_invocation_synchronous_event_service_test_case.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/event_invocation_synchronous_event_service_test_case.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import abc
 import unittest
 
+import six
+
 from grpc.framework.face import interfaces
 from tests.unit.framework.common import test_constants
 from tests.unit.framework.face.testing import callback as testing_callback
@@ -43,12 +45,12 @@ from tests.unit.framework.face.testing import test_case
 
 
 class EventInvocationSynchronousEventServiceTestCase(
-    test_case.FaceTestCase, coverage.FullCoverage):
+    six.with_metaclass(abc.ABCMeta,
+    test_case.FaceTestCase, coverage.FullCoverage)):
   """A test of the Face layer of RPC Framework.
 
   Concrete subclasses must also extend unittest.TestCase.
   """
-  __metaclass__ = abc.ABCMeta
 
   def setUp(self):
     """See unittest.TestCase.setUp for full specification.
@@ -72,7 +74,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testSuccessfulUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -87,7 +89,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testSuccessfulUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -102,7 +104,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testSuccessfulStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = testing_callback.Callback()
@@ -120,7 +122,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testSuccessfulStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = testing_callback.Callback()
@@ -138,7 +140,7 @@ class EventInvocationSynchronousEventServiceTestCase(
   def testSequentialInvocations(self):
     # pylint: disable=cell-var-from-loop
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -163,7 +165,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testExpiredUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -178,7 +180,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testExpiredUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -193,7 +195,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testExpiredStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for unused_test_messages in test_messages_sequence:
         callback = testing_callback.Callback()
 
@@ -206,7 +208,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testExpiredStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = testing_callback.Callback()
@@ -221,7 +223,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testFailedUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -237,7 +239,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testFailedUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -253,7 +255,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testFailedStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = testing_callback.Callback()
@@ -272,7 +274,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testFailedStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = testing_callback.Callback()
@@ -289,7 +291,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testParallelInvocations(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         first_callback = testing_callback.Callback()
@@ -316,7 +318,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testCancelledUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -332,7 +334,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testCancelledUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = testing_callback.Callback()
@@ -347,7 +349,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testCancelledStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = testing_callback.Callback()
@@ -364,7 +366,7 @@ class EventInvocationSynchronousEventServiceTestCase(
 
   def testCancelledStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for unused_test_messages in test_messages_sequence:
         callback = testing_callback.Callback()
 
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/future_invocation_asynchronous_event_service_test_case.py b/src/python/grpcio/tests/unit/framework/face/testing/future_invocation_asynchronous_event_service_test_case.py
index 485524a356..d8706aa39e 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/future_invocation_asynchronous_event_service_test_case.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/future_invocation_asynchronous_event_service_test_case.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -34,6 +34,8 @@ import contextlib
 import threading
 import unittest
 
+import six
+
 from grpc.framework.face import exceptions
 from grpc.framework.foundation import future
 from grpc.framework.foundation import logging_pool
@@ -66,6 +68,9 @@ class _PauseableIterator(object):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while self._paused:
@@ -74,12 +79,12 @@ class _PauseableIterator(object):
 
 
 class FutureInvocationAsynchronousEventServiceTestCase(
-    test_case.FaceTestCase, coverage.FullCoverage):
+    six.with_metaclass(abc.ABCMeta,
+    test_case.FaceTestCase, coverage.FullCoverage)):
   """A test of the Face layer of RPC Framework.
 
   Concrete subclasses must also extend unittest.TestCase.
   """
-  __metaclass__ = abc.ABCMeta
 
   def setUp(self):
     """See unittest.TestCase.setUp for full specification.
@@ -105,7 +110,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testSuccessfulUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -117,7 +122,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testSuccessfulUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -129,7 +134,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testSuccessfulStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         request_iterator = _PauseableIterator(iter(requests))
@@ -145,7 +150,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testSuccessfulStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         request_iterator = _PauseableIterator(iter(requests))
@@ -161,7 +166,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testSequentialInvocations(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -180,7 +185,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testExpiredUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -195,7 +200,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testExpiredUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -207,7 +212,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testExpiredStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -222,7 +227,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testExpiredStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -234,7 +239,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testFailedUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -253,7 +258,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testFailedUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -268,7 +273,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testFailedStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -287,7 +292,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testFailedStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -302,7 +307,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testParallelInvocations(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -324,7 +329,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testCancelledUnaryRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -338,7 +343,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testCancelledUnaryRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -352,7 +357,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testCancelledStreamRequestUnaryResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -366,7 +371,7 @@ class FutureInvocationAsynchronousEventServiceTestCase(
 
   def testCancelledStreamRequestStreamResponse(self):
     for name, test_messages_sequence in (
-        self.digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self.digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/interfaces.py b/src/python/grpcio/tests/unit/framework/face/testing/interfaces.py
index 5932dabf1e..87be836e2d 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/interfaces.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,14 @@
 
 import abc
 
+import six
+
 # cardinality is referenced from specification in this module.
 from grpc.framework.common import cardinality  # pylint: disable=unused-import
 
 
-class Method(object):
+class Method(six.with_metaclass(abc.ABCMeta)):
   """An RPC method to be used in tests of RPC implementations."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def name(self):
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/service.py b/src/python/grpcio/tests/unit/framework/face/testing/service.py
index ac0b89b6ee..dc0f204c04 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/service.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/service.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,16 @@
 
 import abc
 
+import six
+
 # interfaces is referenced from specification in this module.
 from grpc.framework.face import interfaces as face_interfaces  # pylint: disable=unused-import
 from tests.unit.framework.face.testing import interfaces
 
 
-class UnaryUnaryTestMethodImplementation(interfaces.Method):
+class UnaryUnaryTestMethodImplementation(six.with_metaclass(abc.ABCMeta, interfaces.Method)):
   """A controllable implementation of a unary-unary RPC method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, request, response_callback, context, control):
     """Services an RPC that accepts one message and produces one message.
@@ -59,11 +59,9 @@ class UnaryUnaryTestMethodImplementation(interfaces.Method):
     raise NotImplementedError()
 
 
-class UnaryUnaryTestMessages(object):
+class UnaryUnaryTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for unary-request-unary-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def request(self):
     """Affords a request message.
@@ -93,11 +91,9 @@ class UnaryUnaryTestMessages(object):
     raise NotImplementedError()
 
 
-class UnaryStreamTestMethodImplementation(interfaces.Method):
+class UnaryStreamTestMethodImplementation(six.with_metaclass(abc.ABCMeta, interfaces.Method)):
   """A controllable implementation of a unary-stream RPC method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, request, response_consumer, context, control):
     """Services an RPC that takes one message and produces a stream of messages.
@@ -116,11 +112,9 @@ class UnaryStreamTestMethodImplementation(interfaces.Method):
     raise NotImplementedError()
 
 
-class UnaryStreamTestMessages(object):
+class UnaryStreamTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for unary-request-stream-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def request(self):
     """Affords a request message.
@@ -150,11 +144,9 @@ class UnaryStreamTestMessages(object):
     raise NotImplementedError()
 
 
-class StreamUnaryTestMethodImplementation(interfaces.Method):
+class StreamUnaryTestMethodImplementation(six.with_metaclass(abc.ABCMeta, interfaces.Method)):
   """A controllable implementation of a stream-unary RPC method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, response_callback, context, control):
     """Services an RPC that takes a stream of messages and produces one message.
@@ -180,11 +172,9 @@ class StreamUnaryTestMethodImplementation(interfaces.Method):
     raise NotImplementedError()
 
 
-class StreamUnaryTestMessages(object):
+class StreamUnaryTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for stream-request-unary-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def requests(self):
     """Affords a sequence of request messages.
@@ -214,11 +204,9 @@ class StreamUnaryTestMessages(object):
     raise NotImplementedError()
 
 
-class StreamStreamTestMethodImplementation(interfaces.Method):
+class StreamStreamTestMethodImplementation(six.with_metaclass(abc.ABCMeta, interfaces.Method)):
   """A controllable implementation of a stream-stream RPC method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, response_consumer, context, control):
     """Services an RPC that accepts and produces streams of messages.
@@ -244,11 +232,9 @@ class StreamStreamTestMethodImplementation(interfaces.Method):
     raise NotImplementedError()
 
 
-class StreamStreamTestMessages(object):
+class StreamStreamTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for stream-request-stream-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def requests(self):
     """Affords a sequence of request messages.
@@ -278,11 +264,9 @@ class StreamStreamTestMessages(object):
     raise NotImplementedError()
 
 
-class TestService(object):
+class TestService(six.with_metaclass(abc.ABCMeta)):
   """A specification of implemented RPC methods to use in tests."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def name(self):
     """Identifies the RPC service name used during the test.
diff --git a/src/python/grpcio/tests/unit/framework/face/testing/test_case.py b/src/python/grpcio/tests/unit/framework/face/testing/test_case.py
index 23d4d919c2..5be9330a77 100644
--- a/src/python/grpcio/tests/unit/framework/face/testing/test_case.py
+++ b/src/python/grpcio/tests/unit/framework/face/testing/test_case.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,18 +31,19 @@
 
 import abc
 
+import six
+
 # face_interfaces and interfaces are referenced in specification in this module.
 from grpc.framework.face import interfaces as face_interfaces  # pylint: disable=unused-import
 from tests.unit.framework.face.testing import interfaces  # pylint: disable=unused-import
 
 
-class FaceTestCase(object):
+class FaceTestCase(six.with_metaclass(abc.ABCMeta)):
   """Describes a test of the Face Layer of RPC Framework.
 
   Concrete subclasses must also inherit from unittest.TestCase and from at least
   one class that defines test methods.
   """
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def set_up_implementation(
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/base/_control.py b/src/python/grpcio/tests/unit/framework/interfaces/base/_control.py
index 38102b198a..94bcc1428e 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/base/_control.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/base/_control.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,6 +29,8 @@
 
 """Part of the tests of the base interface of RPC Framework."""
 
+from __future__ import division
+
 import abc
 import collections
 import enum
@@ -36,6 +38,8 @@ import random  # pylint: disable=unused-import
 import threading
 import time
 
+import six
+
 from grpc.framework.interfaces.base import base
 from tests.unit.framework.common import test_constants
 from tests.unit.framework.interfaces.base import _sequence
@@ -45,8 +49,8 @@ from tests.unit.framework.interfaces.base import test_interfaces  # pylint: disa
 _GROUP = 'base test cases test group'
 _METHOD = 'base test cases test method'
 
-_PAYLOAD_RANDOM_SECTION_MAXIMUM_SIZE = test_constants.PAYLOAD_SIZE / 20
-_MINIMUM_PAYLOAD_SIZE = test_constants.PAYLOAD_SIZE / 600
+_PAYLOAD_RANDOM_SECTION_MAXIMUM_SIZE = test_constants.PAYLOAD_SIZE // 20
+_MINIMUM_PAYLOAD_SIZE = test_constants.PAYLOAD_SIZE // 600
 
 
 def _create_payload(randomness):
@@ -57,7 +61,7 @@ def _create_payload(randomness):
   random_section = bytes(
       bytearray(
           randomness.getrandbits(8) for _ in range(random_section_length)))
-  sevens_section = '\x07' * (length - random_section_length)
+  sevens_section = b'\x07' * (length - random_section_length)
   return b''.join(randomness.sample((random_section, sevens_section), 2))
 
 
@@ -247,8 +251,7 @@ class Instruction(
     CONCLUDE = 'CONCLUDE'
 
 
-class Controller(object):
-  __metaclass__ = abc.ABCMeta
+class Controller(six.with_metaclass(abc.ABCMeta)):
 
   @abc.abstractmethod
   def failed(self, message):
@@ -308,8 +311,7 @@ class Controller(object):
     raise NotImplementedError()
 
 
-class ControllerCreator(object):
-  __metaclass__ = abc.ABCMeta
+class ControllerCreator(six.with_metaclass(abc.ABCMeta)):
 
   @abc.abstractmethod
   def name(self):
@@ -385,13 +387,13 @@ class _SequenceController(Controller):
     return request + request
 
   def deserialize_request(self, serialized_request):
-    return serialized_request[:len(serialized_request) / 2]
+    return serialized_request[:len(serialized_request) // 2]
 
   def serialize_response(self, response):
     return response * 3
 
   def deserialize_response(self, serialized_response):
-    return serialized_response[2 * len(serialized_response) / 3:]
+    return serialized_response[2 * len(serialized_response) // 3:]
 
   def invocation(self):
     with self._condition:
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/base/test_interfaces.py b/src/python/grpcio/tests/unit/framework/interfaces/base/test_interfaces.py
index 84afd24d47..0594cfeb31 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/base/test_interfaces.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/base/test_interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,12 +31,13 @@
 
 import abc
 
+import six
+
 from grpc.framework.interfaces.base import base  # pylint: disable=unused-import
 
 
-class Serialization(object):
+class Serialization(six.with_metaclass(abc.ABCMeta)):
   """Specifies serialization and deserialization of test payloads."""
-  __metaclass__ = abc.ABCMeta
 
   def serialize_request(self, request):
     """Serializes a request value used in a test.
@@ -85,9 +86,8 @@ class Serialization(object):
     raise NotImplementedError()
 
 
-class Implementation(object):
+class Implementation(six.with_metaclass(abc.ABCMeta)):
   """Specifies an implementation of the Base layer."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def instantiate(self, serializations, servicer):
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py
index c8a3a1bc74..936b87f597 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py
@@ -29,11 +29,15 @@
 
 """Test code for the Face layer of RPC Framework."""
 
+from __future__ import division
+
 import abc
 import itertools
 import unittest
 from concurrent import futures
 
+import six
+
 # test_interfaces is referenced from specification in this module.
 from grpc.framework.foundation import logging_pool
 from grpc.framework.interfaces.face import face
@@ -46,14 +50,13 @@ from tests.unit.framework.interfaces.face import _stock_service
 from tests.unit.framework.interfaces.face import test_interfaces  # pylint: disable=unused-import
 
 
-class TestCase(test_coverage.Coverage, unittest.TestCase):
+class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.TestCase)):
   """A test of the Face layer of RPC Framework.
 
   Concrete subclasses must have an "implementation" attribute of type
   test_interfaces.Implementation and an "invoker_constructor" attribute of type
   _invocation.InvokerConstructor.
   """
-  __metaclass__ = abc.ABCMeta
 
   NAME = 'BlockingInvocationInlineServiceTest'
 
@@ -81,7 +84,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -92,7 +95,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -104,7 +107,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -115,7 +118,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -127,7 +130,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSequentialInvocations(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -145,7 +148,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
   def testParallelInvocations(self):
     pool = logging_pool.pool(test_constants.PARALLELISM)
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures = []
@@ -167,7 +170,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
   def testWaitingForSomeButNotAllParallelInvocations(self):
     pool = logging_pool.pool(test_constants.PARALLELISM)
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures_to_indices = {}
@@ -181,7 +184,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
         some_completed_response_futures_iterator = itertools.islice(
             futures.as_completed(response_futures_to_indices),
-            test_constants.PARALLELISM / 2)
+            test_constants.PARALLELISM // 2)
         for response_future in some_completed_response_futures_iterator:
           index = response_futures_to_indices[response_future]
           test_messages.verify(requests[index], response_future.result(), self)
@@ -205,7 +208,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -216,7 +219,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -228,7 +231,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -239,7 +242,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -251,7 +254,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -261,7 +264,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -272,7 +275,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -282,7 +285,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_digest.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_digest.py
index 9304b6b1db..40c03f9e71 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_digest.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_digest.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,8 @@
 import collections
 import threading
 
+import six
+
 # test_control, _service, and test_interfaces are referenced from specification
 # in this module.
 from grpc.framework.common import cardinality
@@ -363,7 +365,7 @@ def _assemble(
   events = {}
   adaptations = {}
   messages = {}
-  for identifier, scenario in scenarios.iteritems():
+  for identifier, scenario in six.iteritems(scenarios):
     if identifier in identifiers:
       raise ValueError('Repeated identifier "(%s, %s)"!' % identifier)
 
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py
deleted file mode 100644
index 34db6c3e55..0000000000
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py
+++ /dev/null
@@ -1,381 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Test code for the Face layer of RPC Framework."""
-
-import abc
-import unittest
-
-# test_interfaces is referenced from specification in this module.
-from grpc.framework.interfaces.face import face
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.common import test_control
-from tests.unit.framework.common import test_coverage
-from tests.unit.framework.interfaces.face import _3069_test_constant
-from tests.unit.framework.interfaces.face import _digest
-from tests.unit.framework.interfaces.face import _receiver
-from tests.unit.framework.interfaces.face import _stock_service
-from tests.unit.framework.interfaces.face import test_interfaces  # pylint: disable=unused-import
-
-
-class TestCase(test_coverage.Coverage, unittest.TestCase):
-  """A test of the Face layer of RPC Framework.
-
-  Concrete subclasses must have an "implementation" attribute of type
-  test_interfaces.Implementation and an "invoker_constructor" attribute of type
-  _invocation.InvokerConstructor.
-  """
-  __metaclass__ = abc.ABCMeta
-
-  NAME = 'EventInvocationSynchronousEventServiceTest'
-
-  def setUp(self):
-    """See unittest.TestCase.setUp for full specification.
-
-    Overriding implementations must call this implementation.
-    """
-    self._control = test_control.PauseFailControl()
-    self._digest = _digest.digest(
-        _stock_service.STOCK_TEST_SERVICE, self._control, None)
-
-    generic_stub, dynamic_stubs, self._memo = self.implementation.instantiate(
-        self._digest.methods, self._digest.event_method_implementations, None)
-    self._invoker = self.invoker_constructor.construct_invoker(
-        generic_stub, dynamic_stubs, self._digest.methods)
-
-  def tearDown(self):
-    """See unittest.TestCase.tearDown for full specification.
-
-    Overriding implementations must call this implementation.
-    """
-    self._invoker = None
-    self.implementation.destantiate(self._memo)
-
-  def testSuccessfulUnaryRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        self._invoker.event(group, method)(
-            request, receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        receiver.block_until_terminated()
-        response = receiver.unary_response()
-
-        test_messages.verify(request, response, self)
-
-  def testSuccessfulUnaryRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        self._invoker.event(group, method)(
-            request, receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        receiver.block_until_terminated()
-        responses = receiver.stream_responses()
-
-        test_messages.verify(request, responses, self)
-
-  def testSuccessfulStreamRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        requests = test_messages.requests()
-        receiver = _receiver.Receiver()
-
-        call_consumer = self._invoker.event(group, method)(
-            receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        for request in requests:
-          call_consumer.consume(request)
-        call_consumer.terminate()
-        receiver.block_until_terminated()
-        response = receiver.unary_response()
-
-        test_messages.verify(requests, response, self)
-
-  def testSuccessfulStreamRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        requests = test_messages.requests()
-        receiver = _receiver.Receiver()
-
-        call_consumer = self._invoker.event(group, method)(
-            receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        for request in requests:
-          call_consumer.consume(request)
-        call_consumer.terminate()
-        receiver.block_until_terminated()
-        responses = receiver.stream_responses()
-
-        test_messages.verify(requests, responses, self)
-
-  def testSequentialInvocations(self):
-    # pylint: disable=cell-var-from-loop
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        first_request = test_messages.request()
-        second_request = test_messages.request()
-        second_receiver = _receiver.Receiver()
-
-        def make_second_invocation():
-          self._invoker.event(group, method)(
-              second_request, second_receiver, second_receiver.abort,
-              test_constants.LONG_TIMEOUT)
-
-        class FirstReceiver(_receiver.Receiver):
-
-          def complete(self, terminal_metadata, code, details):
-            super(FirstReceiver, self).complete(
-                terminal_metadata, code, details)
-            make_second_invocation()
-
-        first_receiver = FirstReceiver()
-
-        self._invoker.event(group, method)(
-            first_request, first_receiver, first_receiver.abort,
-            test_constants.LONG_TIMEOUT)
-        second_receiver.block_until_terminated()
-
-        first_response = first_receiver.unary_response()
-        second_response = second_receiver.unary_response()
-        test_messages.verify(first_request, first_response, self)
-        test_messages.verify(second_request, second_response, self)
-
-  def testParallelInvocations(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        first_request = test_messages.request()
-        first_receiver = _receiver.Receiver()
-        second_request = test_messages.request()
-        second_receiver = _receiver.Receiver()
-
-        self._invoker.event(group, method)(
-            first_request, first_receiver, first_receiver.abort,
-            test_constants.LONG_TIMEOUT)
-        self._invoker.event(group, method)(
-            second_request, second_receiver, second_receiver.abort,
-            test_constants.LONG_TIMEOUT)
-        first_receiver.block_until_terminated()
-        second_receiver.block_until_terminated()
-
-        first_response = first_receiver.unary_response()
-        second_response = second_receiver.unary_response()
-        test_messages.verify(first_request, first_response, self)
-        test_messages.verify(second_request, second_response, self)
-
-  @unittest.skip('TODO(nathaniel): implement.')
-  def testWaitingForSomeButNotAllParallelInvocations(self):
-    raise NotImplementedError()
-
-  def testCancelledUnaryRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        with self._control.pause():
-          call = self._invoker.event(group, method)(
-              request, receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-          call.cancel()
-          receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind)
-
-  def testCancelledUnaryRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        call = self._invoker.event(group, method)(
-            request, receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        call.cancel()
-        receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind)
-
-  def testCancelledStreamRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        requests = test_messages.requests()
-        receiver = _receiver.Receiver()
-
-        call_consumer = self._invoker.event(group, method)(
-            receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        for request in requests:
-          call_consumer.consume(request)
-        call_consumer.cancel()
-        receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind)
-
-  def testCancelledStreamRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
-      for unused_test_messages in test_messages_sequence:
-        receiver = _receiver.Receiver()
-
-        call_consumer = self._invoker.event(group, method)(
-            receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-        call_consumer.cancel()
-        receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind)
-
-  def testExpiredUnaryRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        with self._control.pause():
-          self._invoker.event(group, method)(
-              request, receiver, receiver.abort,
-              _3069_test_constant.REALLY_SHORT_TIMEOUT)
-          receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind)
-
-  def testExpiredUnaryRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        with self._control.pause():
-          self._invoker.event(group, method)(
-              request, receiver, receiver.abort,
-              _3069_test_constant.REALLY_SHORT_TIMEOUT)
-          receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind)
-
-  def testExpiredStreamRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
-      for unused_test_messages in test_messages_sequence:
-        receiver = _receiver.Receiver()
-
-        self._invoker.event(group, method)(
-            receiver, receiver.abort, _3069_test_constant.REALLY_SHORT_TIMEOUT)
-        receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind)
-
-  def testExpiredStreamRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        requests = test_messages.requests()
-        receiver = _receiver.Receiver()
-
-        call_consumer = self._invoker.event(group, method)(
-            receiver, receiver.abort, _3069_test_constant.REALLY_SHORT_TIMEOUT)
-        for request in requests:
-          call_consumer.consume(request)
-        receiver.block_until_terminated()
-
-        self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind)
-
-  def testFailedUnaryRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        with self._control.fail():
-          self._invoker.event(group, method)(
-              request, receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-          receiver.block_until_terminated()
-
-        self.assertIs(
-            face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind)
-
-  def testFailedUnaryRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        request = test_messages.request()
-        receiver = _receiver.Receiver()
-
-        with self._control.fail():
-          self._invoker.event(group, method)(
-              request, receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-          receiver.block_until_terminated()
-
-        self.assertIs(
-            face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind)
-
-  def testFailedStreamRequestUnaryResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        requests = test_messages.requests()
-        receiver = _receiver.Receiver()
-
-        with self._control.fail():
-          call_consumer = self._invoker.event(group, method)(
-              receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-          for request in requests:
-            call_consumer.consume(request)
-          call_consumer.terminate()
-          receiver.block_until_terminated()
-
-        self.assertIs(
-            face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind)
-
-  def testFailedStreamRequestStreamResponse(self):
-    for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
-      for test_messages in test_messages_sequence:
-        requests = test_messages.requests()
-        receiver = _receiver.Receiver()
-
-        with self._control.fail():
-          call_consumer = self._invoker.event(group, method)(
-              receiver, receiver.abort, test_constants.LONG_TIMEOUT)
-          for request in requests:
-            call_consumer.consume(request)
-          call_consumer.terminate()
-          receiver.block_until_terminated()
-
-        self.assertIs(
-            face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind)
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py
index 1d36a931e8..401b52f614 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py
@@ -29,6 +29,8 @@
 
 """Test code for the Face layer of RPC Framework."""
 
+from __future__ import division
+
 import abc
 import contextlib
 import itertools
@@ -36,6 +38,8 @@ import threading
 import unittest
 from concurrent import futures
 
+import six
+
 # test_interfaces is referenced from specification in this module.
 from grpc.framework.foundation import logging_pool
 from grpc.framework.interfaces.face import face
@@ -67,6 +71,9 @@ class _PauseableIterator(object):
   def __iter__(self):
     return self
 
+  def __next__(self):
+    return self.next()
+
   def next(self):
     with self._condition:
       while self._paused:
@@ -104,14 +111,13 @@ class _Callback(object):
           self._condition.wait()
 
 
-class TestCase(test_coverage.Coverage, unittest.TestCase):
+class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.TestCase)):
   """A test of the Face layer of RPC Framework.
 
   Concrete subclasses must have an "implementation" attribute of type
   test_interfaces.Implementation and an "invoker_constructor" attribute of type
   _invocation.InvokerConstructor.
   """
-  __metaclass__ = abc.ABCMeta
 
   NAME = 'FutureInvocationAsynchronousEventServiceTest'
 
@@ -141,7 +147,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = _Callback()
@@ -156,7 +162,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -168,7 +174,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         request_iterator = _PauseableIterator(iter(requests))
@@ -188,7 +194,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSuccessfulStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         request_iterator = _PauseableIterator(iter(requests))
@@ -204,7 +210,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testSequentialInvocations(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -223,7 +229,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testParallelInvocations(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         first_request = test_messages.request()
         second_request = test_messages.request()
@@ -239,7 +245,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
         test_messages.verify(second_request, second_response, self)
 
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures = []
@@ -259,7 +265,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
   def testWaitingForSomeButNotAllParallelInvocations(self):
     pool = logging_pool.pool(test_constants.PARALLELISM)
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures_to_indices = {}
@@ -273,7 +279,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
         some_completed_response_futures_iterator = itertools.islice(
             futures.as_completed(response_futures_to_indices),
-            test_constants.PARALLELISM / 2)
+            test_constants.PARALLELISM // 2)
         for response_future in some_completed_response_futures_iterator:
           index = response_futures_to_indices[response_future]
           test_messages.verify(requests[index], response_future.result(), self)
@@ -281,7 +287,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testCancelledUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = _Callback()
@@ -298,7 +304,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testCancelledUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -312,7 +318,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testCancelledStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = _Callback()
@@ -329,7 +335,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testCancelledStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -343,7 +349,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = _Callback()
@@ -360,7 +366,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -372,7 +378,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = _Callback()
@@ -389,7 +395,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testExpiredStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
@@ -401,7 +407,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedUnaryRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
         callback = _Callback()
@@ -423,7 +429,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedUnaryRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.unary_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.unary_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         request = test_messages.request()
 
@@ -438,7 +444,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedStreamRequestUnaryResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_unary_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
         callback = _Callback()
@@ -460,7 +466,7 @@ class TestCase(test_coverage.Coverage, unittest.TestCase):
 
   def testFailedStreamRequestStreamResponse(self):
     for (group, method), test_messages_sequence in (
-        self._digest.stream_stream_messages_sequences.iteritems()):
+        six.iteritems(self._digest.stream_stream_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = test_messages.requests()
 
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_invocation.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_invocation.py
index 448e845a08..ff38dc2ece 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_invocation.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_invocation.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
 
 import abc
 
+import six
+
 from grpc.framework.common import cardinality
 
 _CARDINALITY_TO_GENERIC_BLOCKING_BEHAVIOR = {
@@ -62,9 +64,8 @@ _CARDINALITY_TO_MULTI_CALLABLE_ATTRIBUTE = {
 }
 
 
-class Invoker(object):
+class Invoker(six.with_metaclass(abc.ABCMeta)):
   """A type used to invoke test RPCs."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def blocking(self, group, name):
@@ -82,9 +83,8 @@ class Invoker(object):
     raise NotImplementedError()
 
 
-class InvokerConstructor(object):
+class InvokerConstructor(six.with_metaclass(abc.ABCMeta)):
   """A type used to create Invokers."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def name(self):
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_service.py
index 28941e2ad0..bec8d5113c 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_service.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_service.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,16 +31,16 @@
 
 import abc
 
+import six
+
 # face is referenced from specification in this module.
 from grpc.framework.interfaces.face import face  # pylint: disable=unused-import
 from tests.unit.framework.interfaces.face import test_interfaces
 
 
-class UnaryUnaryTestMethodImplementation(test_interfaces.Method):
+class UnaryUnaryTestMethodImplementation(six.with_metaclass(abc.ABCMeta, test_interfaces.Method)):
   """A controllable implementation of a unary-unary method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, request, response_callback, context, control):
     """Services an RPC that accepts one message and produces one message.
@@ -59,11 +59,9 @@ class UnaryUnaryTestMethodImplementation(test_interfaces.Method):
     raise NotImplementedError()
 
 
-class UnaryUnaryTestMessages(object):
+class UnaryUnaryTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for unary-request-unary-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def request(self):
     """Affords a request message.
@@ -93,11 +91,9 @@ class UnaryUnaryTestMessages(object):
     raise NotImplementedError()
 
 
-class UnaryStreamTestMethodImplementation(test_interfaces.Method):
+class UnaryStreamTestMethodImplementation(six.with_metaclass(abc.ABCMeta, test_interfaces.Method)):
   """A controllable implementation of a unary-stream method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, request, response_consumer, context, control):
     """Services an RPC that takes one message and produces a stream of messages.
@@ -116,11 +112,9 @@ class UnaryStreamTestMethodImplementation(test_interfaces.Method):
     raise NotImplementedError()
 
 
-class UnaryStreamTestMessages(object):
+class UnaryStreamTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for unary-request-stream-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def request(self):
     """Affords a request message.
@@ -150,11 +144,9 @@ class UnaryStreamTestMessages(object):
     raise NotImplementedError()
 
 
-class StreamUnaryTestMethodImplementation(test_interfaces.Method):
+class StreamUnaryTestMethodImplementation(six.with_metaclass(abc.ABCMeta, test_interfaces.Method)):
   """A controllable implementation of a stream-unary method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, response_callback, context, control):
     """Services an RPC that takes a stream of messages and produces one message.
@@ -180,11 +172,9 @@ class StreamUnaryTestMethodImplementation(test_interfaces.Method):
     raise NotImplementedError()
 
 
-class StreamUnaryTestMessages(object):
+class StreamUnaryTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for stream-request-unary-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def requests(self):
     """Affords a sequence of request messages.
@@ -214,11 +204,9 @@ class StreamUnaryTestMessages(object):
     raise NotImplementedError()
 
 
-class StreamStreamTestMethodImplementation(test_interfaces.Method):
+class StreamStreamTestMethodImplementation(six.with_metaclass(abc.ABCMeta, test_interfaces.Method)):
   """A controllable implementation of a stream-stream method."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def service(self, response_consumer, context, control):
     """Services an RPC that accepts and produces streams of messages.
@@ -244,11 +232,9 @@ class StreamStreamTestMethodImplementation(test_interfaces.Method):
     raise NotImplementedError()
 
 
-class StreamStreamTestMessages(object):
+class StreamStreamTestMessages(six.with_metaclass(abc.ABCMeta)):
   """A type for stream-request-stream-response message pairings."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def requests(self):
     """Affords a sequence of request messages.
@@ -278,11 +264,9 @@ class StreamStreamTestMessages(object):
     raise NotImplementedError()
 
 
-class TestService(object):
+class TestService(six.with_metaclass(abc.ABCMeta)):
   """A specification of implemented methods to use in tests."""
 
-  __metaclass__ = abc.ABCMeta
-
   @abc.abstractmethod
   def unary_unary_scenarios(self):
     """Affords unary-request-unary-response test methods and their messages.
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py
index 462829b660..06b9d77e52 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -34,14 +34,12 @@ import unittest  # pylint: disable=unused-import
 
 # test_interfaces is referenced from specification in this module.
 from tests.unit.framework.interfaces.face import _blocking_invocation_inline_service
-from tests.unit.framework.interfaces.face import _event_invocation_synchronous_event_service
 from tests.unit.framework.interfaces.face import _future_invocation_asynchronous_event_service
 from tests.unit.framework.interfaces.face import _invocation
 from tests.unit.framework.interfaces.face import test_interfaces  # pylint: disable=unused-import
 
 _TEST_CASE_SUPERCLASSES = (
     _blocking_invocation_inline_service.TestCase,
-    _event_invocation_synchronous_event_service.TestCase,
     _future_invocation_asynchronous_event_service.TestCase,
 )
 
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/test_interfaces.py b/src/python/grpcio/tests/unit/framework/interfaces/face/test_interfaces.py
index b2b5c10fa6..a5e28b7942 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/test_interfaces.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/test_interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,13 +31,14 @@
 
 import abc
 
+import six
+
 from grpc.framework.common import cardinality  # pylint: disable=unused-import
 from grpc.framework.interfaces.face import face  # pylint: disable=unused-import
 
 
-class Method(object):
+class Method(six.with_metaclass(abc.ABCMeta)):
   """Specifies a method to be used in tests."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def group(self):
@@ -126,9 +127,8 @@ class Method(object):
     raise NotImplementedError()
 
 
-class Implementation(object):
+class Implementation(six.with_metaclass(abc.ABCMeta)):
   """Specifies an implementation of the Face layer."""
-  __metaclass__ = abc.ABCMeta
 
   @abc.abstractmethod
   def instantiate(
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/links/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/links/test_cases.py
index dace6c23f3..2283e79f0a 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/links/test_cases.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/links/test_cases.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,8 @@
 import abc
 import unittest  # pylint: disable=unused-import
 
+import six
+
 from grpc.framework.interfaces.links import links
 from tests.unit.framework.common import test_constants
 from tests.unit.framework.interfaces.links import test_utilities
@@ -58,13 +60,12 @@ _TRANSMISSION_GROUP = 'test.Group'
 _TRANSMISSION_METHOD = 'TestMethod'
 
 
-class TransmissionTest(object):
+class TransmissionTest(six.with_metaclass(abc.ABCMeta)):
   """Tests ticket transmission between two connected links.
 
   This class must be mixed into a unittest.TestCase that implements the abstract
   methods it provides.
   """
-  __metaclass__ = abc.ABCMeta
 
   # This is a unittest.TestCase mix-in.
   # pylint: disable=invalid-name
diff --git a/src/python/grpcio/tests/unit/test_common.py b/src/python/grpcio/tests/unit/test_common.py
index 29431bfb9d..824f1cbd16 100644
--- a/src/python/grpcio/tests/unit/test_common.py
+++ b/src/python/grpcio/tests/unit/test_common.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,8 @@
 
 import collections
 
+import six
+
 INVOCATION_INITIAL_METADATA = ((b'0', b'abc'), (b'1', b'def'), (b'2', b'ghi'),)
 SERVICE_INITIAL_METADATA = ((b'3', b'jkl'), (b'4', b'mno'), (b'5', b'pqr'),)
 SERVICE_TERMINAL_METADATA = ((b'6', b'stu'), (b'7', b'vwx'), (b'8', b'yza'),)
@@ -65,7 +67,7 @@ def metadata_transmitted(original_metadata, transmitted_metadata):
     key, value = tuple(key_value_pair)
     transmitted[key].append(value)
 
-  for key, values in original.iteritems():
+  for key, values in six.iteritems(original):
     transmitted_values = transmitted[key]
     transmitted_iterator = iter(transmitted_values)
     try:
diff --git a/src/ruby/.rubocop.yml b/src/ruby/.rubocop.yml
index ff5cf8db83..d13ce42655 100644
--- a/src/ruby/.rubocop.yml
+++ b/src/ruby/.rubocop.yml
@@ -15,3 +15,6 @@ Metrics/CyclomaticComplexity:
 
 Metrics/PerceivedComplexity:
   Max: 8
+
+Metrics/ClassLength:
+  Max: 250
diff --git a/src/ruby/ext/grpc/rb_byte_buffer.c b/src/ruby/ext/grpc/rb_byte_buffer.c
index db7cac363a..9b617e13d3 100644
--- a/src/ruby/ext/grpc/rb_byte_buffer.c
+++ b/src/ruby/ext/grpc/rb_byte_buffer.c
@@ -50,21 +50,18 @@ grpc_byte_buffer* grpc_rb_s_to_byte_buffer(char *string, size_t length) {
 }
 
 VALUE grpc_rb_byte_buffer_to_s(grpc_byte_buffer *buffer) {
-  size_t length = 0;
-  char *string = NULL;
-  size_t offset = 0;
+  VALUE rb_string;
   grpc_byte_buffer_reader reader;
   gpr_slice next;
   if (buffer == NULL) {
     return Qnil;
-
   }
-  length = grpc_byte_buffer_length(buffer);
-  string = xmalloc(length + 1);
+  rb_string = rb_str_buf_new(grpc_byte_buffer_length(buffer));
   grpc_byte_buffer_reader_init(&reader, buffer);
   while (grpc_byte_buffer_reader_next(&reader, &next) != 0) {
-    memcpy(string + offset, GPR_SLICE_START_PTR(next), GPR_SLICE_LENGTH(next));
-    offset += GPR_SLICE_LENGTH(next);
+    rb_str_cat(rb_string, (const char *) GPR_SLICE_START_PTR(next),
+               GPR_SLICE_LENGTH(next));
+    gpr_slice_unref(next);
   }
-  return rb_str_new(string, length);
+  return rb_string;
 }
diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c
index af05ddf6e7..b0829efdc7 100644
--- a/src/ruby/ext/grpc/rb_call.c
+++ b/src/ruby/ext/grpc/rb_call.c
@@ -72,6 +72,10 @@ static ID id_cq;
  * the flags used to create metadata from a Hash */
 static ID id_flags;
 
+/* id_credentials is the name of the hidden ivar that preserves the value
+ * of the credentials added to the call */
+static ID id_credentials;
+
 /* id_input_md is the name of the hidden ivar that preserves the hash used to
  * create metadata, so that references to the strings it contains last as long
  * as the call the metadata is added to. */
@@ -299,6 +303,7 @@ static VALUE grpc_rb_call_set_credentials(VALUE self, VALUE credentials) {
              "grpc_call_set_credentials failed with %s (code=%d)",
              grpc_call_error_detail_of(err), err);
   }
+  rb_ivar_set(self, id_credentials, credentials);
   return Qnil;
 }
 
@@ -546,13 +551,26 @@ static void grpc_run_batch_stack_init(run_batch_stack *st,
 /* grpc_run_batch_stack_cleanup ensures the run_batch_stack is properly
  * cleaned up */
 static void grpc_run_batch_stack_cleanup(run_batch_stack *st) {
+  size_t i = 0;
+
   grpc_metadata_array_destroy(&st->send_metadata);
   grpc_metadata_array_destroy(&st->send_trailing_metadata);
   grpc_metadata_array_destroy(&st->recv_metadata);
   grpc_metadata_array_destroy(&st->recv_trailing_metadata);
+
   if (st->recv_status_details != NULL) {
     gpr_free(st->recv_status_details);
   }
+
+  if (st->recv_message != NULL) {
+    grpc_byte_buffer_destroy(st->recv_message);
+  }
+
+  for (i = 0; i < st->op_num; i++) {
+    if (st->ops[i].op == GRPC_OP_SEND_MESSAGE) {
+      grpc_byte_buffer_destroy(st->ops[i].data.send_message);
+    }
+  }
 }
 
 /* grpc_run_batch_stack_fill_ops fills the run_batch_stack ops array from
@@ -638,7 +656,6 @@ static VALUE grpc_run_batch_stack_build_result(run_batch_stack *st) {
         break;
       case GRPC_OP_SEND_MESSAGE:
         rb_struct_aset(result, sym_send_message, Qtrue);
-        grpc_byte_buffer_destroy(st->ops[i].data.send_message);
         break;
       case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
         rb_struct_aset(result, sym_send_close, Qtrue);
@@ -859,6 +876,7 @@ void Init_grpc_call() {
   id_cq = rb_intern("__cq");
   id_flags = rb_intern("__flags");
   id_input_md = rb_intern("__input_md");
+  id_credentials = rb_intern("__credentials");
 
   /* Ids used in constructing the batch result. */
   sym_send_message = ID2SYM(rb_intern("send_message"));
diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c
index 2426f106a9..2b978fcdd0 100644
--- a/src/ruby/ext/grpc/rb_call_credentials.c
+++ b/src/ruby/ext/grpc/rb_call_credentials.c
@@ -41,6 +41,7 @@
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
 #include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 
 #include "rb_call.h"
 #include "rb_event_thread.h"
@@ -50,9 +51,9 @@
  * grpc_call_credentials */
 static VALUE grpc_rb_cCallCredentials = Qnil;
 
-/* grpc_rb_call_credentials wraps a grpc_call_credentials. It provides a peer
- * ruby object, 'mark' to minimize copying when a credential is created from
- * ruby. */
+/* grpc_rb_call_credentials wraps a grpc_call_credentials. It provides a mark
+ * object that is used to hold references to any objects used to create the
+ * credentials. */
 typedef struct grpc_rb_call_credentials {
   /* Holder of ruby objects involved in contructing the credentials */
   VALUE mark;
@@ -81,14 +82,23 @@ static VALUE grpc_rb_call_credentials_callback(VALUE callback_args) {
 static VALUE grpc_rb_call_credentials_callback_rescue(VALUE args,
                                                       VALUE exception_object) {
   VALUE result = rb_hash_new();
+  VALUE backtrace = rb_funcall(
+      rb_funcall(exception_object, rb_intern("backtrace"), 0),
+      rb_intern("join"),
+      1, rb_str_new2("\n\tfrom "));
+  VALUE exception_info = rb_funcall(exception_object, rb_intern("to_s"), 0);
+  const char *exception_classname = rb_obj_classname(exception_object);
   (void)args;
+  gpr_log(GPR_INFO, "Call credentials callback failed: %s: %s\n%s",
+          exception_classname, StringValueCStr(exception_info),
+          StringValueCStr(backtrace));
   rb_hash_aset(result, rb_str_new2("metadata"), Qnil);
   /* Currently only gives the exception class name. It should be possible get
      more details */
   rb_hash_aset(result, rb_str_new2("status"),
                INT2NUM(GRPC_STATUS_PERMISSION_DENIED));
   rb_hash_aset(result, rb_str_new2("details"),
-               rb_str_new2(rb_obj_classname(exception_object)));
+               rb_str_new2(exception_classname));
   return result;
 }
 
@@ -146,13 +156,8 @@ static void grpc_rb_call_credentials_free(void *p) {
     return;
   }
   wrapper = (grpc_rb_call_credentials *)p;
-
-  /* Delete the wrapped object if the mark object is Qnil, which indicates that
-   * no other object is the actual owner. */
-  if (wrapper->wrapped != NULL && wrapper->mark == Qnil) {
-    grpc_call_credentials_release(wrapper->wrapped);
-    wrapper->wrapped = NULL;
-  }
+  grpc_call_credentials_release(wrapper->wrapped);
+  wrapper->wrapped = NULL;
 
   xfree(p);
 }
@@ -164,8 +169,6 @@ static void grpc_rb_call_credentials_mark(void *p) {
     return;
   }
   wrapper = (grpc_rb_call_credentials *)p;
-
-  /* If it's not already cleaned up, mark the mark object */
   if (wrapper->mark != Qnil) {
     rb_gc_mark(wrapper->mark);
   }
@@ -194,7 +197,7 @@ static VALUE grpc_rb_call_credentials_alloc(VALUE cls) {
 /* Creates a wrapping object for a given call credentials. This should only be
  * called with grpc_call_credentials objects that are not already associated
  * with any Ruby object */
-VALUE grpc_rb_wrap_call_credentials(grpc_call_credentials *c) {
+VALUE grpc_rb_wrap_call_credentials(grpc_call_credentials *c, VALUE mark) {
   VALUE rb_wrapper;
   grpc_rb_call_credentials *wrapper;
   if (c == NULL) {
@@ -204,6 +207,7 @@ VALUE grpc_rb_wrap_call_credentials(grpc_call_credentials *c) {
   TypedData_Get_Struct(rb_wrapper, grpc_rb_call_credentials,
                        &grpc_rb_call_credentials_data_type, wrapper);
   wrapper->wrapped = c;
+  wrapper->mark = mark;
   return rb_wrapper;
 }
 
@@ -267,6 +271,7 @@ static VALUE grpc_rb_call_credentials_init(VALUE self, VALUE proc) {
     return Qnil;
   }
 
+  wrapper->mark = proc;
   wrapper->wrapped = creds;
   rb_ivar_set(self, id_callback, proc);
 
@@ -277,15 +282,18 @@ static VALUE grpc_rb_call_credentials_compose(int argc, VALUE *argv,
                                               VALUE self) {
   grpc_call_credentials *creds;
   grpc_call_credentials *other;
+  VALUE mark;
   if (argc == 0) {
     return self;
   }
+  mark = rb_ary_new();
   creds = grpc_rb_get_wrapped_call_credentials(self);
   for (int i = 0; i < argc; i++) {
+    rb_ary_push(mark, argv[i]);
     other = grpc_rb_get_wrapped_call_credentials(argv[i]);
     creds = grpc_composite_call_credentials_create(creds, other, NULL);
   }
-  return grpc_rb_wrap_call_credentials(creds);
+  return grpc_rb_wrap_call_credentials(creds, mark);
 }
 
 void Init_grpc_call_credentials() {
diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c
index 0e6badbdaf..e1aaa539db 100644
--- a/src/ruby/ext/grpc/rb_channel.c
+++ b/src/ruby/ext/grpc/rb_channel.c
@@ -70,11 +70,10 @@ static VALUE grpc_rb_cChannel = Qnil;
 /* Used during the conversion of a hash to channel args during channel setup */
 static VALUE grpc_rb_cChannelArgs;
 
-/* grpc_rb_channel wraps a grpc_channel.  It provides a peer ruby object,
- * 'mark' to minimize copying when a channel is created from ruby. */
+/* grpc_rb_channel wraps a grpc_channel. */
 typedef struct grpc_rb_channel {
-  /* Holder of ruby objects involved in constructing the channel */
-  VALUE mark;
+  VALUE credentials;
+
   /* The actual channel */
   grpc_channel *wrapped;
 } grpc_rb_channel;
@@ -87,13 +86,8 @@ static void grpc_rb_channel_free(void *p) {
   };
   ch = (grpc_rb_channel *)p;
 
-  /* Deletes the wrapped object if the mark object is Qnil, which indicates
-   * that no other object is the actual owner. */
-  if (ch->wrapped != NULL && ch->mark == Qnil) {
+  if (ch->wrapped != NULL) {
     grpc_channel_destroy(ch->wrapped);
-    rb_warning("channel gc: destroyed the c channel");
-  } else {
-    rb_warning("channel gc: did not destroy the c channel");
   }
 
   xfree(p);
@@ -106,8 +100,8 @@ static void grpc_rb_channel_mark(void *p) {
     return;
   }
   channel = (grpc_rb_channel *)p;
-  if (channel->mark != Qnil) {
-    rb_gc_mark(channel->mark);
+  if (channel->credentials != Qnil) {
+    rb_gc_mark(channel->credentials);
   }
 }
 
@@ -125,7 +119,7 @@ static rb_data_type_t grpc_channel_data_type = {
 static VALUE grpc_rb_channel_alloc(VALUE cls) {
   grpc_rb_channel *wrapper = ALLOC(grpc_rb_channel);
   wrapper->wrapped = NULL;
-  wrapper->mark = Qnil;
+  wrapper->credentials = Qnil;
   return TypedData_Wrap_Struct(cls, &grpc_channel_data_type, wrapper);
 }
 
@@ -162,6 +156,7 @@ static VALUE grpc_rb_channel_init(int argc, VALUE *argv, VALUE self) {
     }
     ch = grpc_insecure_channel_create(target_chars, &args, NULL);
   } else {
+    wrapper->credentials = credentials;
     creds = grpc_rb_get_wrapped_channel_credentials(credentials);
     ch = grpc_secure_channel_create(creds, target_chars, &args, NULL);
   }
@@ -330,7 +325,6 @@ static VALUE grpc_rb_channel_destroy(VALUE self) {
   if (ch != NULL) {
     grpc_channel_destroy(ch);
     wrapper->wrapped = NULL;
-    wrapper->mark = Qnil;
   }
 
   return Qnil;
diff --git a/src/ruby/ext/grpc/rb_channel_credentials.c b/src/ruby/ext/grpc/rb_channel_credentials.c
index 8c6fc3b7eb..f649084311 100644
--- a/src/ruby/ext/grpc/rb_channel_credentials.c
+++ b/src/ruby/ext/grpc/rb_channel_credentials.c
@@ -49,8 +49,8 @@
 static VALUE grpc_rb_cChannelCredentials = Qnil;
 
 /* grpc_rb_channel_credentials wraps a grpc_channel_credentials.  It provides a
- * peer ruby object, 'mark' to minimize copying when a credential is
- * created from ruby. */
+ * mark object that is used to hold references to any objects used to create
+ * the credentials. */
 typedef struct grpc_rb_channel_credentials {
   /* Holder of ruby objects involved in constructing the credentials */
   VALUE mark;
@@ -66,13 +66,8 @@ static void grpc_rb_channel_credentials_free(void *p) {
     return;
   };
   wrapper = (grpc_rb_channel_credentials *)p;
-
-  /* Delete the wrapped object if the mark object is Qnil, which indicates that
-   * no other object is the actual owner. */
-  if (wrapper->wrapped != NULL && wrapper->mark == Qnil) {
-    grpc_channel_credentials_release(wrapper->wrapped);
-    wrapper->wrapped = NULL;
-  }
+  grpc_channel_credentials_release(wrapper->wrapped);
+  wrapper->wrapped = NULL;
 
   xfree(p);
 }
@@ -85,7 +80,6 @@ static void grpc_rb_channel_credentials_mark(void *p) {
   }
   wrapper = (grpc_rb_channel_credentials *)p;
 
-  /* If it's not already cleaned up, mark the mark object */
   if (wrapper->mark != Qnil) {
     rb_gc_mark(wrapper->mark);
   }
@@ -114,7 +108,7 @@ static VALUE grpc_rb_channel_credentials_alloc(VALUE cls) {
 /* Creates a wrapping object for a given channel credentials. This should only
  * be called with grpc_channel_credentials objects that are not already
  * associated with any Ruby object. */
-VALUE grpc_rb_wrap_channel_credentials(grpc_channel_credentials *c) {
+VALUE grpc_rb_wrap_channel_credentials(grpc_channel_credentials *c, VALUE mark) {
   VALUE rb_wrapper;
   grpc_rb_channel_credentials *wrapper;
   if (c == NULL) {
@@ -124,6 +118,7 @@ VALUE grpc_rb_wrap_channel_credentials(grpc_channel_credentials *c) {
   TypedData_Get_Struct(rb_wrapper, grpc_rb_channel_credentials,
                        &grpc_rb_channel_credentials_data_type, wrapper);
   wrapper->wrapped = c;
+  wrapper->mark = mark;
   return rb_wrapper;
 }
 
@@ -222,11 +217,15 @@ static VALUE grpc_rb_channel_credentials_compose(int argc, VALUE *argv,
                                                  VALUE self) {
   grpc_channel_credentials *creds;
   grpc_call_credentials *other;
+  VALUE mark;
   if (argc == 0) {
     return self;
   }
+  mark = rb_ary_new();
+  rb_ary_push(mark, self);
   creds = grpc_rb_get_wrapped_channel_credentials(self);
   for (int i = 0; i < argc; i++) {
+    rb_ary_push(mark, argv[i]);
     other = grpc_rb_get_wrapped_call_credentials(argv[i]);
     creds = grpc_composite_channel_credentials_create(creds, other, NULL);
     if (creds == NULL) {
@@ -234,7 +233,7 @@ static VALUE grpc_rb_channel_credentials_compose(int argc, VALUE *argv,
                "Failed to compose channel and call credentials");
     }
   }
-  return grpc_rb_wrap_channel_credentials(creds);
+  return grpc_rb_wrap_channel_credentials(creds, mark);
 }
 
 void Init_grpc_channel_credentials() {
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
index b972f60fc3..22ea84c750 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -166,7 +166,7 @@ extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_im
 typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name);
 extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import;
 #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import
-typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level);
+typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings);
 extern grpc_compression_algorithm_for_level_type grpc_compression_algorithm_for_level_import;
 #define grpc_compression_algorithm_for_level grpc_compression_algorithm_for_level_import
 typedef void(*grpc_compression_options_init_type)(grpc_compression_options *opts);
@@ -283,7 +283,7 @@ extern grpc_call_destroy_type grpc_call_destroy_import;
 typedef grpc_call_error(*grpc_server_request_call_type)(grpc_server *server, grpc_call **call, grpc_call_details *details, grpc_metadata_array *request_metadata, grpc_completion_queue *cq_bound_to_call, grpc_completion_queue *cq_for_notification, void *tag_new);
 extern grpc_server_request_call_type grpc_server_request_call_import;
 #define grpc_server_request_call grpc_server_request_call_import
-typedef void *(*grpc_server_register_method_type)(grpc_server *server, const char *method, const char *host);
+typedef void *(*grpc_server_register_method_type)(grpc_server *server, const char *method, const char *host, uint32_t flags);
 extern grpc_server_register_method_type grpc_server_register_method_import;
 #define grpc_server_register_method grpc_server_register_method_import
 typedef grpc_call_error(*grpc_server_request_registered_call_type)(grpc_server *server, void *registered_method, grpc_call **call, gpr_timespec *deadline, grpc_metadata_array *request_metadata, grpc_byte_buffer **optional_payload, grpc_completion_queue *cq_bound_to_call, grpc_completion_queue *cq_for_notification, void *tag_new);
diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb
index ef2997c991..b30d19dd2b 100644
--- a/src/ruby/lib/grpc/generic/rpc_server.rb
+++ b/src/ruby/lib/grpc/generic/rpc_server.rb
@@ -107,7 +107,9 @@ module GRPC
 
     # Starts running the jobs in the thread pool.
     def start
-      fail 'already stopped' if @stopped
+      @stop_mutex.synchronize do
+        fail 'already stopped' if @stopped
+      end
       until @workers.size == @size.to_i
         next_thread = Thread.new do
           catch(:exit) do  # allows { throw :exit } to kill a thread
@@ -264,10 +266,10 @@ module GRPC
       @pool = Pool.new(@pool_size)
       @run_cond = ConditionVariable.new
       @run_mutex = Mutex.new
-      @running = false
+      # running_state can take 4 values: :not_started, :running, :stopping, and
+      # :stopped. State transitions can only proceed in that order.
+      @running_state = :not_started
       @server = RpcServer.setup_srv(server_override, @cq, **kw)
-      @stopped = false
-      @stop_mutex = Mutex.new
     end
 
     # stops a running server
@@ -275,27 +277,42 @@ module GRPC
     # the call has no impact if the server is already stopped, otherwise
     # server's current call loop is it's last.
     def stop
-      return unless @running
-      @stop_mutex.synchronize do
-        @stopped = true
+      @run_mutex.synchronize do
+        fail 'Cannot stop before starting' if @running_state == :not_started
+        return if @running_state != :running
+        transition_running_state(:stopping)
       end
       deadline = from_relative_time(@poll_period)
-      return if @server.close(@cq, deadline)
-      deadline = from_relative_time(@poll_period)
       @server.close(@cq, deadline)
       @pool.stop
     end
 
-    # determines if the server has been stopped
-    def stopped?
-      @stop_mutex.synchronize do
-        return @stopped
+    def running_state
+      @run_mutex.synchronize do
+        return @running_state
+      end
+    end
+
+    # Can only be called while holding @run_mutex
+    def transition_running_state(target_state)
+      state_transitions = {
+        not_started: :running,
+        running: :stopping,
+        stopping: :stopped
+      }
+      if state_transitions[@running_state] == target_state
+        @running_state = target_state
+      else
+        fail "Bad server state transition: #{@running_state}->#{target_state}"
       end
     end
 
-    # determines if the server is currently running
     def running?
-      @running
+      running_state == :running
+    end
+
+    def stopped?
+      running_state == :stopped
     end
 
     # Is called from other threads to wait for #run to start up the server.
@@ -304,13 +321,11 @@ module GRPC
     #
     # @param timeout [Numeric] number of seconds to wait
     # @result [true, false] true if the server is running, false otherwise
-    def wait_till_running(timeout = 0.1)
-      end_time, sleep_period = Time.now + timeout, (1.0 * timeout) / 100
-      while Time.now < end_time
-        @run_mutex.synchronize { @run_cond.wait(@run_mutex) } unless running?
-        sleep(sleep_period)
+    def wait_till_running(timeout = nil)
+      @run_mutex.synchronize do
+        @run_cond.wait(@run_mutex, timeout) if @running_state == :not_started
+        return @running_state == :running
       end
-      running?
     end
 
     # Runs the server in its own thread, then waits for signal INT or TERM on
@@ -360,11 +375,14 @@ module GRPC
     # @param service [Object|Class] a service class or object as described
     #        above
     def handle(service)
-      fail 'cannot add services if the server is running' if running?
-      fail 'cannot add services if the server is stopped' if stopped?
-      cls = service.is_a?(Class) ? service : service.class
-      assert_valid_service_class(cls)
-      add_rpc_descs_for(service)
+      @run_mutex.synchronize do
+        unless @running_state == :not_started
+          fail 'cannot add services if the server has been started'
+        end
+        cls = service.is_a?(Class) ? service : service.class
+        assert_valid_service_class(cls)
+        add_rpc_descs_for(service)
+      end
     end
 
     # runs the server
@@ -375,16 +393,13 @@ module GRPC
     # - #running? returns true after this is called, until #stop cause the
     #   the server to stop.
     def run
-      if rpc_descs.size.zero?
-        GRPC.logger.warn('did not run as no services were present')
-        return
-      end
       @run_mutex.synchronize do
-        @running = true
-        @run_cond.signal
+        fail 'cannot run without registering services' if rpc_descs.size.zero?
+        @pool.start
+        @server.start
+        transition_running_state(:running)
+        @run_cond.broadcast
       end
-      @pool.start
-      @server.start
       loop_handle_server_calls
     end
 
@@ -413,9 +428,9 @@ module GRPC
 
     # handles calls to the server
     def loop_handle_server_calls
-      fail 'not running' unless @running
+      fail 'not started' if running_state == :not_started
       loop_tag = Object.new
-      until stopped?
+      while running_state == :running
         begin
           an_rpc = @server.request_call(@cq, loop_tag, INFINITE_FUTURE)
           break if (!an_rpc.nil?) && an_rpc.call.nil?
@@ -430,11 +445,14 @@ module GRPC
         rescue Core::CallError, RuntimeError => e
           # these might happen for various reasonse.  The correct behaviour of
           # the server is to log them and continue, if it's not shutting down.
-          GRPC.logger.warn("server call failed: #{e}") unless stopped?
+          if running_state == :running
+            GRPC.logger.warn("server call failed: #{e}")
+          end
           next
         end
       end
-      @running = false
+      # @running_state should be :stopping here
+      @run_mutex.synchronize { transition_running_state(:stopped) }
       GRPC.logger.info("stopped: #{self}")
     end
 
@@ -484,9 +502,10 @@ module GRPC
       cls.assert_rpc_descs_have_methods
     end
 
+    # This should be called while holding @run_mutex
     def add_rpc_descs_for(service)
       cls = service.is_a?(Class) ? service : service.class
-      specs, handlers = rpc_descs, rpc_handlers
+      specs, handlers = (@rpc_descs ||= {}), (@rpc_handlers ||= {})
       cls.rpc_descs.each_pair do |name, spec|
         route = "/#{cls.service_name}/#{name}".to_sym
         fail "already registered: rpc #{route} from #{spec}" if specs.key? route
diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb
index be6331d68b..dfaec6d6ed 100644
--- a/src/ruby/spec/generic/rpc_server_spec.rb
+++ b/src/ruby/spec/generic/rpc_server_spec.rb
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -220,19 +220,10 @@ describe GRPC::RpcServer do
       @srv = RpcServer.new(**opts)
     end
 
-    after(:each) do
-      @srv.stop
-    end
-
     it 'starts out false' do
       expect(@srv.stopped?).to be(false)
     end
 
-    it 'stays false after a #stop is called before #run' do
-      @srv.stop
-      expect(@srv.stopped?).to be(false)
-    end
-
     it 'stays false after the server starts running', server: true do
       @srv.handle(EchoService)
       t = Thread.new { @srv.run }
@@ -247,8 +238,8 @@ describe GRPC::RpcServer do
       t = Thread.new { @srv.run }
       @srv.wait_till_running
       @srv.stop
-      expect(@srv.stopped?).to be(true)
       t.join
+      expect(@srv.stopped?).to be(true)
     end
   end
 
@@ -266,9 +257,7 @@ describe GRPC::RpcServer do
         server_override: @server
       }
       r = RpcServer.new(**opts)
-      r.run
-      expect(r.running?).to be(false)
-      r.stop
+      expect { r.run }.to raise_error(RuntimeError)
     end
 
     it 'is true after run is called with a registered service' do
@@ -293,10 +282,6 @@ describe GRPC::RpcServer do
       @srv = RpcServer.new(**@opts)
     end
 
-    after(:each) do
-      @srv.stop
-    end
-
     it 'raises if #run has already been called' do
       @srv.handle(EchoService)
       t = Thread.new { @srv.run }
@@ -528,10 +513,6 @@ describe GRPC::RpcServer do
         @srv = RpcServer.new(**server_opts)
       end
 
-      after(:each) do
-        @srv.stop
-      end
-
       it 'should be added to BadStatus when requests fail', server: true do
         service = FailingService.new
         @srv.handle(service)