Merge master into merge-0.14-into-master

Conflicts:
- gRPC.podspec
  - Only had non-trivial changes in the core file list, which will need to be
    regenerated (in gRPC-Core.podspec).
- src/objective-c/BoringSSL.podspec
  - Had trivial conflicts in the version.
- src/objective-c/examples/RemoteTestClient/RemoteTest.podspec
  - Trivial conflicts in quoting.
- src/objective-c/examples/Sample/Sample.xcodeproj/project.pbxproj and
src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/project.pbxproj
  - The master version is used, pending testing. The 0.14 version had emoji and
    some unneeded entries.
- src/objective-c/tests/Podfile
  - Added CronetFramework pod, and warning silencing from master.
- templates/gRPC.podspec.template
  - Deleted.
- third_party/protobuf
  - Using master commit, but need to verify if it works for frameworks.

624 files changed, 36286 insertions, 11706 deletions

diff --git a/src/compiler/config.h b/src/compiler/config.h
index d8b95818db..1cbd842f0a 100644
--- a/src/compiler/config.h
+++ b/src/compiler/config.h
@@ -34,19 +34,7 @@
 #ifndef SRC_COMPILER_CONFIG_H
 #define SRC_COMPILER_CONFIG_H
 
-#include <grpc++/support/config.h>
-#include <grpc++/support/config_protobuf.h>
-
-#ifndef GRPC_CUSTOM_DESCRIPTOR
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/descriptor.pb.h>
-#define GRPC_CUSTOM_DESCRIPTOR ::google::protobuf::Descriptor
-#define GRPC_CUSTOM_FILEDESCRIPTOR ::google::protobuf::FileDescriptor
-#define GRPC_CUSTOM_FILEDESCRIPTORPROTO ::google::protobuf::FileDescriptorProto
-#define GRPC_CUSTOM_METHODDESCRIPTOR ::google::protobuf::MethodDescriptor
-#define GRPC_CUSTOM_SERVICEDESCRIPTOR ::google::protobuf::ServiceDescriptor
-#define GRPC_CUSTOM_SOURCELOCATION ::google::protobuf::SourceLocation
-#endif
+#include <grpc++/impl/codegen/config_protobuf.h>
 
 #ifndef GRPC_CUSTOM_CODEGENERATOR
 #include <google/protobuf/compiler/code_generator.h>
@@ -75,14 +63,17 @@
 #define GRPC_CUSTOM_PARSEGENERATORPARAMETER ::google::protobuf::compiler::ParseGeneratorParameter
 #endif
 
+#ifndef GRPC_CUSTOM_STRING
+#include <string>
+#define GRPC_CUSTOM_STRING std::string
+#endif
+
 namespace grpc {
+
+typedef GRPC_CUSTOM_STRING string;
+
 namespace protobuf {
-typedef GRPC_CUSTOM_DESCRIPTOR Descriptor;
-typedef GRPC_CUSTOM_FILEDESCRIPTOR FileDescriptor;
-typedef GRPC_CUSTOM_FILEDESCRIPTORPROTO FileDescriptorProto;
-typedef GRPC_CUSTOM_METHODDESCRIPTOR MethodDescriptor;
-typedef GRPC_CUSTOM_SERVICEDESCRIPTOR ServiceDescriptor;
-typedef GRPC_CUSTOM_SOURCELOCATION SourceLocation;
+
 namespace compiler {
 typedef GRPC_CUSTOM_CODEGENERATOR CodeGenerator;
 typedef GRPC_CUSTOM_GENERATORCONTEXT GeneratorContext;
diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc
index e2f127094a..2288ba4163 100644
--- a/src/compiler/cpp_generator.cc
+++ b/src/compiler/cpp_generator.cc
@@ -73,9 +73,10 @@ void PrintIncludes(Printer *printer, const std::vector<grpc::string>& headers, c
   vars["l"] = params.use_system_headers ? '<' : '"';
   vars["r"] = params.use_system_headers ? '>' : '"';
 
-  if (!params.grpc_search_path.empty()) {
-    vars["l"] += params.grpc_search_path;
-    if (params.grpc_search_path.back() != '/') {
+  auto& s = params.grpc_search_path;
+  if (!s.empty()) {
+    vars["l"] += s;
+    if (s[s.size()-1] != '/') {
       vars["l"] += '/';
     }
   }
diff --git a/src/compiler/cpp_generator_helpers.h b/src/compiler/cpp_generator_helpers.h
index be68cbe695..87e278f1b9 100644
--- a/src/compiler/cpp_generator_helpers.h
+++ b/src/compiler/cpp_generator_helpers.h
@@ -65,6 +65,13 @@ inline grpc::string ClassName(const grpc::protobuf::Descriptor *descriptor,
   }
 }
 
+// Get leading or trailing comments in a string. Comment lines start with "// ".
+// Leading detached comments are put in in front of leading comments.
+template <typename DescriptorType>
+inline grpc::string GetCppComments(const DescriptorType *desc, bool leading) {
+  return grpc_generator::GetPrefixedComments(desc, leading, "//");
+}
+
 }  // namespace grpc_cpp_generator
 
 #endif  // GRPC_INTERNAL_COMPILER_CPP_GENERATOR_HELPERS_H
diff --git a/src/compiler/cpp_plugin.cc b/src/compiler/cpp_plugin.cc
index 0ec183e474..fc0296cd28 100644
--- a/src/compiler/cpp_plugin.cc
+++ b/src/compiler/cpp_plugin.cc
@@ -43,7 +43,7 @@
 #include "src/compiler/cpp_generator_helpers.h"
 #include "src/compiler/generator_helpers.h"
 
-using grpc_generator::GetCppComments;
+using grpc_cpp_generator::GetCppComments;
 
 class ProtoBufMethod : public grpc_cpp_generator::Method {
  public:
diff --git a/src/compiler/csharp_generator.cc b/src/compiler/csharp_generator.cc
index 29c359c539..f5a0876cf9 100644
--- a/src/compiler/csharp_generator.cc
+++ b/src/compiler/csharp_generator.cc
@@ -52,7 +52,6 @@ using grpc::protobuf::MethodDescriptor;
 using grpc::protobuf::io::Printer;
 using grpc::protobuf::io::StringOutputStream;
 using grpc_generator::MethodType;
-using grpc_generator::GetCppComments;
 using grpc_generator::GetMethodType;
 using grpc_generator::METHODTYPE_NO_STREAMING;
 using grpc_generator::METHODTYPE_CLIENT_STREAMING;
@@ -120,18 +119,10 @@ std::string GetServiceClassName(const ServiceDescriptor* service) {
   return service->name();
 }
 
-std::string GetClientInterfaceName(const ServiceDescriptor* service) {
-  return "I" + service->name() + "Client";
-}
-
 std::string GetClientClassName(const ServiceDescriptor* service) {
   return service->name() + "Client";
 }
 
-std::string GetServerInterfaceName(const ServiceDescriptor* service) {
-  return "I" + service->name();
-}
-
 std::string GetServerClassName(const ServiceDescriptor* service) {
   return service->name() + "Base";
 }
@@ -303,86 +294,6 @@ void GenerateServiceDescriptorProperty(Printer* out, const ServiceDescriptor *se
   out->Print("\n");
 }
 
-void GenerateClientInterface(Printer* out, const ServiceDescriptor *service) {
-  out->Print("/// <summary>Client for $servicename$</summary>\n",
-             "servicename", GetServiceClassName(service));
-  out->Print("[System.Obsolete(\"Client side interfaced will be removed "
-             "in the next release. Use client class directly.\")]\n");
-  out->Print("public interface $name$\n", "name",
-             GetClientInterfaceName(service));
-  out->Print("{\n");
-  out->Indent();
-  for (int i = 0; i < service->method_count(); i++) {
-    const MethodDescriptor *method = service->method(i);
-    MethodType method_type = GetMethodType(method);
-
-    if (method_type == METHODTYPE_NO_STREAMING) {
-      // unary calls have an extra synchronous stub method
-      GenerateDocCommentBody(out, method);
-      out->Print(
-          "$response$ $methodname$($request$ request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));\n",
-          "methodname", method->name(), "request",
-          GetClassName(method->input_type()), "response",
-          GetClassName(method->output_type()));
-
-      // overload taking CallOptions as a param
-      GenerateDocCommentBody(out, method);
-      out->Print(
-          "$response$ $methodname$($request$ request, CallOptions options);\n",
-          "methodname", method->name(), "request",
-          GetClassName(method->input_type()), "response",
-          GetClassName(method->output_type()));
-    }
-
-    std::string method_name = method->name();
-    if (method_type == METHODTYPE_NO_STREAMING) {
-      method_name += "Async";  // prevent name clash with synchronous method.
-    }
-    GenerateDocCommentBody(out, method);
-    out->Print(
-        "$returntype$ $methodname$($request_maybe$Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));\n",
-        "methodname", method_name, "request_maybe",
-        GetMethodRequestParamMaybe(method), "returntype",
-        GetMethodReturnTypeClient(method));
-
-    // overload taking CallOptions as a param
-    GenerateDocCommentBody(out, method);
-    out->Print(
-        "$returntype$ $methodname$($request_maybe$CallOptions options);\n",
-        "methodname", method_name, "request_maybe",
-        GetMethodRequestParamMaybe(method), "returntype",
-        GetMethodReturnTypeClient(method));
-  }
-  out->Outdent();
-  out->Print("}\n");
-  out->Print("\n");
-}
-
-void GenerateServerInterface(Printer* out, const ServiceDescriptor *service) {
-  out->Print("/// <summary>Interface of server-side implementations of $servicename$</summary>\n",
-             "servicename", GetServiceClassName(service));
-  out->Print("[System.Obsolete(\"Service implementations should inherit"
-      " from the generated abstract base class instead.\")]\n");
-  out->Print("public interface $name$\n", "name",
-             GetServerInterfaceName(service));
-  out->Print("{\n");
-  out->Indent();
-  for (int i = 0; i < service->method_count(); i++) {
-    const MethodDescriptor *method = service->method(i);
-    GenerateDocCommentBody(out, method);
-    out->Print(
-        "$returntype$ $methodname$($request$$response_stream_maybe$, "
-        "ServerCallContext context);\n",
-        "methodname", method->name(), "returntype",
-        GetMethodReturnTypeServer(method), "request",
-        GetMethodRequestParamServer(method), "response_stream_maybe",
-        GetMethodResponseStreamMaybe(method));
-  }
-  out->Outdent();
-  out->Print("}\n");
-  out->Print("\n");
-}
-
 void GenerateServerClass(Printer* out, const ServiceDescriptor *service) {
   out->Print("/// <summary>Base class for server-side implementations of $servicename$</summary>\n",
              "servicename", GetServiceClassName(service));
@@ -415,32 +326,35 @@ void GenerateServerClass(Printer* out, const ServiceDescriptor *service) {
 void GenerateClientStub(Printer* out, const ServiceDescriptor *service) {
   out->Print("/// <summary>Client for $servicename$</summary>\n",
              "servicename", GetServiceClassName(service));
-  out->Print("#pragma warning disable 0618\n");
   out->Print(
-      "public class $name$ : ClientBase<$name$>, $interface$\n",
-      "name", GetClientClassName(service),
-      "interface", GetClientInterfaceName(service));
-  out->Print("#pragma warning restore 0618\n");
+      "public class $name$ : ClientBase<$name$>\n",
+      "name", GetClientClassName(service));
   out->Print("{\n");
   out->Indent();
 
   // constructors
+  out->Print("/// <summary>Creates a new client for $servicename$</summary>\n"
+             "/// <param name=\"channel\">The channel to use to make remote calls.</param>\n",
+             "servicename", GetServiceClassName(service));
   out->Print("public $name$(Channel channel) : base(channel)\n",
              "name", GetClientClassName(service));
   out->Print("{\n");
   out->Print("}\n");
+  out->Print("/// <summary>Creates a new client for $servicename$ that uses a custom <c>CallInvoker</c>.</summary>\n"
+             "/// <param name=\"callInvoker\">The callInvoker to use to make remote calls.</param>\n",
+             "servicename", GetServiceClassName(service));
   out->Print("public $name$(CallInvoker callInvoker) : base(callInvoker)\n",
              "name", GetClientClassName(service));
   out->Print("{\n");
   out->Print("}\n");
-  out->Print("///<summary>Protected parameterless constructor to allow creation"
+  out->Print("/// <summary>Protected parameterless constructor to allow creation"
              " of test doubles.</summary>\n");
   out->Print("protected $name$() : base()\n",
              "name", GetClientClassName(service));
   out->Print("{\n");
   out->Print("}\n");
-  out->Print("///<summary>Protected constructor to allow creation of configured"
-             " clients.</summary>\n");
+  out->Print("/// <summary>Protected constructor to allow creation of configured clients.</summary>\n"
+             "/// <param name=\"configuration\">The client configuration.</param>\n");
   out->Print("protected $name$(ClientBaseConfiguration configuration)"
              " : base(configuration)\n",
              "name", GetClientClassName(service));
@@ -547,22 +461,16 @@ void GenerateClientStub(Printer* out, const ServiceDescriptor *service) {
   out->Print("\n");
 }
 
-void GenerateBindServiceMethod(Printer* out, const ServiceDescriptor *service,
-                               bool use_server_class) {
+void GenerateBindServiceMethod(Printer* out, const ServiceDescriptor *service) {
   out->Print(
       "/// <summary>Creates service definition that can be registered with a server</summary>\n");
-  out->Print("#pragma warning disable 0618\n");
   out->Print(
-      "public static ServerServiceDefinition BindService($interface$ serviceImpl)\n",
-      "interface", use_server_class ? GetServerClassName(service) :
-          GetServerInterfaceName(service));
-  out->Print("#pragma warning restore 0618\n");
+      "public static ServerServiceDefinition BindService($implclass$ serviceImpl)\n",
+      "implclass", GetServerClassName(service));
   out->Print("{\n");
   out->Indent();
 
-  out->Print(
-      "return ServerServiceDefinition.CreateBuilder($servicenamefield$)\n",
-      "servicenamefield", GetServiceNameFieldName());
+  out->Print("return ServerServiceDefinition.CreateBuilder()\n");
   out->Indent();
   out->Indent();
   for (int i = 0; i < service->method_count(); i++) {
@@ -583,20 +491,6 @@ void GenerateBindServiceMethod(Printer* out, const ServiceDescriptor *service,
   out->Print("\n");
 }
 
-void GenerateNewStubMethods(Printer* out, const ServiceDescriptor *service) {
-  out->Print("/// <summary>Creates a new client for $servicename$</summary>\n",
-             "servicename", GetServiceClassName(service));
-  out->Print("public static $classname$ NewClient(Channel channel)\n",
-             "classname", GetClientClassName(service));
-  out->Print("{\n");
-  out->Indent();
-  out->Print("return new $classname$(channel);\n", "classname",
-             GetClientClassName(service));
-  out->Outdent();
-  out->Print("}\n");
-  out->Print("\n");
-}
-
 void GenerateService(Printer* out, const ServiceDescriptor *service,
                      bool generate_client, bool generate_server,
                      bool internal_access) {
@@ -617,20 +511,14 @@ void GenerateService(Printer* out, const ServiceDescriptor *service,
   }
   GenerateServiceDescriptorProperty(out, service);
 
-  if (generate_client) {
-    GenerateClientInterface(out, service);
-  }
   if (generate_server) {
-    GenerateServerInterface(out, service);
     GenerateServerClass(out, service);
   }
   if (generate_client) {
     GenerateClientStub(out, service);
-    GenerateNewStubMethods(out, service);
   }
   if (generate_server) {
-    GenerateBindServiceMethod(out, service, false);
-    GenerateBindServiceMethod(out, service, true);
+    GenerateBindServiceMethod(out, service);
   }
 
   out->Outdent();
@@ -659,7 +547,7 @@ grpc::string GetServices(const FileDescriptor *file, bool generate_client,
     out.Print("// source: $filename$\n", "filename", file->name());
 
     // use C++ style as there are no file-level XML comments in .NET
-    grpc::string leading_comments = GetCppComments(file, true);
+    grpc::string leading_comments = GetCsharpComments(file, true);
     if (!leading_comments.empty()) {
       out.Print("// Original file comments:\n");
       out.Print(leading_comments.c_str());
diff --git a/src/compiler/csharp_generator_helpers.h b/src/compiler/csharp_generator_helpers.h
index 5639ea058b..9bdf6fb535 100644
--- a/src/compiler/csharp_generator_helpers.h
+++ b/src/compiler/csharp_generator_helpers.h
@@ -45,6 +45,13 @@ inline bool ServicesFilename(const grpc::protobuf::FileDescriptor *file,
   return true;
 }
 
+// Get leading or trailing comments in a string. Comment lines start with "// ".
+// Leading detached comments are put in in front of leading comments.
+template <typename DescriptorType>
+inline grpc::string GetCsharpComments(const DescriptorType *desc, bool leading) {
+  return grpc_generator::GetPrefixedComments(desc, leading, "//");
+}
+
 }  // namespace grpc_csharp_generator
 
 #endif  // GRPC_INTERNAL_COMPILER_CSHARP_GENERATOR_HELPERS_H
diff --git a/src/compiler/generator_helpers.h b/src/compiler/generator_helpers.h
index bd077cf798..9a88c2bfcc 100644
--- a/src/compiler/generator_helpers.h
+++ b/src/compiler/generator_helpers.h
@@ -253,7 +253,8 @@ inline void GetComment(const grpc::protobuf::FileDescriptor *desc,
 inline grpc::string GenerateCommentsWithPrefix(
     const std::vector<grpc::string> &in, const grpc::string &prefix) {
   std::ostringstream oss;
-  for (const grpc::string &elem : in) {
+  for (auto it = in.begin(); it != in.end(); it++) {
+    const grpc::string& elem = *it;
     if (elem.empty()) {
       oss << prefix << "\n";
     } else if (elem[0] == ' ') {
@@ -265,10 +266,10 @@ inline grpc::string GenerateCommentsWithPrefix(
   return oss.str();
 }
 
-// Get leading or trailing comments in a string. Comment lines start with "// ".
-// Leading detached comments are put in in front of leading comments.
 template <typename DescriptorType>
-inline grpc::string GetCppComments(const DescriptorType *desc, bool leading) {
+inline grpc::string GetPrefixedComments(const DescriptorType *desc,
+                                        bool leading,
+                                        const grpc::string &prefix) {
   std::vector<grpc::string> out;
   if (leading) {
     grpc_generator::GetComment(
@@ -281,7 +282,7 @@ inline grpc::string GetCppComments(const DescriptorType *desc, bool leading) {
     grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_TRAILING,
                                &out);
   }
-  return GenerateCommentsWithPrefix(out, "//");
+  return GenerateCommentsWithPrefix(out, prefix);
 }
 
 }  // namespace grpc_generator
diff --git a/src/compiler/node_generator.cc b/src/compiler/node_generator.cc
index 822622cccf..1fe090d17a 100644
--- a/src/compiler/node_generator.cc
+++ b/src/compiler/node_generator.cc
@@ -74,8 +74,16 @@ grpc::string GetJSMessageFilename(const grpc::string& filename) {
 
 // Given a filename like foo/bar/baz.proto, returns the root directory
 // path ../../
-grpc::string GetRootPath(const grpc::string& filename) {
-  size_t slashes = std::count(filename.begin(), filename.end(), '/');
+grpc::string GetRootPath(const grpc::string& from_filename,
+                         const grpc::string& to_filename) {
+  if (to_filename.find("google/protobuf") == 0) {
+    // Well-known types (.proto files in the google/protobuf directory) are
+    // assumed to come from the 'google-protobuf' npm package.  We may want to
+    // generalize this exception later by letting others put generated code in
+    // their own npm packages.
+    return "google-protobuf/";
+  }
+  size_t slashes = std::count(from_filename.begin(), from_filename.end(), '/');
   if (slashes == 0) {
     return "./";
   }
@@ -90,7 +98,7 @@ grpc::string GetRootPath(const grpc::string& filename) {
 // from_file, assuming that both paths are relative to the same directory
 grpc::string GetRelativePath(const grpc::string& from_file,
                              const grpc::string& to_file) {
-  return GetRootPath(from_file) + to_file;
+  return GetRootPath(from_file, to_file) + to_file;
 }
 
 /* Finds all message types used in all services in the file, and returns them
@@ -181,62 +189,67 @@ void PrintMethod(const MethodDescriptor *method, Printer *out) {
 // Prints out the service descriptor object
 void PrintService(const ServiceDescriptor *service, Printer *out) {
   map<grpc::string, grpc::string> template_vars;
+  out->Print(GetNodeComments(service, true).c_str());
   template_vars["name"] = service->name();
   out->Print(template_vars, "var $name$Service = exports.$name$Service = {\n");
   out->Indent();
   for (int i = 0; i < service->method_count(); i++) {
     grpc::string method_name = grpc_generator::LowercaseFirstLetter(
         service->method(i)->name());
+    out->Print(GetNodeComments(service->method(i), true).c_str());
     out->Print("$method_name$: ",
                "method_name", method_name);
     PrintMethod(service->method(i), out);
     out->Print(",\n");
+    out->Print(GetNodeComments(service->method(i), false).c_str());
   }
   out->Outdent();
   out->Print("};\n\n");
   out->Print(template_vars, "exports.$name$Client = "
              "grpc.makeGenericClientConstructor($name$Service);\n");
+  out->Print(GetNodeComments(service, false).c_str());
 }
 
-}
-
-grpc::string GetImports(const FileDescriptor *file) {
-  grpc::string output;
-  {
-    StringOutputStream output_stream(&output);
-    Printer out(&output_stream, '$');
-
-    if (file->service_count() == 0) {
-      return output;
-    }
-
-    out.Print("// GENERATED CODE -- DO NOT EDIT!\n\n");
+void PrintImports(const FileDescriptor *file, Printer *out) {
+  out->Print("var grpc = require('grpc');\n");
+  if (file->message_type_count() > 0) {
+    grpc::string file_path = GetRelativePath(file->name(),
+                                             GetJSMessageFilename(
+                                                 file->name()));
+    out->Print("var $module_alias$ = require('$file_path$');\n",
+               "module_alias", ModuleAlias(file->name()),
+               "file_path", file_path);
+  }
 
-    out.Print("'use strict';\n");
+  for (int i = 0; i < file->dependency_count(); i++) {
+    grpc::string file_path = GetRelativePath(
+        file->name(), GetJSMessageFilename(file->dependency(i)->name()));
+    out->Print("var $module_alias$ = require('$file_path$');\n",
+               "module_alias", ModuleAlias(file->dependency(i)->name()),
+               "file_path", file_path);
+  }
+  out->Print("\n");
+}
 
-    out.Print("var grpc = require('grpc');\n");
-    if (file->message_type_count() > 0) {
-      grpc::string file_path = GetRelativePath(file->name(),
-                                               GetJSMessageFilename(
-                                                   file->name()));
-      out.Print("var $module_alias$ = require('$file_path$');\n",
-                "module_alias", ModuleAlias(file->name()),
-                "file_path", file_path);
-    }
+void PrintTransformers(const FileDescriptor *file, Printer *out) {
+  map<grpc::string, const Descriptor*> messages = GetAllMessages(file);
+  for (std::map<grpc::string, const Descriptor*>::iterator it =
+           messages.begin();
+       it != messages.end(); it++) {
+    PrintMessageTransformer(it->second, out);
+  }
+  out->Print("\n");
+}
 
-    for (int i = 0; i < file->dependency_count(); i++) {
-      grpc::string file_path = GetRelativePath(
-          file->name(), GetJSMessageFilename(file->dependency(i)->name()));
-      out.Print("var $module_alias$ = require('$file_path$');\n",
-                "module_alias", ModuleAlias(file->dependency(i)->name()),
-                "file_path", file_path);
-    }
-    out.Print("\n");
+void PrintServices(const FileDescriptor *file, Printer *out) {
+  for (int i = 0; i < file->service_count(); i++) {
+    PrintService(file->service(i), out);
   }
-  return output;
 }
 
-grpc::string GetTransformers(const FileDescriptor *file) {
+}
+
+grpc::string GenerateFile(const FileDescriptor *file) {
   grpc::string output;
   {
     StringOutputStream output_stream(&output);
@@ -245,31 +258,23 @@ grpc::string GetTransformers(const FileDescriptor *file) {
     if (file->service_count() == 0) {
       return output;
     }
+    out.Print("// GENERATED CODE -- DO NOT EDIT!\n\n");
 
-    map<grpc::string, const Descriptor*> messages = GetAllMessages(file);
-    for (std::map<grpc::string, const Descriptor*>::iterator it =
-             messages.begin();
-         it != messages.end(); it++) {
-      PrintMessageTransformer(it->second, &out);
+    grpc::string leading_comments = GetNodeComments(file, true);
+    if (!leading_comments.empty()) {
+      out.Print("// Original file comments:\n");
+      out.Print(leading_comments.c_str());
     }
-    out.Print("\n");
-  }
-  return output;
-}
 
-grpc::string GetServices(const FileDescriptor *file) {
-  grpc::string output;
-  {
-    StringOutputStream output_stream(&output);
-    Printer out(&output_stream, '$');
+    out.Print("'use strict';\n");
 
-    if (file->service_count() == 0) {
-      return output;
-    }
+    PrintImports(file, &out);
 
-    for (int i = 0; i < file->service_count(); i++) {
-      PrintService(file->service(i), &out);
-    }
+    PrintTransformers(file, &out);
+
+    PrintServices(file, &out);
+
+    out.Print(GetNodeComments(file, false).c_str());
   }
   return output;
 }
diff --git a/src/compiler/node_generator.h b/src/compiler/node_generator.h
index 249a0d011f..d7765e2d28 100644
--- a/src/compiler/node_generator.h
+++ b/src/compiler/node_generator.h
@@ -38,11 +38,7 @@
 
 namespace grpc_node_generator {
 
-grpc::string GetImports(const grpc::protobuf::FileDescriptor *file);
-
-grpc::string GetTransformers(const grpc::protobuf::FileDescriptor *file);
-
-grpc::string GetServices(const grpc::protobuf::FileDescriptor *file);
+grpc::string GenerateFile(const grpc::protobuf::FileDescriptor *file);
 
 }  // namespace grpc_node_generator
 
diff --git a/src/compiler/node_generator_helpers.h b/src/compiler/node_generator_helpers.h
index f41a2bcf59..5862772841 100644
--- a/src/compiler/node_generator_helpers.h
+++ b/src/compiler/node_generator_helpers.h
@@ -45,6 +45,13 @@ inline grpc::string GetJSServiceFilename(const grpc::string& filename) {
   return grpc_generator::StripProto(filename) + "_grpc_pb.js";
 }
 
+// Get leading or trailing comments in a string. Comment lines start with "// ".
+// Leading detached comments are put in in front of leading comments.
+template <typename DescriptorType>
+inline grpc::string GetNodeComments(const DescriptorType *desc, bool leading) {
+  return grpc_generator::GetPrefixedComments(desc, leading, "//");
+}
+
 }  // namespace grpc_node_generator
 
 #endif  // GRPC_INTERNAL_COMPILER_NODE_GENERATOR_HELPERS_H
diff --git a/src/compiler/node_plugin.cc b/src/compiler/node_plugin.cc
index ac5ced3558..39dfa77b8d 100644
--- a/src/compiler/node_plugin.cc
+++ b/src/compiler/node_plugin.cc
@@ -39,10 +39,8 @@
 #include "src/compiler/node_generator.h"
 #include "src/compiler/node_generator_helpers.h"
 
-using grpc_node_generator::GetImports;
+using grpc_node_generator::GenerateFile;
 using grpc_node_generator::GetJSServiceFilename;
-using grpc_node_generator::GetServices;
-using grpc_node_generator::GetTransformers;
 
 class NodeGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
  public:
@@ -53,9 +51,7 @@ class NodeGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
                 const grpc::string &parameter,
                 grpc::protobuf::compiler::GeneratorContext *context,
                 grpc::string *error) const {
-    grpc::string code = GetImports(file) +
-        GetTransformers(file) +
-        GetServices(file);
+    grpc::string code = GenerateFile(file);
     if (code.size() == 0) {
       return true;
     }
diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc
index 465491e385..4be8cb4187 100644
--- a/src/compiler/objective_c_generator.cc
+++ b/src/compiler/objective_c_generator.cc
@@ -60,9 +60,34 @@ void PrintProtoRpcDeclarationAsPragma(Printer *printer,
                  " returns ($server_stream$$response_type$)\n\n");
 }
 
+template <typename DescriptorType>
+static void PrintAllComments(const DescriptorType* desc, Printer* printer) {
+  std::vector<grpc::string> comments;
+  grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING_DETACHED,
+                             &comments);
+  grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING,
+                             &comments);
+  grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_TRAILING,
+                             &comments);
+  if (comments.empty()) {
+    return;
+  }
+  printer->Print("/**\n");
+  for (auto it = comments.begin(); it != comments.end(); ++it) {
+    printer->Print(" * ");
+    size_t start_pos = it->find_first_not_of(' ');
+    if (start_pos != grpc::string::npos) {
+      printer->Print(it->c_str() + start_pos);
+    }
+    printer->Print("\n");
+  }
+  printer->Print(" */\n");
+}
+
 void PrintMethodSignature(Printer *printer, const MethodDescriptor *method,
                           const map< ::grpc::string, ::grpc::string> &vars) {
-  // TODO(jcanizales): Print method comments.
+  // Print comment
+  PrintAllComments(method, printer);
 
   printer->Print(vars, "- ($return_type$)$method_name$With");
   if (method->client_streaming()) {
@@ -94,7 +119,7 @@ void PrintSimpleSignature(Printer *printer, const MethodDescriptor *method,
 void PrintAdvancedSignature(Printer *printer, const MethodDescriptor *method,
                             map< ::grpc::string, ::grpc::string> vars) {
   vars["method_name"] = "RPCTo" + vars["method_name"];
-  vars["return_type"] = "ProtoRPC *";
+  vars["return_type"] = "GRPCProtoCall *";
   PrintMethodSignature(printer, method, vars);
 }
 
@@ -195,11 +220,13 @@ void PrintMethodImplementations(Printer *printer,
     printer.Print("@end\n\n");
 
     printer.Print(
-        "// Basic service implementation, over gRPC, that only does"
-        " marshalling and parsing.\n");
+        "/**\n"
+        " * Basic service implementation, over gRPC, that only does\n"
+        " * marshalling and parsing.\n"
+        " */\n");
     printer.Print(vars,
                   "@interface $service_class$ :"
-                  " ProtoService<$service_class$>\n");
+                  " GRPCProtoService<$service_class$>\n");
     printer.Print(
         "- (instancetype)initWithHost:(NSString *)host"
         " NS_DESIGNATED_INITIALIZER;\n");
@@ -220,18 +247,13 @@ void PrintMethodImplementations(Printer *printer,
                                 {"service_class", ServiceClassName(service)},
                                 {"package", service->file()->package()}};
 
-    printer.Print(vars,
-                  "static NSString *const kPackageName = @\"$package$\";\n");
-    printer.Print(
-        vars, "static NSString *const kServiceName = @\"$service_name$\";\n\n");
-
     printer.Print(vars, "@implementation $service_class$\n\n");
 
     printer.Print("// Designated initializer\n");
     printer.Print("- (instancetype)initWithHost:(NSString *)host {\n");
-    printer.Print(
+    printer.Print(vars,
         "  return (self = [super initWithHost:host"
-        " packageName:kPackageName serviceName:kServiceName]);\n");
+        " packageName:@\"$package$\" serviceName:@\"$service_name$\"]);\n");
     printer.Print("}\n\n");
     printer.Print(
         "// Override superclass initializer to disallow different"
diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc
index 8e76e6dce6..c5f1ed8061 100644
--- a/src/compiler/python_generator.cc
+++ b/src/compiler/python_generator.cc
@@ -66,6 +66,9 @@ using std::vector;
 
 namespace grpc_python_generator {
 
+GeneratorConfiguration::GeneratorConfiguration()
+    : grpc_package_root("grpc"), beta_package_root("grpc.beta") {}
+
 PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config)
     : config_(config) {}
 
@@ -147,7 +150,8 @@ class IndentScope {
 // END FORMATTING BOILERPLATE //
 ////////////////////////////////
 
-// TODO(protobuf team): Export `ModuleName` from protobuf's
+// TODO(https://github.com/google/protobuf/issues/888):
+// Export `ModuleName` from protobuf's
 // `src/google/protobuf/compiler/python/python_generator.cc` file.
 grpc::string ModuleName(const grpc::string& filename) {
   grpc::string basename = StripProto(filename);
@@ -156,8 +160,23 @@ grpc::string ModuleName(const grpc::string& filename) {
   return basename + "_pb2";
 }
 
+// TODO(https://github.com/google/protobuf/issues/888):
+// Export `ModuleAlias` from protobuf's
+// `src/google/protobuf/compiler/python/python_generator.cc` file.
+grpc::string ModuleAlias(const grpc::string& filename) {
+  grpc::string module_name = ModuleName(filename);
+  // We can't have dots in the module name, so we replace each with _dot_.
+  // But that could lead to a collision between a.b and a_dot_b, so we also
+  // duplicate each underscore.
+  module_name = StringReplace(module_name, "_", "__");
+  module_name = StringReplace(module_name, ".", "_dot_");
+  return module_name;
+}
+
+
 bool GetModuleAndMessagePath(const Descriptor* type,
-                             pair<grpc::string, grpc::string>* out) {
+                             const ServiceDescriptor* service,
+                             grpc::string* out) {
   const Descriptor* path_elem_type = type;
   vector<const Descriptor*> message_path;
   do {
@@ -170,7 +189,9 @@ bool GetModuleAndMessagePath(const Descriptor* type,
         file_name.find_last_of(".proto") == file_name.size() - 1)) {
     return false;
   }
-  grpc::string module = ModuleName(file_name);
+  grpc::string service_file_name = service->file()->name();
+  grpc::string module = service_file_name == file_name ?
+          "" : ModuleAlias(file_name) + ".";
   grpc::string message_type;
   for (auto path_iter = message_path.rbegin();
        path_iter != message_path.rend(); ++path_iter) {
@@ -178,7 +199,7 @@ bool GetModuleAndMessagePath(const Descriptor* type,
   }
   // no pop_back prior to C++11
   message_type.resize(message_type.size() - 1);
-  *out = make_pair(module, message_type);
+  *out = module + message_type;
   return true;
 }
 
@@ -210,7 +231,7 @@ static void PrintAllComments(const DescriptorType* desc, Printer* printer) {
 
 bool PrintBetaServicer(const ServiceDescriptor* service,
                        Printer* out) {
-  out->Print("\n");
+  out->Print("\n\n");
   out->Print("class Beta$Service$Servicer(object):\n", "Service",
              service->name());
   {
@@ -234,7 +255,7 @@ bool PrintBetaServicer(const ServiceDescriptor* service,
 
 bool PrintBetaStub(const ServiceDescriptor* service,
                    Printer* out) {
-  out->Print("\n");
+  out->Print("\n\n");
   out->Print("class Beta$Service$Stub(object):\n", "Service", service->name());
   {
     IndentScope raii_class_indent(out);
@@ -244,7 +265,7 @@ bool PrintBetaStub(const ServiceDescriptor* service,
       grpc::string arg_name = meth->client_streaming() ?
           "request_iterator" : "request";
       auto methdict = ListToDict({"Method", meth->name(), "ArgName", arg_name});
-      out->Print(methdict, "def $Method$(self, $ArgName$, timeout):\n");
+      out->Print(methdict, "def $Method$(self, $ArgName$, timeout, metadata=None, with_call=False, protocol_options=None):\n");
       {
         IndentScope raii_method_indent(out);
         PrintAllComments(meth, out);
@@ -260,38 +281,31 @@ bool PrintBetaStub(const ServiceDescriptor* service,
 
 bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
                             const ServiceDescriptor* service, Printer* out) {
-  out->Print("\n");
+  out->Print("\n\n");
   out->Print("def beta_create_$Service$_server(servicer, pool=None, "
              "pool_size=None, default_timeout=None, maximum_timeout=None):\n",
              "Service", service->name());
   {
     IndentScope raii_create_server_indent(out);
     map<grpc::string, grpc::string> method_implementation_constructors;
-    map<grpc::string, pair<grpc::string, grpc::string>>
-        input_message_modules_and_classes;
-    map<grpc::string, pair<grpc::string, grpc::string>>
-        output_message_modules_and_classes;
+    map<grpc::string, grpc::string> input_message_modules_and_classes;
+    map<grpc::string, grpc::string> output_message_modules_and_classes;
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* method = service->method(i);
       const grpc::string method_implementation_constructor =
           grpc::string(method->client_streaming() ? "stream_" : "unary_") +
           grpc::string(method->server_streaming() ? "stream_" : "unary_") +
           "inline";
-      pair<grpc::string, grpc::string> input_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->input_type(),
+      grpc::string input_message_module_and_class;
+      if (!GetModuleAndMessagePath(method->input_type(), service,
                                    &input_message_module_and_class)) {
         return false;
       }
-      pair<grpc::string, grpc::string> output_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->output_type(),
+      grpc::string output_message_module_and_class;
+      if (!GetModuleAndMessagePath(method->output_type(), service,
                                    &output_message_module_and_class)) {
         return false;
       }
-      // Import the modules that define the messages used in RPCs.
-      out->Print("import $Module$\n", "Module",
-                 input_message_module_and_class.first);
-      out->Print("import $Module$\n", "Module",
-                 output_message_module_and_class.first);
       method_implementation_constructors.insert(
           make_pair(method->name(), method_implementation_constructor));
       input_message_modules_and_classes.insert(
@@ -307,13 +321,11 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
          name_and_input_module_class_pair++) {
       IndentScope raii_indent(out);
       out->Print("(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): "
-                 "$InputTypeModule$.$InputTypeClass$.FromString,\n",
+                 "$InputTypeModuleAndClass$.FromString,\n",
                  "PackageQualifiedServiceName", package_qualified_service_name,
                  "MethodName", name_and_input_module_class_pair->first,
-                 "InputTypeModule",
-                 name_and_input_module_class_pair->second.first,
-                 "InputTypeClass",
-                 name_and_input_module_class_pair->second.second);
+                 "InputTypeModuleAndClass",
+                 name_and_input_module_class_pair->second);
     }
     out->Print("}\n");
     out->Print("response_serializers = {\n");
@@ -324,18 +336,16 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
          name_and_output_module_class_pair++) {
       IndentScope raii_indent(out);
       out->Print("(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): "
-                 "$OutputTypeModule$.$OutputTypeClass$.SerializeToString,\n",
+                 "$OutputTypeModuleAndClass$.SerializeToString,\n",
                  "PackageQualifiedServiceName", package_qualified_service_name,
                  "MethodName", name_and_output_module_class_pair->first,
-                 "OutputTypeModule",
-                 name_and_output_module_class_pair->second.first,
-                 "OutputTypeClass",
-                 name_and_output_module_class_pair->second.second);
+                 "OutputTypeModuleAndClass",
+                 name_and_output_module_class_pair->second);
     }
     out->Print("}\n");
     out->Print("method_implementations = {\n");
     for (auto name_and_implementation_constructor =
-	   method_implementation_constructors.begin();
+           method_implementation_constructors.begin();
 	 name_and_implementation_constructor !=
 	   method_implementation_constructors.end();
 	 name_and_implementation_constructor++) {
@@ -366,37 +376,30 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
   map<grpc::string, grpc::string> dict = ListToDict({
         "Service", service->name(),
       });
-  out->Print("\n");
+  out->Print("\n\n");
   out->Print(dict, "def beta_create_$Service$_stub(channel, host=None,"
              " metadata_transformer=None, pool=None, pool_size=None):\n");
   {
     IndentScope raii_create_server_indent(out);
     map<grpc::string, grpc::string> method_cardinalities;
-    map<grpc::string, pair<grpc::string, grpc::string>>
-        input_message_modules_and_classes;
-    map<grpc::string, pair<grpc::string, grpc::string>>
-        output_message_modules_and_classes;
+    map<grpc::string, grpc::string> input_message_modules_and_classes;
+    map<grpc::string, grpc::string> output_message_modules_and_classes;
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* method = service->method(i);
       const grpc::string method_cardinality =
           grpc::string(method->client_streaming() ? "STREAM" : "UNARY") +
           "_" +
-	  grpc::string(method->server_streaming() ? "STREAM" : "UNARY");
-      pair<grpc::string, grpc::string> input_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->input_type(),
+          grpc::string(method->server_streaming() ? "STREAM" : "UNARY");
+      grpc::string input_message_module_and_class;
+      if (!GetModuleAndMessagePath(method->input_type(), service,
                                    &input_message_module_and_class)) {
         return false;
       }
-      pair<grpc::string, grpc::string> output_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->output_type(),
+      grpc::string output_message_module_and_class;
+      if (!GetModuleAndMessagePath(method->output_type(), service,
                                    &output_message_module_and_class)) {
         return false;
       }
-      // Import the modules that define the messages used in RPCs.
-      out->Print("import $Module$\n", "Module",
-                 input_message_module_and_class.first);
-      out->Print("import $Module$\n", "Module",
-                 output_message_module_and_class.first);
       method_cardinalities.insert(
           make_pair(method->name(), method_cardinality));
       input_message_modules_and_classes.insert(
@@ -412,13 +415,11 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
          name_and_input_module_class_pair++) {
       IndentScope raii_indent(out);
       out->Print("(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): "
-                 "$InputTypeModule$.$InputTypeClass$.SerializeToString,\n",
+                 "$InputTypeModuleAndClass$.SerializeToString,\n",
                  "PackageQualifiedServiceName", package_qualified_service_name,
                  "MethodName", name_and_input_module_class_pair->first,
-                 "InputTypeModule",
-                 name_and_input_module_class_pair->second.first,
-                 "InputTypeClass",
-                 name_and_input_module_class_pair->second.second);
+                 "InputTypeModuleAndClass",
+                 name_and_input_module_class_pair->second);
     }
     out->Print("}\n");
     out->Print("response_deserializers = {\n");
@@ -429,13 +430,11 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
          name_and_output_module_class_pair++) {
       IndentScope raii_indent(out);
       out->Print("(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): "
-                 "$OutputTypeModule$.$OutputTypeClass$.FromString,\n",
+                 "$OutputTypeModuleAndClass$.FromString,\n",
                  "PackageQualifiedServiceName", package_qualified_service_name,
                  "MethodName", name_and_output_module_class_pair->first,
-                 "OutputTypeModule",
-                 name_and_output_module_class_pair->second.first,
-                 "OutputTypeClass",
-                 name_and_output_module_class_pair->second.second);
+                 "OutputTypeModuleAndClass",
+                 name_and_output_module_class_pair->second);
     }
     out->Print("}\n");
     out->Print("cardinalities = {\n");
@@ -461,10 +460,149 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
   return true;
 }
 
+bool PrintStub(const grpc::string& package_qualified_service_name,
+               const ServiceDescriptor* service, Printer* out) {
+  out->Print("\n\n");
+  out->Print("class $Service$Stub(object):\n", "Service", service->name());
+  {
+    IndentScope raii_class_indent(out);
+    PrintAllComments(service, out);
+    out->Print("\n");
+    out->Print("def __init__(self, channel):\n");
+    {
+      IndentScope raii_init_indent(out);
+      out->Print("\"\"\"Constructor.\n");
+      out->Print("\n");
+      out->Print("Args:\n");
+      {
+        IndentScope raii_args_indent(out);
+	out->Print("channel: A grpc.Channel.\n");
+      }
+      out->Print("\"\"\"\n");
+      for (int i = 0; i < service->method_count(); ++i) {
+        auto method = service->method(i);
+	auto multi_callable_constructor =
+	    grpc::string(method->client_streaming() ? "stream" : "unary") +
+	    "_" +
+	    grpc::string(method->server_streaming() ? "stream" : "unary");
+	grpc::string request_module_and_class;
+	if (!GetModuleAndMessagePath(method->input_type(), service,
+				     &request_module_and_class)) {
+	  return false;
+	}
+	grpc::string response_module_and_class;
+	if (!GetModuleAndMessagePath(method->output_type(), service,
+				     &response_module_and_class)) {
+          return false;
+	}
+	out->Print("self.$Method$ = channel.$MultiCallableConstructor$(\n",
+		   "Method", method->name(),
+		   "MultiCallableConstructor", multi_callable_constructor);
+	{
+          IndentScope raii_first_attribute_indent(out);
+          IndentScope raii_second_attribute_indent(out);
+	  out->Print(
+	      "'/$PackageQualifiedService$/$Method$',\n",
+	      "PackageQualifiedService", package_qualified_service_name,
+	      "Method", method->name());
+	  out->Print(
+	      "request_serializer=$RequestModuleAndClass$.SerializeToString,\n",
+	      "RequestModuleAndClass", request_module_and_class);
+	  out->Print(
+              "response_deserializer=$ResponseModuleAndClass$.FromString,\n",
+	      "ResponseModuleAndClass", response_module_and_class);
+	  out->Print(")\n");
+	}
+      }
+    }
+  }
+  return true;
+}
+
+bool PrintServicer(const ServiceDescriptor* service, Printer* out) {
+  out->Print("\n\n");
+  out->Print("class $Service$Servicer(object):\n", "Service", service->name());
+  {
+    IndentScope raii_class_indent(out);
+    PrintAllComments(service, out);
+    for (int i = 0; i < service->method_count(); ++i) {
+      auto method = service->method(i);
+      grpc::string arg_name = method->client_streaming() ?
+	  "request_iterator" : "request";
+      out->Print("\n");
+      out->Print("def $Method$(self, $ArgName$, context):\n",
+                 "Method", method->name(), "ArgName", arg_name);
+      {
+        IndentScope raii_method_indent(out);
+        PrintAllComments(method, out);
+        out->Print("context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n");
+        out->Print("context.set_details('Method not implemented!')\n");
+        out->Print("raise NotImplementedError('Method not implemented!')\n");
+      }
+    }
+  }
+  return true;
+}
+
+bool PrintAddServicerToServer(const grpc::string& package_qualified_service_name,
+			      const ServiceDescriptor* service, Printer* out) {
+  out->Print("\n\n");
+  out->Print("def add_$Service$Servicer_to_server(servicer, server):\n",
+	     "Service", service->name());
+  {
+    IndentScope raii_class_indent(out);
+    out->Print("rpc_method_handlers = {\n");
+    {
+      IndentScope raii_dict_first_indent(out);
+      IndentScope raii_dict_second_indent(out);
+      for (int i = 0; i < service->method_count(); ++i) {
+        auto method = service->method(i);
+	auto method_handler_constructor =
+            grpc::string(method->client_streaming() ? "stream" : "unary") +
+	    "_" +
+            grpc::string(method->server_streaming() ? "stream" : "unary") +
+            "_rpc_method_handler";
+	grpc::string request_module_and_class;
+	if (!GetModuleAndMessagePath(method->input_type(), service,
+				     &request_module_and_class)) {
+	  return false;
+	}
+	grpc::string response_module_and_class;
+	if (!GetModuleAndMessagePath(method->output_type(), service,
+				     &response_module_and_class)) {
+          return false;
+	}
+        out->Print("'$Method$': grpc.$MethodHandlerConstructor$(\n",
+		   "Method", method->name(),
+		   "MethodHandlerConstructor", method_handler_constructor);
+	{
+          IndentScope raii_call_first_indent(out);
+	  IndentScope raii_call_second_indent(out);
+	  out->Print("servicer.$Method$,\n", "Method", method->name());
+	  out->Print("request_deserializer=$RequestModuleAndClass$.FromString,\n",
+		     "RequestModuleAndClass", request_module_and_class);
+	  out->Print("response_serializer=$ResponseModuleAndClass$.SerializeToString,\n",
+		     "ResponseModuleAndClass", response_module_and_class);
+	}
+	out->Print("),\n");
+      }
+    }
+    out->Print("}\n");
+    out->Print("generic_handler = grpc.method_handlers_generic_handler(\n");
+    {
+      IndentScope raii_call_first_indent(out);
+      IndentScope raii_call_second_indent(out);
+      out->Print("'$PackageQualifiedServiceName$', rpc_method_handlers)\n",
+		 "PackageQualifiedServiceName", package_qualified_service_name);
+    }
+    out->Print("server.add_generic_rpc_handlers((generic_handler,))\n");
+  }
+  return true;
+}
+
 bool PrintPreamble(const FileDescriptor* file,
                    const GeneratorConfiguration& config, Printer* out) {
-  out->Print("import abc\n");
-  out->Print("import six\n");
+  out->Print("import $Package$\n", "Package", config.grpc_package_root);
   out->Print("from $Package$ import implementations as beta_implementations\n",
              "Package", config.beta_package_root);
   out->Print("from $Package$ import interfaces as beta_interfaces\n",
@@ -493,7 +631,10 @@ pair<bool, grpc::string> GetServices(const FileDescriptor* file,
     for (int i = 0; i < file->service_count(); ++i) {
       auto service = file->service(i);
       auto package_qualified_service_name = package + service->name();
-      if (!(PrintBetaServicer(service, &out) &&
+      if (!(PrintStub(package_qualified_service_name, service, &out) &&
+	    PrintServicer(service, &out) &&
+	    PrintAddServicerToServer(package_qualified_service_name, service, &out) &&
+	    PrintBetaServicer(service, &out) &&
             PrintBetaStub(service, &out) &&
             PrintBetaServerFactory(package_qualified_service_name, service, &out) &&
             PrintBetaStubFactory(package_qualified_service_name, service, &out))) {
diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h
index e56b6790ef..7ed99eff0b 100644
--- a/src/compiler/python_generator.h
+++ b/src/compiler/python_generator.h
@@ -43,6 +43,8 @@ namespace grpc_python_generator {
 // Data pertaining to configuration of the generator with respect to anything
 // that may be used internally at Google.
 struct GeneratorConfiguration {
+  GeneratorConfiguration();
+  grpc::string grpc_package_root;
   grpc::string beta_package_root;
 };
 
diff --git a/src/compiler/python_plugin.cc b/src/compiler/python_plugin.cc
index 92a07b2f80..3457aa631a 100644
--- a/src/compiler/python_plugin.cc
+++ b/src/compiler/python_plugin.cc
@@ -38,7 +38,6 @@
 
 int main(int argc, char* argv[]) {
   grpc_python_generator::GeneratorConfiguration config;
-  config.beta_package_root = "grpc.beta";
   grpc_python_generator::PythonGrpcGenerator generator(config);
   return grpc::protobuf::compiler::PluginMain(argc, argv, &generator);
 }
diff --git a/src/compiler/ruby_generator.cc b/src/compiler/ruby_generator.cc
index 5ac56ad289..1501c3f3e0 100644
--- a/src/compiler/ruby_generator.cc
+++ b/src/compiler/ruby_generator.cc
@@ -66,7 +66,9 @@ void PrintMethod(const MethodDescriptor *method, const grpc::string &package,
   std::map<grpc::string, grpc::string> method_vars =
       ListToDict({"mth.name", method->name(), "input.type", input_type,
                   "output.type", output_type, });
+  out->Print(GetRubyComments(method, true).c_str());
   out->Print(method_vars, "rpc :$mth.name$, $input.type$, $output.type$\n");
+  out->Print(GetRubyComments(method, false).c_str());
 }
 
 // Prints out the service using the ruby gRPC DSL.
@@ -82,12 +84,7 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package,
   out->Print(module_vars, "module $module.name$\n");
   out->Indent();
 
-  // TODO(temiola): add documentation
-  grpc::string doc = "TODO: add proto service documentation here";
-  std::map<grpc::string, grpc::string> template_vars =
-      ListToDict({"Documentation", doc, });
-  out->Print("\n");
-  out->Print(template_vars, "# $Documentation$\n");
+  out->Print(GetRubyComments(service, true).c_str());
   out->Print("class Service\n");
 
   // Write the indented class body.
@@ -98,8 +95,8 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package,
   out->Print("self.marshal_class_method = :encode\n");
   out->Print("self.unmarshal_class_method = :decode\n");
   std::map<grpc::string, grpc::string> pkg_vars =
-      ListToDict({"service.name", service->name(), "pkg.name", package, });
-  out->Print(pkg_vars, "self.service_name = '$pkg.name$.$service.name$'\n");
+      ListToDict({"service_full_name", service->full_name()});
+  out->Print(pkg_vars, "self.service_name = '$service_full_name$'\n");
   out->Print("\n");
   for (int i = 0; i < service->method_count(); ++i) {
     PrintMethod(service->method(i), package, out);
@@ -113,6 +110,7 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package,
   // End the service module
   out->Outdent();
   out->Print("end\n");
+  out->Print(GetRubyComments(service, false).c_str());
 }
 
 }  // namespace
@@ -138,6 +136,12 @@ grpc::string GetServices(const FileDescriptor *file) {
     out.Print(header_comment_vars,
               "# Source: $file.name$ for package '$file.package$'\n");
 
+    grpc::string leading_comments = GetRubyComments(file, true);
+    if (!leading_comments.empty()) {
+      out.Print("# Original file comments:\n");
+      out.Print(leading_comments.c_str());
+    }
+
     out.Print("\n");
     out.Print("require 'grpc'\n");
     // Write out require statemment to import the separately generated file
@@ -164,6 +168,8 @@ grpc::string GetServices(const FileDescriptor *file) {
       out.Outdent();
       out.Print("end\n");
     }
+
+    out.Print(GetRubyComments(file, false).c_str());
   }
   return output;
 }
diff --git a/src/compiler/ruby_generator_helpers-inl.h b/src/compiler/ruby_generator_helpers-inl.h
index 9da7cab3c7..ff6939ed9f 100644
--- a/src/compiler/ruby_generator_helpers-inl.h
+++ b/src/compiler/ruby_generator_helpers-inl.h
@@ -35,6 +35,7 @@
 #define GRPC_INTERNAL_COMPILER_RUBY_GENERATOR_HELPERS_INL_H
 
 #include "src/compiler/config.h"
+#include "src/compiler/generator_helpers.h"
 #include "src/compiler/ruby_generator_string-inl.h"
 
 namespace grpc_ruby_generator {
@@ -60,6 +61,13 @@ inline grpc::string MessagesRequireName(
   return Replace(file->name(), ".proto", "");
 }
 
+// Get leading or trailing comments in a string. Comment lines start with "# ".
+// Leading detached comments are put in in front of leading comments.
+template <typename DescriptorType>
+inline grpc::string GetRubyComments(const DescriptorType *desc, bool leading) {
+  return grpc_generator::GetPrefixedComments(desc, leading, "#");
+}
+
 }  // namespace grpc_ruby_generator
 
 #endif  // GRPC_INTERNAL_COMPILER_RUBY_GENERATOR_HELPERS_INL_H
diff --git a/src/core/ext/census/gen/README.md b/src/core/ext/census/gen/README.md
new file mode 100644
index 0000000000..72bef6542d
--- /dev/null
+++ b/src/core/ext/census/gen/README.md
@@ -0,0 +1,6 @@
+Files generated for use by Census stats and trace recording subsystem.
+
+#Files
+* census.pb.{h,c} - Generated from src/core/ext/census/census.proto, using the
+  script `tools/codegen/core/gen_nano_proto.sh src/proto/census/census.proto
+  $PWD/src/core/ext/census/gen src/core/ext/census/gen`
diff --git a/src/core/ext/census/gen/census.pb.c b/src/core/ext/census/gen/census.pb.c
new file mode 100644
index 0000000000..d614636c90
--- /dev/null
+++ b/src/core/ext/census/gen/census.pb.c
@@ -0,0 +1,179 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.3.5-dev */
+
+#include "src/core/ext/census/gen/census.pb.h"
+
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+
+
+const pb_field_t google_census_Duration_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Duration, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Duration, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Timestamp_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Timestamp, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Timestamp, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_fields[5] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_Metric, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_Metric, description, name, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_Metric, unit, description, &google_census_Metric_MeasurementUnit_fields),
+    PB_FIELD(  4, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Metric, id, unit, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_BasicUnit_fields[2] = {
+    PB_FIELD(  1, UENUM   , OPTIONAL, STATIC  , FIRST, google_census_Metric_BasicUnit, type, type, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_MeasurementUnit_fields[4] = {
+    PB_FIELD(  1, INT32   , OPTIONAL, STATIC  , FIRST, google_census_Metric_MeasurementUnit, prefix, prefix, 0),
+    PB_FIELD(  2, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Metric_MeasurementUnit, numerator, prefix, &google_census_Metric_BasicUnit_fields),
+    PB_FIELD(  3, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Metric_MeasurementUnit, denominator, numerator, &google_census_Metric_BasicUnit_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_fields[3] = {
+    PB_ONEOF_FIELD(options,   1, MESSAGE , ONEOF, STATIC  , FIRST, google_census_AggregationDescriptor, bucket_boundaries, bucket_boundaries, &google_census_AggregationDescriptor_BucketBoundaries_fields),
+    PB_ONEOF_FIELD(options,   2, MESSAGE , ONEOF, STATIC  , FIRST, google_census_AggregationDescriptor, interval_boundaries, interval_boundaries, &google_census_AggregationDescriptor_IntervalBoundaries_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_BucketBoundaries_fields[2] = {
+    PB_FIELD(  1, DOUBLE  , REPEATED, CALLBACK, FIRST, google_census_AggregationDescriptor_BucketBoundaries, bounds, bounds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_IntervalBoundaries_fields[2] = {
+    PB_FIELD(  1, DOUBLE  , REPEATED, CALLBACK, FIRST, google_census_AggregationDescriptor_IntervalBoundaries, window_size, window_size, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Distribution_fields[5] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Distribution, count, count, 0),
+    PB_FIELD(  2, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_Distribution, mean, count, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_Distribution, range, mean, &google_census_Distribution_Range_fields),
+    PB_FIELD(  4, INT64   , REPEATED, CALLBACK, OTHER, google_census_Distribution, bucket_count, range, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Distribution_Range_fields[3] = {
+    PB_FIELD(  1, DOUBLE  , OPTIONAL, STATIC  , FIRST, google_census_Distribution_Range, min, min, 0),
+    PB_FIELD(  2, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_Distribution_Range, max, min, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_IntervalStats_fields[2] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, google_census_IntervalStats, window, window, &google_census_IntervalStats_Window_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_IntervalStats_Window_fields[4] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, google_census_IntervalStats_Window, window_size, window_size, &google_census_Duration_fields),
+    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, google_census_IntervalStats_Window, count, window_size, 0),
+    PB_FIELD(  3, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_IntervalStats_Window, mean, count, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Tag_fields[3] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, google_census_Tag, key, key, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , OTHER, google_census_Tag, value, key, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_View_fields[6] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_View, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_View, description, name, 0),
+    PB_FIELD(  3, INT32   , OPTIONAL, STATIC  , OTHER, google_census_View, metric_id, description, 0),
+    PB_FIELD(  4, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_View, aggregation, metric_id, &google_census_AggregationDescriptor_fields),
+    PB_FIELD(  5, STRING  , REPEATED, CALLBACK, OTHER, google_census_View, tag_key, aggregation, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Aggregation_fields[6] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_Aggregation, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_Aggregation, description, name, 0),
+    PB_ONEOF_FIELD(data,   3, MESSAGE , ONEOF, STATIC  , OTHER, google_census_Aggregation, distribution, description, &google_census_Distribution_fields),
+    PB_ONEOF_FIELD(data,   4, MESSAGE , ONEOF, STATIC  , OTHER, google_census_Aggregation, interval_stats, description, &google_census_IntervalStats_fields),
+    PB_FIELD(  5, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Aggregation, tag, data.interval_stats, &google_census_Tag_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_ViewAggregations_fields[4] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, google_census_ViewAggregations, aggregation, aggregation, &google_census_Aggregation_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_ViewAggregations, start, aggregation, &google_census_Timestamp_fields),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_ViewAggregations, end, start, &google_census_Timestamp_fields),
+    PB_LAST_FIELD
+};
+
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_32BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in 8 or 16 bit
+ * field descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Distribution, range) < 65536 && pb_membersize(google_census_IntervalStats, window) < 65536 && pb_membersize(google_census_IntervalStats_Window, window_size) < 65536 && pb_membersize(google_census_View, aggregation) < 65536 && pb_membersize(google_census_Aggregation, data.distribution) < 65536 && pb_membersize(google_census_Aggregation, data.interval_stats) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Distribution, range) < 65536 && pb_membersize(google_census_IntervalStats, window) < 65536 && pb_membersize(google_census_IntervalStats_Window, window_size) < 65536 && pb_membersize(google_census_View, aggregation) < 65536 && pb_membersize(google_census_Aggregation, data.distribution) < 65536 && pb_membersize(google_census_Aggregation, data.interval_stats) < 65536 && pb_membersize(google_census_Aggregation, tag) < 65536 && pb_membersize(google_census_ViewAggregations, aggregation) < 65536 && pb_membersize(google_census_ViewAggregations, start) < 65536 && pb_membersize(google_census_ViewAggregations, end) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_google_census_Duration_google_census_Timestamp_google_census_Metric_google_census_Metric_BasicUnit_google_census_Metric_MeasurementUnit_google_census_AggregationDescriptor_google_census_AggregationDescriptor_BucketBoundaries_google_census_AggregationDescriptor_IntervalBoundaries_google_census_Distribution_google_census_Distribution_Range_google_census_IntervalStats_google_census_IntervalStats_Window_google_census_Tag_google_census_View_google_census_Aggregation_google_census_ViewAggregations)
+#endif
+
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_16BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in the default
+ * 8 bit descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Distribution, range) < 256 && pb_membersize(google_census_IntervalStats, window) < 256 && pb_membersize(google_census_IntervalStats_Window, window_size) < 256 && pb_membersize(google_census_View, aggregation) < 256 && pb_membersize(google_census_Aggregation, data.distribution) < 256 && pb_membersize(google_census_Aggregation, data.interval_stats) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Distribution, range) < 256 && pb_membersize(google_census_IntervalStats, window) < 256 && pb_membersize(google_census_IntervalStats_Window, window_size) < 256 && pb_membersize(google_census_View, aggregation) < 256 && pb_membersize(google_census_Aggregation, data.distribution) < 256 && pb_membersize(google_census_Aggregation, data.interval_stats) < 256 && pb_membersize(google_census_Aggregation, tag) < 256 && pb_membersize(google_census_ViewAggregations, aggregation) < 256 && pb_membersize(google_census_ViewAggregations, start) < 256 && pb_membersize(google_census_ViewAggregations, end) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_google_census_Duration_google_census_Timestamp_google_census_Metric_google_census_Metric_BasicUnit_google_census_Metric_MeasurementUnit_google_census_AggregationDescriptor_google_census_AggregationDescriptor_BucketBoundaries_google_census_AggregationDescriptor_IntervalBoundaries_google_census_Distribution_google_census_Distribution_Range_google_census_IntervalStats_google_census_IntervalStats_Window_google_census_Tag_google_census_View_google_census_Aggregation_google_census_ViewAggregations)
+#endif
+
+
+/* On some platforms (such as AVR), double is really float.
+ * These are not directly supported by nanopb, but see example_avr_double.
+ * To get rid of this error, remove any double fields from your .proto.
+ */
+PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)
+
diff --git a/src/core/ext/census/gen/census.pb.h b/src/core/ext/census/gen/census.pb.h
new file mode 100644
index 0000000000..d040fe29e7
--- /dev/null
+++ b/src/core/ext/census/gen/census.pb.h
@@ -0,0 +1,294 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.3.5-dev */
+
+#ifndef GRPC_CORE_EXT_CENSUS_GEN_CENSUS_PB_H
+#define GRPC_CORE_EXT_CENSUS_GEN_CENSUS_PB_H
+#include "third_party/nanopb/pb.h"
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Enum definitions */
+typedef enum _google_census_Metric_BasicUnit_Measure {
+    google_census_Metric_BasicUnit_Measure_UNKNOWN = 0,
+    google_census_Metric_BasicUnit_Measure_BITS = 1,
+    google_census_Metric_BasicUnit_Measure_BYTES = 2,
+    google_census_Metric_BasicUnit_Measure_SECS = 3,
+    google_census_Metric_BasicUnit_Measure_CORES = 4,
+    google_census_Metric_BasicUnit_Measure_MAX_UNITS = 5
+} google_census_Metric_BasicUnit_Measure;
+
+/* Struct definitions */
+typedef struct _google_census_AggregationDescriptor_BucketBoundaries {
+    pb_callback_t bounds;
+} google_census_AggregationDescriptor_BucketBoundaries;
+
+typedef struct _google_census_AggregationDescriptor_IntervalBoundaries {
+    pb_callback_t window_size;
+} google_census_AggregationDescriptor_IntervalBoundaries;
+
+typedef struct _google_census_IntervalStats {
+    pb_callback_t window;
+} google_census_IntervalStats;
+
+typedef struct _google_census_AggregationDescriptor {
+    pb_size_t which_options;
+    union {
+        google_census_AggregationDescriptor_BucketBoundaries bucket_boundaries;
+        google_census_AggregationDescriptor_IntervalBoundaries interval_boundaries;
+    } options;
+} google_census_AggregationDescriptor;
+
+typedef struct _google_census_Distribution_Range {
+    bool has_min;
+    double min;
+    bool has_max;
+    double max;
+} google_census_Distribution_Range;
+
+typedef struct _google_census_Duration {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} google_census_Duration;
+
+typedef struct _google_census_Metric_BasicUnit {
+    bool has_type;
+    google_census_Metric_BasicUnit_Measure type;
+} google_census_Metric_BasicUnit;
+
+typedef struct _google_census_Metric_MeasurementUnit {
+    bool has_prefix;
+    int32_t prefix;
+    pb_callback_t numerator;
+    pb_callback_t denominator;
+} google_census_Metric_MeasurementUnit;
+
+typedef struct _google_census_Tag {
+    bool has_key;
+    char key[255];
+    bool has_value;
+    char value[255];
+} google_census_Tag;
+
+typedef struct _google_census_Timestamp {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} google_census_Timestamp;
+
+typedef struct _google_census_Distribution {
+    bool has_count;
+    int64_t count;
+    bool has_mean;
+    double mean;
+    bool has_range;
+    google_census_Distribution_Range range;
+    pb_callback_t bucket_count;
+} google_census_Distribution;
+
+typedef struct _google_census_IntervalStats_Window {
+    bool has_window_size;
+    google_census_Duration window_size;
+    bool has_count;
+    int64_t count;
+    bool has_mean;
+    double mean;
+} google_census_IntervalStats_Window;
+
+typedef struct _google_census_Metric {
+    pb_callback_t name;
+    pb_callback_t description;
+    bool has_unit;
+    google_census_Metric_MeasurementUnit unit;
+    bool has_id;
+    int32_t id;
+} google_census_Metric;
+
+typedef struct _google_census_View {
+    pb_callback_t name;
+    pb_callback_t description;
+    bool has_metric_id;
+    int32_t metric_id;
+    bool has_aggregation;
+    google_census_AggregationDescriptor aggregation;
+    pb_callback_t tag_key;
+} google_census_View;
+
+typedef struct _google_census_ViewAggregations {
+    pb_callback_t aggregation;
+    bool has_start;
+    google_census_Timestamp start;
+    bool has_end;
+    google_census_Timestamp end;
+} google_census_ViewAggregations;
+
+typedef struct _google_census_Aggregation {
+    pb_callback_t name;
+    pb_callback_t description;
+    pb_size_t which_data;
+    union {
+        google_census_Distribution distribution;
+        google_census_IntervalStats interval_stats;
+    } data;
+    pb_callback_t tag;
+} google_census_Aggregation;
+
+/* Default values for struct fields */
+
+/* Initializer values for message structs */
+#define google_census_Duration_init_default      {false, 0, false, 0}
+#define google_census_Timestamp_init_default     {false, 0, false, 0}
+#define google_census_Metric_init_default        {{{NULL}, NULL}, {{NULL}, NULL}, false, google_census_Metric_MeasurementUnit_init_default, false, 0}
+#define google_census_Metric_BasicUnit_init_default {false, (google_census_Metric_BasicUnit_Measure)0}
+#define google_census_Metric_MeasurementUnit_init_default {false, 0, {{NULL}, NULL}, {{NULL}, NULL}}
+#define google_census_AggregationDescriptor_init_default {0, {google_census_AggregationDescriptor_BucketBoundaries_init_default}}
+#define google_census_AggregationDescriptor_BucketBoundaries_init_default {{{NULL}, NULL}}
+#define google_census_AggregationDescriptor_IntervalBoundaries_init_default {{{NULL}, NULL}}
+#define google_census_Distribution_init_default  {false, 0, false, 0, false, google_census_Distribution_Range_init_default, {{NULL}, NULL}}
+#define google_census_Distribution_Range_init_default {false, 0, false, 0}
+#define google_census_IntervalStats_init_default {{{NULL}, NULL}}
+#define google_census_IntervalStats_Window_init_default {false, google_census_Duration_init_default, false, 0, false, 0}
+#define google_census_Tag_init_default           {false, "", false, ""}
+#define google_census_View_init_default          {{{NULL}, NULL}, {{NULL}, NULL}, false, 0, false, google_census_AggregationDescriptor_init_default, {{NULL}, NULL}}
+#define google_census_Aggregation_init_default   {{{NULL}, NULL}, {{NULL}, NULL}, 0, {google_census_Distribution_init_default}, {{NULL}, NULL}}
+#define google_census_ViewAggregations_init_default {{{NULL}, NULL}, false, google_census_Timestamp_init_default, false, google_census_Timestamp_init_default}
+#define google_census_Duration_init_zero         {false, 0, false, 0}
+#define google_census_Timestamp_init_zero        {false, 0, false, 0}
+#define google_census_Metric_init_zero           {{{NULL}, NULL}, {{NULL}, NULL}, false, google_census_Metric_MeasurementUnit_init_zero, false, 0}
+#define google_census_Metric_BasicUnit_init_zero {false, (google_census_Metric_BasicUnit_Measure)0}
+#define google_census_Metric_MeasurementUnit_init_zero {false, 0, {{NULL}, NULL}, {{NULL}, NULL}}
+#define google_census_AggregationDescriptor_init_zero {0, {google_census_AggregationDescriptor_BucketBoundaries_init_zero}}
+#define google_census_AggregationDescriptor_BucketBoundaries_init_zero {{{NULL}, NULL}}
+#define google_census_AggregationDescriptor_IntervalBoundaries_init_zero {{{NULL}, NULL}}
+#define google_census_Distribution_init_zero     {false, 0, false, 0, false, google_census_Distribution_Range_init_zero, {{NULL}, NULL}}
+#define google_census_Distribution_Range_init_zero {false, 0, false, 0}
+#define google_census_IntervalStats_init_zero    {{{NULL}, NULL}}
+#define google_census_IntervalStats_Window_init_zero {false, google_census_Duration_init_zero, false, 0, false, 0}
+#define google_census_Tag_init_zero              {false, "", false, ""}
+#define google_census_View_init_zero             {{{NULL}, NULL}, {{NULL}, NULL}, false, 0, false, google_census_AggregationDescriptor_init_zero, {{NULL}, NULL}}
+#define google_census_Aggregation_init_zero      {{{NULL}, NULL}, {{NULL}, NULL}, 0, {google_census_Distribution_init_zero}, {{NULL}, NULL}}
+#define google_census_ViewAggregations_init_zero {{{NULL}, NULL}, false, google_census_Timestamp_init_zero, false, google_census_Timestamp_init_zero}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define google_census_AggregationDescriptor_BucketBoundaries_bounds_tag 1
+#define google_census_AggregationDescriptor_IntervalBoundaries_window_size_tag 1
+#define google_census_IntervalStats_window_tag   1
+#define google_census_AggregationDescriptor_bucket_boundaries_tag 1
+
+#define google_census_AggregationDescriptor_interval_boundaries_tag 2
+#define google_census_Distribution_Range_min_tag 1
+#define google_census_Distribution_Range_max_tag 2
+#define google_census_Duration_seconds_tag       1
+#define google_census_Duration_nanos_tag         2
+#define google_census_Metric_BasicUnit_type_tag  1
+#define google_census_Metric_MeasurementUnit_prefix_tag 1
+#define google_census_Metric_MeasurementUnit_numerator_tag 2
+#define google_census_Metric_MeasurementUnit_denominator_tag 3
+#define google_census_Tag_key_tag                1
+#define google_census_Tag_value_tag              2
+#define google_census_Timestamp_seconds_tag      1
+#define google_census_Timestamp_nanos_tag        2
+#define google_census_Distribution_count_tag     1
+#define google_census_Distribution_mean_tag      2
+#define google_census_Distribution_range_tag     3
+#define google_census_Distribution_bucket_count_tag 4
+#define google_census_IntervalStats_Window_window_size_tag 1
+#define google_census_IntervalStats_Window_count_tag 2
+#define google_census_IntervalStats_Window_mean_tag 3
+#define google_census_Metric_name_tag            1
+#define google_census_Metric_description_tag     2
+#define google_census_Metric_unit_tag            3
+#define google_census_Metric_id_tag              4
+#define google_census_View_name_tag              1
+#define google_census_View_description_tag       2
+#define google_census_View_metric_id_tag         3
+#define google_census_View_aggregation_tag       4
+#define google_census_View_tag_key_tag           5
+#define google_census_ViewAggregations_aggregation_tag 1
+#define google_census_ViewAggregations_start_tag 2
+#define google_census_ViewAggregations_end_tag   3
+#define google_census_Aggregation_distribution_tag 3
+
+#define google_census_Aggregation_interval_stats_tag 4
+#define google_census_Aggregation_name_tag       1
+#define google_census_Aggregation_description_tag 2
+#define google_census_Aggregation_tag_tag        5
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t google_census_Duration_fields[3];
+extern const pb_field_t google_census_Timestamp_fields[3];
+extern const pb_field_t google_census_Metric_fields[5];
+extern const pb_field_t google_census_Metric_BasicUnit_fields[2];
+extern const pb_field_t google_census_Metric_MeasurementUnit_fields[4];
+extern const pb_field_t google_census_AggregationDescriptor_fields[3];
+extern const pb_field_t google_census_AggregationDescriptor_BucketBoundaries_fields[2];
+extern const pb_field_t google_census_AggregationDescriptor_IntervalBoundaries_fields[2];
+extern const pb_field_t google_census_Distribution_fields[5];
+extern const pb_field_t google_census_Distribution_Range_fields[3];
+extern const pb_field_t google_census_IntervalStats_fields[2];
+extern const pb_field_t google_census_IntervalStats_Window_fields[4];
+extern const pb_field_t google_census_Tag_fields[3];
+extern const pb_field_t google_census_View_fields[6];
+extern const pb_field_t google_census_Aggregation_fields[6];
+extern const pb_field_t google_census_ViewAggregations_fields[4];
+
+/* Maximum encoded size of messages (where known) */
+#define google_census_Duration_size              22
+#define google_census_Timestamp_size             22
+#define google_census_Metric_BasicUnit_size      2
+#define google_census_Distribution_Range_size    18
+#define google_census_IntervalStats_Window_size  44
+#define google_census_Tag_size                   516
+
+/* Message IDs (where set with "msgid" option) */
+#ifdef PB_MSGID
+
+#define CENSUS_MESSAGES \
+
+
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/src/core/ext/census/grpc_filter.c b/src/core/ext/census/grpc_filter.c
index 5e278ef127..f51d850e01 100644
--- a/src/core/ext/census/grpc_filter.c
+++ b/src/core/ext/census/grpc_filter.c
@@ -45,6 +45,7 @@
 #include "src/core/ext/census/census_interface.h"
 #include "src/core/ext/census/census_rpc_stats.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 typedef struct call_data {
@@ -91,14 +92,16 @@ static void client_start_transport_op(grpc_exec_ctx *exec_ctx,
 }
 
 static void server_on_done_recv(grpc_exec_ctx *exec_ctx, void *ptr,
-                                bool success) {
+                                grpc_error *error) {
+  GPR_TIMER_BEGIN("census-server:server_on_done_recv", 0);
   grpc_call_element *elem = ptr;
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     extract_and_annotate_method_tag(calld->recv_initial_metadata, calld, chand);
   }
-  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success);
+  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, error);
+  GPR_TIMER_END("census-server:server_on_done_recv", 0);
 }
 
 static void server_mutate_op(grpc_call_element *elem,
@@ -134,7 +137,9 @@ static void client_init_call_elem(grpc_exec_ctx *exec_ctx,
 }
 
 static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_element *elem, void *ignored) {
+                                     grpc_call_element *elem,
+                                     const grpc_call_stats *stats,
+                                     void *ignored) {
   call_data *d = elem->call_data;
   GPR_ASSERT(d != NULL);
   /* TODO(hongyu): record rpc client stats and census_rpc_end_op here */
@@ -152,7 +157,9 @@ static void server_init_call_elem(grpc_exec_ctx *exec_ctx,
 }
 
 static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_element *elem, void *ignored) {
+                                     grpc_call_element *elem,
+                                     const grpc_call_stats *stats,
+                                     void *ignored) {
   call_data *d = elem->call_data;
   GPR_ASSERT(d != NULL);
   /* TODO(hongyu): record rpc server stats and census_tracing_end_op here */
@@ -176,7 +183,7 @@ const grpc_channel_filter grpc_client_census_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     client_init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     client_destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
@@ -189,7 +196,7 @@ const grpc_channel_filter grpc_server_census_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     server_init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     server_destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/ext/client_config/channel_connectivity.c b/src/core/ext/client_config/channel_connectivity.c
index 3ebc333608..c1220e3a8c 100644
--- a/src/core/ext/client_config/channel_connectivity.c
+++ b/src/core/ext/client_config/channel_connectivity.c
@@ -62,7 +62,7 @@ grpc_connectivity_state grpc_channel_check_connectivity_state(
           "not a (u)client channel, but '%s'",
           client_channel_elem->filter->name);
   grpc_exec_ctx_finish(&exec_ctx);
-  return GRPC_CHANNEL_FATAL_FAILURE;
+  return GRPC_CHANNEL_SHUTDOWN;
 }
 
 typedef enum {
@@ -75,7 +75,6 @@ typedef enum {
 typedef struct {
   gpr_mu mu;
   callback_phase phase;
-  int success;
   grpc_closure on_complete;
   grpc_timer alarm;
   grpc_connectivity_state state;
@@ -122,7 +121,7 @@ static void finished_completion(grpc_exec_ctx *exec_ctx, void *pw,
 }
 
 static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w,
-                        int due_to_completion) {
+                        bool due_to_completion, grpc_error *error) {
   int delete = 0;
 
   if (due_to_completion) {
@@ -130,14 +129,26 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w,
   }
 
   gpr_mu_lock(&w->mu);
+
   if (due_to_completion) {
-    w->success = 1;
+    if (grpc_trace_operation_failures) {
+      GRPC_LOG_IF_ERROR("watch_completion_error", GRPC_ERROR_REF(error));
+    }
+    GRPC_ERROR_UNREF(error);
+    error = GRPC_ERROR_NONE;
+  } else {
+    if (error == GRPC_ERROR_NONE) {
+      error =
+          GRPC_ERROR_CREATE("Timed out waiting for connection state change");
+    } else if (error == GRPC_ERROR_CANCELLED) {
+      error = GRPC_ERROR_NONE;
+    }
   }
   switch (w->phase) {
     case WAITING:
       w->phase = CALLING_BACK;
-      grpc_cq_end_op(exec_ctx, w->cq, w->tag, w->success, finished_completion,
-                     w, &w->completion_storage);
+      grpc_cq_end_op(exec_ctx, w->cq, w->tag, GRPC_ERROR_REF(error),
+                     finished_completion, w, &w->completion_storage);
       break;
     case CALLING_BACK:
       w->phase = CALLING_BACK_AND_FINISHED;
@@ -153,14 +164,18 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w,
   if (delete) {
     delete_state_watcher(exec_ctx, w);
   }
+
+  GRPC_ERROR_UNREF(error);
 }
 
-static void watch_complete(grpc_exec_ctx *exec_ctx, void *pw, bool success) {
-  partly_done(exec_ctx, pw, 1);
+static void watch_complete(grpc_exec_ctx *exec_ctx, void *pw,
+                           grpc_error *error) {
+  partly_done(exec_ctx, pw, true, GRPC_ERROR_REF(error));
 }
 
-static void timeout_complete(grpc_exec_ctx *exec_ctx, void *pw, bool success) {
-  partly_done(exec_ctx, pw, 0);
+static void timeout_complete(grpc_exec_ctx *exec_ctx, void *pw,
+                             grpc_error *error) {
+  partly_done(exec_ctx, pw, false, GRPC_ERROR_REF(error));
 }
 
 void grpc_channel_watch_connectivity_state(
@@ -174,10 +189,11 @@ void grpc_channel_watch_connectivity_state(
   GRPC_API_TRACE(
       "grpc_channel_watch_connectivity_state("
       "channel=%p, last_observed_state=%d, "
-      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %" PRId64
+      ", tv_nsec: %d, clock_type: %d }, "
       "cq=%p, tag=%p)",
-      7, (channel, (int)last_observed_state, (long long)deadline.tv_sec,
-          (int)deadline.tv_nsec, (int)deadline.clock_type, cq, tag));
+      7, (channel, (int)last_observed_state, deadline.tv_sec, deadline.tv_nsec,
+          (int)deadline.clock_type, cq, tag));
 
   grpc_cq_begin_op(cq, tag);
 
@@ -185,7 +201,6 @@ void grpc_channel_watch_connectivity_state(
   grpc_closure_init(&w->on_complete, watch_complete, w);
   w->phase = WAITING;
   w->state = last_observed_state;
-  w->success = 0;
   w->cq = cq;
   w->tag = tag;
   w->channel = channel;
diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c
index 9b5a078aec..a096435c98 100644
--- a/src/core/ext/client_config/client_channel.c
+++ b/src/core/ext/client_config/client_channel.c
@@ -117,9 +117,10 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
 static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx,
                                                   channel_data *chand,
                                                   grpc_connectivity_state state,
+                                                  grpc_error *error,
                                                   const char *reason) {
   if ((state == GRPC_CHANNEL_TRANSIENT_FAILURE ||
-       state == GRPC_CHANNEL_FATAL_FAILURE) &&
+       state == GRPC_CHANNEL_SHUTDOWN) &&
       chand->lb_policy != NULL) {
     /* cancel fail-fast picks */
     grpc_lb_policy_cancel_picks(
@@ -127,35 +128,36 @@ static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx,
         /* mask= */ GRPC_INITIAL_METADATA_IGNORE_CONNECTIVITY,
         /* check= */ 0);
   }
-  grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, reason);
+  grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, error,
+                              reason);
 }
 
-static void on_lb_policy_state_changed_locked(
-    grpc_exec_ctx *exec_ctx, lb_policy_connectivity_watcher *w) {
+static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx,
+                                              lb_policy_connectivity_watcher *w,
+                                              grpc_error *error) {
   grpc_connectivity_state publish_state = w->state;
   /* check if the notification is for a stale policy */
   if (w->lb_policy != w->chand->lb_policy) return;
 
-  if (publish_state == GRPC_CHANNEL_FATAL_FAILURE &&
-      w->chand->resolver != NULL) {
+  if (publish_state == GRPC_CHANNEL_SHUTDOWN && w->chand->resolver != NULL) {
     publish_state = GRPC_CHANNEL_TRANSIENT_FAILURE;
     grpc_resolver_channel_saw_error(exec_ctx, w->chand->resolver);
     GRPC_LB_POLICY_UNREF(exec_ctx, w->chand->lb_policy, "channel");
     w->chand->lb_policy = NULL;
   }
   set_channel_connectivity_state_locked(exec_ctx, w->chand, publish_state,
-                                        "lb_changed");
-  if (w->state != GRPC_CHANNEL_FATAL_FAILURE) {
+                                        GRPC_ERROR_REF(error), "lb_changed");
+  if (w->state != GRPC_CHANNEL_SHUTDOWN) {
     watch_lb_policy(exec_ctx, w->chand, w->lb_policy, w->state);
   }
 }
 
 static void on_lb_policy_state_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                       bool iomgr_success) {
+                                       grpc_error *error) {
   lb_policy_connectivity_watcher *w = arg;
 
   gpr_mu_lock(&w->chand->mu_config);
-  on_lb_policy_state_changed_locked(exec_ctx, w);
+  on_lb_policy_state_changed_locked(exec_ctx, w, error);
   gpr_mu_unlock(&w->chand->mu_config);
 
   GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "watch_lb_policy");
@@ -177,19 +179,22 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
 }
 
 static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                 bool iomgr_success) {
+                                 grpc_error *error) {
   channel_data *chand = arg;
   grpc_lb_policy *lb_policy = NULL;
   grpc_lb_policy *old_lb_policy;
   grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE;
   int exit_idle = 0;
+  grpc_error *state_error = GRPC_ERROR_CREATE("No load balancing policy");
 
   if (chand->incoming_configuration != NULL) {
     lb_policy = grpc_client_config_get_lb_policy(chand->incoming_configuration);
     if (lb_policy != NULL) {
       GRPC_LB_POLICY_REF(lb_policy, "channel");
       GRPC_LB_POLICY_REF(lb_policy, "config_change");
-      state = grpc_lb_policy_check_connectivity(exec_ctx, lb_policy);
+      GRPC_ERROR_UNREF(state_error);
+      state =
+          grpc_lb_policy_check_connectivity(exec_ctx, lb_policy, &state_error);
     }
 
     grpc_client_config_unref(exec_ctx, chand->incoming_configuration);
@@ -209,7 +214,9 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_exec_ctx_enqueue_list(exec_ctx, &chand->waiting_for_config_closures,
                                NULL);
   } else if (chand->resolver == NULL /* disconnected */) {
-    grpc_closure_list_fail_all(&chand->waiting_for_config_closures);
+    grpc_closure_list_fail_all(
+        &chand->waiting_for_config_closures,
+        GRPC_ERROR_CREATE_REFERENCING("Channel disconnected", &error, 1));
     grpc_exec_ctx_enqueue_list(exec_ctx, &chand->waiting_for_config_closures,
                                NULL);
   }
@@ -219,9 +226,9 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
     chand->exit_idle_when_lb_policy_arrives = 0;
   }
 
-  if (iomgr_success && chand->resolver) {
-    set_channel_connectivity_state_locked(exec_ctx, chand, state,
-                                          "new_lb+resolver");
+  if (error == GRPC_ERROR_NONE && chand->resolver) {
+    set_channel_connectivity_state_locked(
+        exec_ctx, chand, state, GRPC_ERROR_REF(state_error), "new_lb+resolver");
     if (lb_policy != NULL) {
       watch_lb_policy(exec_ctx, chand, lb_policy, state);
     }
@@ -236,8 +243,12 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
       GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
       chand->resolver = NULL;
     }
+    grpc_error *refs[] = {error, state_error};
     set_channel_connectivity_state_locked(
-        exec_ctx, chand, GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone");
+        exec_ctx, chand, GRPC_CHANNEL_SHUTDOWN,
+        GRPC_ERROR_CREATE_REFERENCING("Got config after disconnection", refs,
+                                      GPR_ARRAY_SIZE(refs)),
+        "resolver_gone");
     gpr_mu_unlock(&chand->mu_config);
   }
 
@@ -257,6 +268,7 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
   }
 
   GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->owning_stack, "resolver");
+  GRPC_ERROR_UNREF(state_error);
 }
 
 static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
@@ -264,7 +276,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                   grpc_transport_op *op) {
   channel_data *chand = elem->channel_data;
 
-  grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
 
   GPR_ASSERT(op->set_accept_stream == false);
   if (op->bind_pollset != NULL) {
@@ -283,7 +295,9 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
 
   if (op->send_ping != NULL) {
     if (chand->lb_policy == NULL) {
-      grpc_exec_ctx_enqueue(exec_ctx, op->send_ping, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, op->send_ping,
+                          GRPC_ERROR_CREATE("Ping with no load balancing"),
+                          NULL);
     } else {
       grpc_lb_policy_ping_one(exec_ctx, chand->lb_policy, op->send_ping);
       op->bind_pollset = NULL;
@@ -291,24 +305,29 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
     op->send_ping = NULL;
   }
 
-  if (op->disconnect && chand->resolver != NULL) {
-    set_channel_connectivity_state_locked(
-        exec_ctx, chand, GRPC_CHANNEL_FATAL_FAILURE, "disconnect");
-    grpc_resolver_shutdown(exec_ctx, chand->resolver);
-    GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
-    chand->resolver = NULL;
-    if (!chand->started_resolving) {
-      grpc_closure_list_fail_all(&chand->waiting_for_config_closures);
-      grpc_exec_ctx_enqueue_list(exec_ctx, &chand->waiting_for_config_closures,
-                                 NULL);
-    }
-    if (chand->lb_policy != NULL) {
-      grpc_pollset_set_del_pollset_set(exec_ctx,
-                                       chand->lb_policy->interested_parties,
-                                       chand->interested_parties);
-      GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
-      chand->lb_policy = NULL;
+  if (op->disconnect_with_error != GRPC_ERROR_NONE) {
+    if (chand->resolver != NULL) {
+      set_channel_connectivity_state_locked(
+          exec_ctx, chand, GRPC_CHANNEL_SHUTDOWN,
+          GRPC_ERROR_REF(op->disconnect_with_error), "disconnect");
+      grpc_resolver_shutdown(exec_ctx, chand->resolver);
+      GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
+      chand->resolver = NULL;
+      if (!chand->started_resolving) {
+        grpc_closure_list_fail_all(&chand->waiting_for_config_closures,
+                                   GRPC_ERROR_REF(op->disconnect_with_error));
+        grpc_exec_ctx_enqueue_list(exec_ctx,
+                                   &chand->waiting_for_config_closures, NULL);
+      }
+      if (chand->lb_policy != NULL) {
+        grpc_pollset_set_del_pollset_set(exec_ctx,
+                                         chand->lb_policy->interested_parties,
+                                         chand->interested_parties);
+        GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel");
+        chand->lb_policy = NULL;
+      }
     }
+    GRPC_ERROR_UNREF(op->disconnect_with_error);
   }
   gpr_mu_unlock(&chand->mu_config);
 }
@@ -328,16 +347,17 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg,
                               grpc_connected_subchannel **connected_subchannel,
                               grpc_closure *on_ready);
 
-static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
+                             grpc_error *error) {
   continue_picking_args *cpa = arg;
   if (cpa->connected_subchannel == NULL) {
     /* cancelled, do nothing */
-  } else if (!success) {
-    grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, false, NULL);
+  } else if (error != GRPC_ERROR_NONE) {
+    grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error), NULL);
   } else if (cc_pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
                                 cpa->initial_metadata_flags,
                                 cpa->connected_subchannel, cpa->on_ready)) {
-    grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE, NULL);
   }
   gpr_free(cpa);
 }
@@ -347,6 +367,8 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
                               uint32_t initial_metadata_flags,
                               grpc_connected_subchannel **connected_subchannel,
                               grpc_closure *on_ready) {
+  GPR_TIMER_BEGIN("cc_pick_subchannel", 0);
+
   grpc_call_element *elem = elemp;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
@@ -362,14 +384,16 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
                                  connected_subchannel);
     }
     for (closure = chand->waiting_for_config_closures.head; closure != NULL;
-         closure = grpc_closure_next(closure)) {
+         closure = closure->next_data.next) {
       cpa = closure->cb_arg;
       if (cpa->connected_subchannel == connected_subchannel) {
         cpa->connected_subchannel = NULL;
-        grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, false, NULL);
+        grpc_exec_ctx_sched(exec_ctx, cpa->on_ready,
+                            GRPC_ERROR_CREATE("Pick cancelled"), NULL);
       }
     }
     gpr_mu_unlock(&chand->mu_config);
+    GPR_TIMER_END("cc_pick_subchannel", 0);
     return 1;
   }
   if (chand->lb_policy != NULL) {
@@ -377,10 +401,11 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
     int r;
     GRPC_LB_POLICY_REF(lb_policy, "cc_pick_subchannel");
     gpr_mu_unlock(&chand->mu_config);
-    r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollset,
+    r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollent,
                             initial_metadata, initial_metadata_flags,
                             connected_subchannel, on_ready);
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "cc_pick_subchannel");
+    GPR_TIMER_END("cc_pick_subchannel", 0);
     return r;
   }
   if (chand->resolver != NULL && !chand->started_resolving) {
@@ -398,12 +423,15 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp,
     cpa->on_ready = on_ready;
     cpa->elem = elem;
     grpc_closure_init(&cpa->closure, continue_picking, cpa);
-    grpc_closure_list_add(&chand->waiting_for_config_closures, &cpa->closure,
-                          1);
+    grpc_closure_list_append(&chand->waiting_for_config_closures, &cpa->closure,
+                             GRPC_ERROR_NONE);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, on_ready, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected"),
+                        NULL);
   }
   gpr_mu_unlock(&chand->mu_config);
+
+  GPR_TIMER_END("cc_pick_subchannel", 0);
   return 0;
 }
 
@@ -416,6 +444,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats,
                               void *and_free_memory) {
   grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data);
   gpr_free(and_free_memory);
@@ -461,10 +490,11 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   gpr_mu_destroy(&chand->mu_config);
 }
 
-static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                           grpc_pollset *pollset) {
+static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                          grpc_call_element *elem,
+                                          grpc_polling_entity *pollent) {
   call_data *calld = elem->call_data;
-  calld->pollset = pollset;
+  calld->pollent = pollent;
 }
 
 const grpc_channel_filter grpc_client_channel_filter = {
@@ -472,7 +502,7 @@ const grpc_channel_filter grpc_client_channel_filter = {
     cc_start_transport_op,
     sizeof(call_data),
     init_call_elem,
-    cc_set_pollset,
+    cc_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
@@ -506,7 +536,7 @@ grpc_connectivity_state grpc_client_channel_check_connectivity_state(
   channel_data *chand = elem->channel_data;
   grpc_connectivity_state out;
   gpr_mu_lock(&chand->mu_config);
-  out = grpc_connectivity_state_check(&chand->state_tracker);
+  out = grpc_connectivity_state_check(&chand->state_tracker, NULL);
   if (out == GRPC_CHANNEL_IDLE && try_to_connect) {
     if (chand->lb_policy != NULL) {
       grpc_lb_policy_exit_idle(exec_ctx, chand->lb_policy);
@@ -533,7 +563,7 @@ typedef struct {
 } external_connectivity_watcher;
 
 static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg,
-                                       bool iomgr_success) {
+                                       grpc_error *error) {
   external_connectivity_watcher *w = arg;
   grpc_closure *follow_up = w->on_complete;
   grpc_pollset_set_del_pollset(exec_ctx, w->chand->interested_parties,
@@ -541,7 +571,7 @@ static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg,
   GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack,
                            "external_connectivity_watcher");
   gpr_free(w);
-  follow_up->cb(exec_ctx, follow_up->cb_arg, iomgr_success);
+  follow_up->cb(exec_ctx, follow_up->cb_arg, error);
 }
 
 void grpc_client_channel_watch_connectivity_state(
diff --git a/src/core/ext/client_config/connector.h b/src/core/ext/client_config/connector.h
index dd85dfcb7d..ea9d23706e 100644
--- a/src/core/ext/client_config/connector.h
+++ b/src/core/ext/client_config/connector.h
@@ -64,7 +64,7 @@ typedef struct {
   grpc_transport *transport;
 
   /** channel arguments (to be passed to the filters) */
-  const grpc_channel_args *channel_args;
+  grpc_channel_args *channel_args;
 } grpc_connect_out_args;
 
 struct grpc_connector_vtable {
diff --git a/src/core/ext/client_config/lb_policy.c b/src/core/ext/client_config/lb_policy.c
index a7ad9842dc..8b980b2cca 100644
--- a/src/core/ext/client_config/lb_policy.c
+++ b/src/core/ext/client_config/lb_policy.c
@@ -60,8 +60,9 @@ static gpr_atm ref_mutate(grpc_lb_policy *c, gpr_atm delta,
                             : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
 #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-          "LB_POLICY: %p % 12s 0x%08x -> 0x%08x [%s]", c, purpose, old_val,
-          old_val + delta, reason);
+          "LB_POLICY: 0x%" PRIxPTR " %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR
+          " [%s]",
+          (intptr_t)c, purpose, old_val, old_val + delta, reason);
 #endif
   return old_val;
 }
@@ -99,12 +100,12 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx,
 }
 
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                        grpc_pollset *pollset,
+                        grpc_polling_entity *pollent,
                         grpc_metadata_batch *initial_metadata,
                         uint32_t initial_metadata_flags,
                         grpc_connected_subchannel **target,
                         grpc_closure *on_complete) {
-  return policy->vtable->pick(exec_ctx, policy, pollset, initial_metadata,
+  return policy->vtable->pick(exec_ctx, policy, pollent, initial_metadata,
                               initial_metadata_flags, target, on_complete);
 }
 
@@ -138,6 +139,8 @@ void grpc_lb_policy_notify_on_state_change(grpc_exec_ctx *exec_ctx,
 }
 
 grpc_connectivity_state grpc_lb_policy_check_connectivity(
-    grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy) {
-  return policy->vtable->check_connectivity(exec_ctx, policy);
+    grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+    grpc_error **connectivity_error) {
+  return policy->vtable->check_connectivity(exec_ctx, policy,
+                                            connectivity_error);
 }
diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h
index 0384e0b2eb..3cfd041d3a 100644
--- a/src/core/ext/client_config/lb_policy.h
+++ b/src/core/ext/client_config/lb_policy.h
@@ -35,6 +35,7 @@
 #define GRPC_CORE_EXT_CLIENT_CONFIG_LB_POLICY_H
 
 #include "src/core/ext/client_config/subchannel.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /** A load balancing policy: specified by a vtable and a struct (which
@@ -59,7 +60,8 @@ struct grpc_lb_policy_vtable {
 
   /** implement grpc_lb_policy_pick */
   int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-              grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
+              grpc_polling_entity *pollent,
+              grpc_metadata_batch *initial_metadata,
               uint32_t initial_metadata_flags,
               grpc_connected_subchannel **target, grpc_closure *on_complete);
   void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
@@ -75,8 +77,9 @@ struct grpc_lb_policy_vtable {
   void (*exit_idle)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 
   /** check the current connectivity of the lb_policy */
-  grpc_connectivity_state (*check_connectivity)(grpc_exec_ctx *exec_ctx,
-                                                grpc_lb_policy *policy);
+  grpc_connectivity_state (*check_connectivity)(
+      grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+      grpc_error **connectivity_error);
 
   /** call notify when the connectivity state of a channel changes from *state.
       Updates *state with the new state of the policy */
@@ -124,7 +127,7 @@ void grpc_lb_policy_init(grpc_lb_policy *policy,
     \a target.
     Picking can be asynchronous. Any IO should be done under \a pollset. */
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                        grpc_pollset *pollset,
+                        grpc_polling_entity *pollent,
                         grpc_metadata_batch *initial_metadata,
                         uint32_t initial_metadata_flags,
                         grpc_connected_subchannel **target,
@@ -152,6 +155,7 @@ void grpc_lb_policy_notify_on_state_change(grpc_exec_ctx *exec_ctx,
                                            grpc_closure *closure);
 
 grpc_connectivity_state grpc_lb_policy_check_connectivity(
-    grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
+    grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
+    grpc_error **connectivity_error);
 
 #endif /* GRPC_CORE_EXT_CLIENT_CONFIG_LB_POLICY_H */
diff --git a/src/core/ext/client_config/subchannel.c b/src/core/ext/client_config/subchannel.c
index bd45d3825c..d089cd4399 100644
--- a/src/core/ext/client_config/subchannel.c
+++ b/src/core/ext/client_config/subchannel.c
@@ -54,7 +54,7 @@
 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1))
 
 #define GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS 20
-#define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 2
+#define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 1
 #define GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER 1.6
 #define GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_SUBCHANNEL_RECONNECT_JITTER 0.2
@@ -147,7 +147,7 @@ struct grpc_subchannel_call {
   (((grpc_subchannel_call *)(callstack)) - 1)
 
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *subchannel,
-                                 bool iomgr_success);
+                                 grpc_error *error);
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 #define REF_REASON reason
@@ -177,7 +177,7 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *subchannel,
  */
 
 static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg,
-                               bool success) {
+                               grpc_error *error) {
   grpc_connected_subchannel *c = arg;
   grpc_channel_stack_destroy(exec_ctx, CHANNEL_STACK_FROM_CONNECTION(c));
   gpr_free(c);
@@ -200,7 +200,7 @@ void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx,
  */
 
 static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg,
-                               bool success) {
+                               grpc_error *error) {
   grpc_subchannel *c = arg;
   gpr_free((void *)c->filters);
   grpc_channel_args_destroy(c->args);
@@ -290,8 +290,8 @@ void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
   gpr_atm old_refs;
   old_refs = ref_mutate(c, -(gpr_atm)1, 1 REF_MUTATE_PURPOSE("WEAK_UNREF"));
   if (old_refs == 1) {
-    grpc_exec_ctx_enqueue(exec_ctx, grpc_closure_create(subchannel_destroy, c),
-                          true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(subchannel_destroy, c),
+                        GRPC_ERROR_NONE, NULL);
   }
 }
 
@@ -320,7 +320,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
     c->filters = NULL;
   }
   c->addr = gpr_malloc(args->addr_len);
-  memcpy(c->addr, args->addr, args->addr_len);
+  if (args->addr_len) memcpy(c->addr, args->addr, args->addr_len);
   c->pollset_set = grpc_pollset_set_create();
   c->addr_len = args->addr_len;
   grpc_set_initial_connect_string(&c->addr, &c->addr_len,
@@ -382,7 +382,8 @@ static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   args.initial_connect_string = c->initial_connect_string;
 
   grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
-                              GRPC_CHANNEL_CONNECTING, "state_change");
+                              GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+                              "state_change");
   grpc_connector_connect(exec_ctx, c->connector, &args, &c->connecting_result,
                          &c->connected);
 }
@@ -393,16 +394,17 @@ static void start_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   continue_connect(exec_ctx, c);
 }
 
-grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel *c) {
+grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel *c,
+                                                           grpc_error **error) {
   grpc_connectivity_state state;
   gpr_mu_lock(&c->mu);
-  state = grpc_connectivity_state_check(&c->state_tracker);
+  state = grpc_connectivity_state_check(&c->state_tracker, error);
   gpr_mu_unlock(&c->mu);
   return state;
 }
 
 static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg,
-                                           bool success) {
+                                           grpc_error *error) {
   external_state_watcher *w = arg;
   grpc_closure *follow_up = w->notify;
   if (w->pollset_set != NULL) {
@@ -415,7 +417,7 @@ static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_mu_unlock(&w->subchannel->mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, w->subchannel, "external_state_watcher");
   gpr_free(w);
-  follow_up->cb(exec_ctx, follow_up->cb_arg, success);
+  follow_up->cb(exec_ctx, follow_up->cb_arg, error);
 }
 
 void grpc_subchannel_notify_on_state_change(
@@ -469,7 +471,7 @@ void grpc_connected_subchannel_process_transport_op(
 }
 
 static void subchannel_on_child_state_changed(grpc_exec_ctx *exec_ctx, void *p,
-                                              bool iomgr_success) {
+                                              grpc_error *error) {
   state_watcher *sw = p;
   grpc_subchannel *c = sw->subchannel;
   gpr_mu *mu = &c->mu;
@@ -477,20 +479,19 @@ static void subchannel_on_child_state_changed(grpc_exec_ctx *exec_ctx, void *p,
   gpr_mu_lock(mu);
 
   /* if we failed just leave this closure */
-  if (iomgr_success) {
-    if (sw->connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-      /* any errors on a subchannel ==> we're done, create a new one */
-      sw->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE;
-    }
-    grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
-                                sw->connectivity_state, "reflect_child");
-    if (sw->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) {
-      grpc_connected_subchannel_notify_on_state_change(
-          exec_ctx, GET_CONNECTED_SUBCHANNEL(c, no_barrier), NULL,
-          &sw->connectivity_state, &sw->closure);
-      GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
-      sw = NULL;
-    }
+  if (sw->connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    /* any errors on a subchannel ==> we're done, create a new one */
+    sw->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+  }
+  grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
+                              sw->connectivity_state, GRPC_ERROR_REF(error),
+                              "reflect_child");
+  if (sw->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
+    grpc_connected_subchannel_notify_on_state_change(
+        exec_ctx, GET_CONNECTED_SUBCHANNEL(c, no_barrier), NULL,
+        &sw->connectivity_state, &sw->closure);
+    GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
+    sw = NULL;
   }
 
   gpr_mu_unlock(mu);
@@ -592,17 +593,20 @@ static void publish_transport_locked(grpc_exec_ctx *exec_ctx,
 
   /* signal completion */
   grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_READY,
-                              "connected");
+                              GRPC_ERROR_NONE, "connected");
 }
 
-static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) {
+static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_subchannel *c = arg;
   gpr_mu_lock(&c->mu);
   c->have_alarm = 0;
   if (c->disconnected) {
-    iomgr_success = 0;
+    error = GRPC_ERROR_CREATE_REFERENCING("Disconnected", &error, 1);
+  } else {
+    GRPC_ERROR_REF(error);
   }
-  if (iomgr_success) {
+  if (error == GRPC_ERROR_NONE) {
+    gpr_log(GPR_INFO, "Failed to connect to channel, retrying");
     c->next_attempt =
         gpr_backoff_step(&c->backoff_state, gpr_now(GPR_CLOCK_MONOTONIC));
     continue_connect(exec_ctx, c);
@@ -611,11 +615,13 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) {
     gpr_mu_unlock(&c->mu);
     GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   }
+  GRPC_ERROR_UNREF(error);
 }
 
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
-                                 bool iomgr_success) {
+                                 grpc_error *error) {
   grpc_subchannel *c = arg;
+  grpc_channel_args *delete_channel_args = c->connecting_result.channel_args;
 
   GRPC_SUBCHANNEL_WEAK_REF(c, "connected");
   gpr_mu_lock(&c->mu);
@@ -627,13 +633,26 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
     gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
     GPR_ASSERT(!c->have_alarm);
     c->have_alarm = 1;
-    grpc_connectivity_state_set(exec_ctx, &c->state_tracker,
-                                GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                "connect_failed");
+    grpc_connectivity_state_set(
+        exec_ctx, &c->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
+        GRPC_ERROR_CREATE_REFERENCING("Connect Failed", &error, 1),
+        "connect_failed");
+    gpr_timespec time_til_next = gpr_time_sub(c->next_attempt, now);
+    const char *errmsg = grpc_error_string(error);
+    gpr_log(GPR_INFO, "Connect failed: %s", errmsg);
+    if (gpr_time_cmp(time_til_next, gpr_time_0(time_til_next.clock_type)) <=
+        0) {
+      gpr_log(GPR_INFO, "Retry immediately");
+    } else {
+      gpr_log(GPR_INFO, "Retry in %" PRId64 ".%09d seconds",
+              time_til_next.tv_sec, time_til_next.tv_nsec);
+    }
     grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now);
+    grpc_error_free_string(errmsg);
   }
   gpr_mu_unlock(&c->mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
+  grpc_channel_args_destroy(delete_channel_args);
 }
 
 /*
@@ -641,11 +660,11 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
  */
 
 static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call,
-                                    bool success) {
+                                    grpc_error *error) {
   grpc_subchannel_call *c = call;
   GPR_TIMER_BEGIN("grpc_subchannel_call_unref.destroy", 0);
   grpc_connected_subchannel *connection = c->connection;
-  grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), c);
+  grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), NULL, c);
   GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, connection, "subchannel_call");
   GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0);
 }
@@ -671,9 +690,11 @@ char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx,
 void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
                                      grpc_subchannel_call *call,
                                      grpc_transport_stream_op *op) {
+  GPR_TIMER_BEGIN("grpc_subchannel_call_process_op", 0);
   grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
   grpc_call_element *top_elem = grpc_call_stack_element(call_stack, 0);
   top_elem->filter->start_transport_stream_op(exec_ctx, top_elem, op);
+  GPR_TIMER_END("grpc_subchannel_call_process_op", 0);
 }
 
 grpc_connected_subchannel *grpc_subchannel_get_connected_subchannel(
@@ -683,7 +704,7 @@ grpc_connected_subchannel *grpc_subchannel_get_connected_subchannel(
 
 grpc_subchannel_call *grpc_connected_subchannel_create_call(
     grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *con,
-    grpc_pollset *pollset) {
+    grpc_polling_entity *pollent) {
   grpc_channel_stack *chanstk = CHANNEL_STACK_FROM_CONNECTION(con);
   grpc_subchannel_call *call =
       gpr_malloc(sizeof(grpc_subchannel_call) + chanstk->call_stack_size);
@@ -692,7 +713,7 @@ grpc_subchannel_call *grpc_connected_subchannel_create_call(
   GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call");
   grpc_call_stack_init(exec_ctx, chanstk, 1, subchannel_call_destroy, call,
                        NULL, NULL, callstk);
-  grpc_call_stack_set_pollset(exec_ctx, callstk, pollset);
+  grpc_call_stack_set_pollset_or_pollset_set(exec_ctx, callstk, pollent);
   return call;
 }
 
diff --git a/src/core/ext/client_config/subchannel.h b/src/core/ext/client_config/subchannel.h
index 0765a544e8..b6d39f5dc5 100644
--- a/src/core/ext/client_config/subchannel.h
+++ b/src/core/ext/client_config/subchannel.h
@@ -36,6 +36,7 @@
 
 #include "src/core/ext/client_config/connector.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /** A (sub-)channel that knows how to connect to exactly one target
@@ -109,7 +110,7 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
 /** construct a subchannel call */
 grpc_subchannel_call *grpc_connected_subchannel_create_call(
     grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *connected_subchannel,
-    grpc_pollset *pollset);
+    grpc_polling_entity *pollent);
 
 /** process a transport level op */
 void grpc_connected_subchannel_process_transport_op(
@@ -118,7 +119,7 @@ void grpc_connected_subchannel_process_transport_op(
 
 /** poll the current connectivity state of a channel */
 grpc_connectivity_state grpc_subchannel_check_connectivity(
-    grpc_subchannel *channel);
+    grpc_subchannel *channel, grpc_error **error);
 
 /** call notify when the connectivity state of a channel changes from *state.
     Updates *state with the new state of the channel */
diff --git a/src/core/ext/client_config/subchannel_call_holder.c b/src/core/ext/client_config/subchannel_call_holder.c
index 9918fbdcb4..b96a0ad093 100644
--- a/src/core/ext/client_config/subchannel_call_holder.c
+++ b/src/core/ext/client_config/subchannel_call_holder.c
@@ -43,14 +43,14 @@
 #define CANCELLED_CALL ((grpc_subchannel_call *)1)
 
 static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *holder,
-                             bool success);
+                             grpc_error *error);
 static void retry_ops(grpc_exec_ctx *exec_ctx, void *retry_ops_args,
-                      bool success);
+                      grpc_error *error);
 
 static void add_waiting_locked(grpc_subchannel_call_holder *holder,
                                grpc_transport_stream_op *op);
 static void fail_locked(grpc_exec_ctx *exec_ctx,
-                        grpc_subchannel_call_holder *holder);
+                        grpc_subchannel_call_holder *holder, grpc_error *error);
 static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
                                  grpc_subchannel_call_holder *holder);
 
@@ -68,6 +68,7 @@ void grpc_subchannel_call_holder_init(
   holder->waiting_ops_capacity = 0;
   holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
   holder->owning_call = owning_call;
+  holder->pollent = NULL;
 }
 
 void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx,
@@ -90,7 +91,8 @@ void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx,
   grpc_subchannel_call *call = GET_CALL(holder);
   GPR_TIMER_BEGIN("grpc_subchannel_call_holder_perform_op", 0);
   if (call == CANCELLED_CALL) {
-    grpc_transport_stream_op_finish_with_failure(exec_ctx, op);
+    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                 GRPC_ERROR_CANCELLED);
     GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
     return;
   }
@@ -106,7 +108,8 @@ retry:
   call = GET_CALL(holder);
   if (call == CANCELLED_CALL) {
     gpr_mu_unlock(&holder->mu);
-    grpc_transport_stream_op_finish_with_failure(exec_ctx, op);
+    grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                 GRPC_ERROR_CANCELLED);
     GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
     return;
   }
@@ -117,13 +120,13 @@ retry:
     return;
   }
   /* if this is a cancellation, then we can raise our cancelled flag */
-  if (op->cancel_with_status != GRPC_STATUS_OK) {
+  if (op->cancel_error != GRPC_ERROR_NONE) {
     if (!gpr_atm_rel_cas(&holder->subchannel_call, 0, 1)) {
       goto retry;
     } else {
       switch (holder->creation_phase) {
         case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING:
-          fail_locked(exec_ctx, holder);
+          fail_locked(exec_ctx, holder, GRPC_ERROR_REF(op->cancel_error));
           break;
         case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL:
           holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, NULL,
@@ -131,7 +134,8 @@ retry:
           break;
       }
       gpr_mu_unlock(&holder->mu);
-      grpc_transport_stream_op_finish_with_failure(exec_ctx, op);
+      grpc_transport_stream_op_finish_with_failure(exec_ctx, op,
+                                                   GRPC_ERROR_CANCELLED);
       GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
       return;
     }
@@ -157,7 +161,7 @@ retry:
     gpr_atm_rel_store(
         &holder->subchannel_call,
         (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
-            exec_ctx, holder->connected_subchannel, holder->pollset));
+            exec_ctx, holder->connected_subchannel, holder->pollent));
     retry_waiting_locked(exec_ctx, holder);
     goto retry;
   }
@@ -167,22 +171,28 @@ retry:
   GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0);
 }
 
-static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
+                             grpc_error *error) {
   grpc_subchannel_call_holder *holder = arg;
   gpr_mu_lock(&holder->mu);
   GPR_ASSERT(holder->creation_phase ==
              GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL);
   holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
   if (holder->connected_subchannel == NULL) {
-    fail_locked(exec_ctx, holder);
+    gpr_atm_no_barrier_store(&holder->subchannel_call, 1);
+    fail_locked(exec_ctx, holder,
+                GRPC_ERROR_CREATE_REFERENCING("Failed to create subchannel",
+                                              &error, 1));
   } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) {
     /* already cancelled before subchannel became ready */
-    fail_locked(exec_ctx, holder);
+    fail_locked(exec_ctx, holder,
+                GRPC_ERROR_CREATE_REFERENCING(
+                    "Cancelled before creating subchannel", &error, 1));
   } else {
     gpr_atm_rel_store(
         &holder->subchannel_call,
         (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
-            exec_ctx, holder->connected_subchannel, holder->pollset));
+            exec_ctx, holder->connected_subchannel, holder->pollent));
     retry_waiting_locked(exec_ctx, holder);
   }
   gpr_mu_unlock(&holder->mu);
@@ -203,18 +213,18 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx,
   a->call = GET_CALL(holder);
   if (a->call == CANCELLED_CALL) {
     gpr_free(a);
-    fail_locked(exec_ctx, holder);
+    fail_locked(exec_ctx, holder, GRPC_ERROR_CANCELLED);
     return;
   }
   holder->waiting_ops = NULL;
   holder->waiting_ops_count = 0;
   holder->waiting_ops_capacity = 0;
   GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
-  grpc_exec_ctx_enqueue(exec_ctx, grpc_closure_create(retry_ops, a), true,
-                        NULL);
+  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
+                      GRPC_ERROR_NONE, NULL);
 }
 
-static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, bool success) {
+static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
   retry_ops_args *a = args;
   size_t i;
   for (i = 0; i < a->nops; i++) {
@@ -239,13 +249,15 @@ static void add_waiting_locked(grpc_subchannel_call_holder *holder,
 }
 
 static void fail_locked(grpc_exec_ctx *exec_ctx,
-                        grpc_subchannel_call_holder *holder) {
+                        grpc_subchannel_call_holder *holder,
+                        grpc_error *error) {
   size_t i;
   for (i = 0; i < holder->waiting_ops_count; i++) {
-    grpc_transport_stream_op_finish_with_failure(exec_ctx,
-                                                 &holder->waiting_ops[i]);
+    grpc_transport_stream_op_finish_with_failure(
+        exec_ctx, &holder->waiting_ops[i], GRPC_ERROR_REF(error));
   }
   holder->waiting_ops_count = 0;
+  GRPC_ERROR_UNREF(error);
 }
 
 char *grpc_subchannel_call_holder_get_peer(
diff --git a/src/core/ext/client_config/subchannel_call_holder.h b/src/core/ext/client_config/subchannel_call_holder.h
index 9299908788..8d2deb02f3 100644
--- a/src/core/ext/client_config/subchannel_call_holder.h
+++ b/src/core/ext/client_config/subchannel_call_holder.h
@@ -35,6 +35,7 @@
 #define GRPC_CORE_EXT_CLIENT_CONFIG_SUBCHANNEL_CALL_HOLDER_H
 
 #include "src/core/ext/client_config/subchannel.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 
 /** Pick a subchannel for grpc_subchannel_call_holder;
     Return 1 if subchannel is available immediately (in which case on_ready
@@ -71,7 +72,7 @@ typedef struct grpc_subchannel_call_holder {
 
   grpc_subchannel_call_holder_creation_phase creation_phase;
   grpc_connected_subchannel *connected_subchannel;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
 
   grpc_transport_stream_op *waiting_ops;
   size_t waiting_ops_count;
diff --git a/src/core/ext/client_config/subchannel_index.c b/src/core/ext/client_config/subchannel_index.c
index ab8d9bd91d..690cb16b96 100644
--- a/src/core/ext/client_config/subchannel_index.c
+++ b/src/core/ext/client_config/subchannel_index.c
@@ -77,12 +77,19 @@ static grpc_subchannel_key *create_key(
   grpc_subchannel_key *k = gpr_malloc(sizeof(*k));
   k->connector = grpc_connector_ref(connector);
   k->args.filter_count = args->filter_count;
-  k->args.filters = gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
-  memcpy((grpc_channel_filter *)k->args.filters, args->filters,
-         sizeof(*k->args.filters) * k->args.filter_count);
+  if (k->args.filter_count > 0) {
+    k->args.filters =
+        gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
+    memcpy((grpc_channel_filter *)k->args.filters, args->filters,
+           sizeof(*k->args.filters) * k->args.filter_count);
+  } else {
+    k->args.filters = NULL;
+  }
   k->args.addr_len = args->addr_len;
   k->args.addr = gpr_malloc(args->addr_len);
-  memcpy(k->args.addr, args->addr, k->args.addr_len);
+  if (k->args.addr_len > 0) {
+    memcpy(k->args.addr, args->addr, k->args.addr_len);
+  }
   k->args.args = copy_channel_args(args->args);
   return k;
 }
@@ -104,11 +111,15 @@ static int subchannel_key_compare(grpc_subchannel_key *a,
   if (c != 0) return c;
   c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
   if (c != 0) return c;
-  c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
-  if (c != 0) return c;
-  c = memcmp(a->args.filters, b->args.filters,
-             a->args.filter_count * sizeof(*a->args.filters));
-  if (c != 0) return c;
+  if (a->args.addr_len) {
+    c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
+    if (c != 0) return c;
+  }
+  if (a->args.filter_count > 0) {
+    c = memcmp(a->args.filters, b->args.filters,
+               a->args.filter_count * sizeof(*a->args.filters));
+    if (c != 0) return c;
+  }
   return grpc_channel_args_compare(a->args.args, b->args.args);
 }
 
diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/lb_policy/grpclb/load_balancer_api.c
index 459d6d9954..59b89997dd 100644
--- a/src/core/ext/lb_policy/grpclb/load_balancer_api.c
+++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.c
@@ -50,7 +50,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field,
   decode_serverlist_arg *dec_arg = *arg;
   if (dec_arg->first_pass != 0) { /* first pass */
     grpc_grpclb_server server;
-    if (!pb_decode(stream, grpc_lb_v0_Server_fields, &server)) {
+    if (!pb_decode(stream, grpc_lb_v1_Server_fields, &server)) {
       return false;
     }
     dec_arg->num_servers++;
@@ -61,7 +61,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field,
       dec_arg->servers =
           gpr_malloc(sizeof(grpc_grpclb_server *) * dec_arg->num_servers);
     }
-    if (!pb_decode(stream, grpc_lb_v0_Server_fields, server)) {
+    if (!pb_decode(stream, grpc_lb_v1_Server_fields, server)) {
       return false;
     }
     dec_arg->servers[dec_arg->i++] = server;
@@ -87,13 +87,13 @@ gpr_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) {
   pb_ostream_t outputstream;
   gpr_slice slice;
   memset(&sizestream, 0, sizeof(pb_ostream_t));
-  pb_encode(&sizestream, grpc_lb_v0_LoadBalanceRequest_fields, request);
+  pb_encode(&sizestream, grpc_lb_v1_LoadBalanceRequest_fields, request);
   encoded_length = sizestream.bytes_written;
 
   slice = gpr_slice_malloc(encoded_length);
   outputstream =
       pb_ostream_from_buffer(GPR_SLICE_START_PTR(slice), encoded_length);
-  GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v0_LoadBalanceRequest_fields,
+  GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v1_LoadBalanceRequest_fields,
                        request) != 0);
   return slice;
 }
@@ -109,7 +109,7 @@ grpc_grpclb_response *grpc_grpclb_response_parse(gpr_slice encoded_response) {
                              GPR_SLICE_LENGTH(encoded_response));
   grpc_grpclb_response *res = gpr_malloc(sizeof(grpc_grpclb_response));
   memset(res, 0, sizeof(*res));
-  status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res);
+  status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, res);
   if (!status) {
     grpc_grpclb_response_destroy(res);
     return NULL;
@@ -132,7 +132,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist(
   res->server_list.servers.funcs.decode = decode_serverlist;
   res->server_list.servers.arg = &arg;
   arg.first_pass = 1;
-  status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res);
+  status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, res);
   if (!status) {
     grpc_grpclb_response_destroy(res);
     return NULL;
@@ -140,7 +140,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist(
 
   arg.first_pass = 0;
   status =
-      pb_decode(&stream_at_start, grpc_lb_v0_LoadBalanceResponse_fields, res);
+      pb_decode(&stream_at_start, grpc_lb_v1_LoadBalanceResponse_fields, res);
   if (!status) {
     grpc_grpclb_response_destroy(res);
     return NULL;
diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.h b/src/core/ext/lb_policy/grpclb/load_balancer_api.h
index 968f7d278a..71b5616d0c 100644
--- a/src/core/ext/lb_policy/grpclb/load_balancer_api.h
+++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.h
@@ -37,7 +37,7 @@
 #include <grpc/support/slice_buffer.h>
 
 #include "src/core/ext/client_config/lb_policy_factory.h"
-#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h"
+#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -45,10 +45,10 @@ extern "C" {
 
 #define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128
 
-typedef grpc_lb_v0_LoadBalanceRequest grpc_grpclb_request;
-typedef grpc_lb_v0_LoadBalanceResponse grpc_grpclb_response;
-typedef grpc_lb_v0_Server grpc_grpclb_server;
-typedef grpc_lb_v0_Duration grpc_grpclb_duration;
+typedef grpc_lb_v1_LoadBalanceRequest grpc_grpclb_request;
+typedef grpc_lb_v1_LoadBalanceResponse grpc_grpclb_response;
+typedef grpc_lb_v1_Server grpc_grpclb_server;
+typedef grpc_lb_v1_Duration grpc_grpclb_duration;
 typedef struct grpc_grpclb_serverlist {
   grpc_grpclb_server **servers;
   size_t num_servers;
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h
deleted file mode 100644
index 3599f881bb..0000000000
--- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- *
- * Copyright 2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-/* Automatically generated nanopb header */
-/* Generated by nanopb-0.3.5-dev */
-
-#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED
-#define PB_LOAD_BALANCER_PB_H_INCLUDED
-#include "third_party/nanopb/pb.h"
-#if PB_PROTO_HEADER_VERSION != 30
-#error Regenerate this file with the current version of nanopb generator.
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Struct definitions */
-typedef struct _grpc_lb_v0_ClientStats {
-    bool has_total_requests;
-    int64_t total_requests;
-    bool has_client_rpc_errors;
-    int64_t client_rpc_errors;
-    bool has_dropped_requests;
-    int64_t dropped_requests;
-} grpc_lb_v0_ClientStats;
-
-typedef struct _grpc_lb_v0_Duration {
-    bool has_seconds;
-    int64_t seconds;
-    bool has_nanos;
-    int32_t nanos;
-} grpc_lb_v0_Duration;
-
-typedef struct _grpc_lb_v0_InitialLoadBalanceRequest {
-    bool has_name;
-    char name[128];
-} grpc_lb_v0_InitialLoadBalanceRequest;
-
-typedef PB_BYTES_ARRAY_T(64) grpc_lb_v0_Server_load_balance_token_t;
-typedef struct _grpc_lb_v0_Server {
-    bool has_ip_address;
-    char ip_address[46];
-    bool has_port;
-    int32_t port;
-    bool has_load_balance_token;
-    grpc_lb_v0_Server_load_balance_token_t load_balance_token;
-    bool has_drop_request;
-    bool drop_request;
-} grpc_lb_v0_Server;
-
-typedef struct _grpc_lb_v0_InitialLoadBalanceResponse {
-    bool has_client_config;
-    char client_config[64];
-    bool has_load_balancer_delegate;
-    char load_balancer_delegate[64];
-    bool has_client_stats_report_interval;
-    grpc_lb_v0_Duration client_stats_report_interval;
-} grpc_lb_v0_InitialLoadBalanceResponse;
-
-typedef struct _grpc_lb_v0_LoadBalanceRequest {
-    bool has_initial_request;
-    grpc_lb_v0_InitialLoadBalanceRequest initial_request;
-    bool has_client_stats;
-    grpc_lb_v0_ClientStats client_stats;
-} grpc_lb_v0_LoadBalanceRequest;
-
-typedef struct _grpc_lb_v0_ServerList {
-    pb_callback_t servers;
-    bool has_expiration_interval;
-    grpc_lb_v0_Duration expiration_interval;
-} grpc_lb_v0_ServerList;
-
-typedef struct _grpc_lb_v0_LoadBalanceResponse {
-    bool has_initial_response;
-    grpc_lb_v0_InitialLoadBalanceResponse initial_response;
-    bool has_server_list;
-    grpc_lb_v0_ServerList server_list;
-} grpc_lb_v0_LoadBalanceResponse;
-
-/* Default values for struct fields */
-
-/* Initializer values for message structs */
-#define grpc_lb_v0_Duration_init_default         {false, 0, false, 0}
-#define grpc_lb_v0_LoadBalanceRequest_init_default {false, grpc_lb_v0_InitialLoadBalanceRequest_init_default, false, grpc_lb_v0_ClientStats_init_default}
-#define grpc_lb_v0_InitialLoadBalanceRequest_init_default {false, ""}
-#define grpc_lb_v0_ClientStats_init_default      {false, 0, false, 0, false, 0}
-#define grpc_lb_v0_LoadBalanceResponse_init_default {false, grpc_lb_v0_InitialLoadBalanceResponse_init_default, false, grpc_lb_v0_ServerList_init_default}
-#define grpc_lb_v0_InitialLoadBalanceResponse_init_default {false, "", false, "", false, grpc_lb_v0_Duration_init_default}
-#define grpc_lb_v0_ServerList_init_default       {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_default}
-#define grpc_lb_v0_Server_init_default           {false, "", false, 0, false, {0, {0}}, false, 0}
-#define grpc_lb_v0_Duration_init_zero            {false, 0, false, 0}
-#define grpc_lb_v0_LoadBalanceRequest_init_zero  {false, grpc_lb_v0_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v0_ClientStats_init_zero}
-#define grpc_lb_v0_InitialLoadBalanceRequest_init_zero {false, ""}
-#define grpc_lb_v0_ClientStats_init_zero         {false, 0, false, 0, false, 0}
-#define grpc_lb_v0_LoadBalanceResponse_init_zero {false, grpc_lb_v0_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v0_ServerList_init_zero}
-#define grpc_lb_v0_InitialLoadBalanceResponse_init_zero {false, "", false, "", false, grpc_lb_v0_Duration_init_zero}
-#define grpc_lb_v0_ServerList_init_zero          {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_zero}
-#define grpc_lb_v0_Server_init_zero              {false, "", false, 0, false, {0, {0}}, false, 0}
-
-/* Field tags (for use in manual encoding/decoding) */
-#define grpc_lb_v0_ClientStats_total_requests_tag 1
-#define grpc_lb_v0_ClientStats_client_rpc_errors_tag 2
-#define grpc_lb_v0_ClientStats_dropped_requests_tag 3
-#define grpc_lb_v0_Duration_seconds_tag          1
-#define grpc_lb_v0_Duration_nanos_tag            2
-#define grpc_lb_v0_InitialLoadBalanceRequest_name_tag 1
-#define grpc_lb_v0_Server_ip_address_tag         1
-#define grpc_lb_v0_Server_port_tag               2
-#define grpc_lb_v0_Server_load_balance_token_tag 3
-#define grpc_lb_v0_Server_drop_request_tag       4
-#define grpc_lb_v0_InitialLoadBalanceResponse_client_config_tag 1
-#define grpc_lb_v0_InitialLoadBalanceResponse_load_balancer_delegate_tag 2
-#define grpc_lb_v0_InitialLoadBalanceResponse_client_stats_report_interval_tag 3
-#define grpc_lb_v0_LoadBalanceRequest_initial_request_tag 1
-#define grpc_lb_v0_LoadBalanceRequest_client_stats_tag 2
-#define grpc_lb_v0_ServerList_servers_tag        1
-#define grpc_lb_v0_ServerList_expiration_interval_tag 3
-#define grpc_lb_v0_LoadBalanceResponse_initial_response_tag 1
-#define grpc_lb_v0_LoadBalanceResponse_server_list_tag 2
-
-/* Struct field encoding specification for nanopb */
-extern const pb_field_t grpc_lb_v0_Duration_fields[3];
-extern const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3];
-extern const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2];
-extern const pb_field_t grpc_lb_v0_ClientStats_fields[4];
-extern const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3];
-extern const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4];
-extern const pb_field_t grpc_lb_v0_ServerList_fields[3];
-extern const pb_field_t grpc_lb_v0_Server_fields[5];
-
-/* Maximum encoded size of messages (where known) */
-#define grpc_lb_v0_Duration_size                 22
-#define grpc_lb_v0_LoadBalanceRequest_size       169
-#define grpc_lb_v0_InitialLoadBalanceRequest_size 131
-#define grpc_lb_v0_ClientStats_size              33
-#define grpc_lb_v0_LoadBalanceResponse_size      (165 + grpc_lb_v0_ServerList_size)
-#define grpc_lb_v0_InitialLoadBalanceResponse_size 156
-#define grpc_lb_v0_Server_size                   127
-
-/* Message IDs (where set with "msgid" option) */
-#ifdef PB_MSGID
-
-#define LOAD_BALANCER_MESSAGES \
-
-
-#endif
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
index 9719673181..52e11c40bb 100644
--- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c
+++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
@@ -33,7 +33,7 @@
 /* Automatically generated nanopb constant definitions */
 /* Generated by nanopb-0.3.5-dev */
 
-#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h"
+#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h"
 
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.
@@ -41,54 +41,53 @@
 
 
 
-const pb_field_t grpc_lb_v0_Duration_fields[3] = {
-    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_Duration, seconds, seconds, 0),
-    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Duration, nanos, seconds, 0),
+const pb_field_t grpc_lb_v1_Duration_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_Duration, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Duration, nanos, seconds, 0),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3] = {
-    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v0_InitialLoadBalanceRequest_fields),
-    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v0_ClientStats_fields),
+const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v1_InitialLoadBalanceRequest_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v1_ClientStats_fields),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2] = {
-    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_InitialLoadBalanceRequest, name, name, 0),
+const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_InitialLoadBalanceRequest, name, name, 0),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_ClientStats_fields[4] = {
-    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_ClientStats, total_requests, total_requests, 0),
-    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_ClientStats, client_rpc_errors, total_requests, 0),
-    PB_FIELD(  3, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_ClientStats, dropped_requests, client_rpc_errors, 0),
+const pb_field_t grpc_lb_v1_ClientStats_fields[4] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_ClientStats, total_requests, total_requests, 0),
+    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, client_rpc_errors, total_requests, 0),
+    PB_FIELD(  3, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ClientStats, dropped_requests, client_rpc_errors, 0),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3] = {
-    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_LoadBalanceResponse, initial_response, initial_response, &grpc_lb_v0_InitialLoadBalanceResponse_fields),
-    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_LoadBalanceResponse, server_list, initial_response, &grpc_lb_v0_ServerList_fields),
+const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_LoadBalanceResponse, initial_response, initial_response, &grpc_lb_v1_InitialLoadBalanceResponse_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_LoadBalanceResponse, server_list, initial_response, &grpc_lb_v1_ServerList_fields),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4] = {
-    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_InitialLoadBalanceResponse, client_config, client_config, 0),
-    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, load_balancer_delegate, client_config, 0),
-    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v0_Duration_fields),
+const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3] = {
+    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v1_Duration_fields),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_ServerList_fields[3] = {
-    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, grpc_lb_v0_ServerList, servers, servers, &grpc_lb_v0_Server_fields),
-    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_ServerList, expiration_interval, servers, &grpc_lb_v0_Duration_fields),
+const pb_field_t grpc_lb_v1_ServerList_fields[3] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, grpc_lb_v1_ServerList, servers, servers, &grpc_lb_v1_Server_fields),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_ServerList, expiration_interval, servers, &grpc_lb_v1_Duration_fields),
     PB_LAST_FIELD
 };
 
-const pb_field_t grpc_lb_v0_Server_fields[5] = {
-    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_Server, ip_address, ip_address, 0),
-    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Server, port, ip_address, 0),
-    PB_FIELD(  3, BYTES   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Server, load_balance_token, port, 0),
-    PB_FIELD(  4, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Server, drop_request, load_balance_token, 0),
+const pb_field_t grpc_lb_v1_Server_fields[5] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, port, ip_address, 0),
+    PB_FIELD(  3, STRING  , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, load_balance_token, port, 0),
+    PB_FIELD(  4, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, drop_request, load_balance_token, 0),
     PB_LAST_FIELD
 };
 
@@ -102,7 +101,7 @@ const pb_field_t grpc_lb_v0_Server_fields[5] = {
  * numbers or field sizes that are larger than what can fit in 8 or 16 bit
  * field descriptors.
  */
-PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v0_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server)
+PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v1_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server)
 #endif
 
 #if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
@@ -113,7 +112,7 @@ PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request)
  * numbers or field sizes that are larger than what can fit in the default
  * 8 bit descriptors.
  */
-PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v0_ServerList, servers) < 256 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server)
+PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v1_ServerList, servers) < 256 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server)
 #endif
 
 
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
new file mode 100644
index 0000000000..46fe588f72
--- /dev/null
+++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
@@ -0,0 +1,178 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.3.5-dev */
+
+#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
+#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
+#include "third_party/nanopb/pb.h"
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Struct definitions */
+typedef struct _grpc_lb_v1_ClientStats {
+    bool has_total_requests;
+    int64_t total_requests;
+    bool has_client_rpc_errors;
+    int64_t client_rpc_errors;
+    bool has_dropped_requests;
+    int64_t dropped_requests;
+} grpc_lb_v1_ClientStats;
+
+typedef struct _grpc_lb_v1_Duration {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} grpc_lb_v1_Duration;
+
+typedef struct _grpc_lb_v1_InitialLoadBalanceRequest {
+    bool has_name;
+    char name[128];
+} grpc_lb_v1_InitialLoadBalanceRequest;
+
+typedef struct _grpc_lb_v1_Server {
+    bool has_ip_address;
+    char ip_address[46];
+    bool has_port;
+    int32_t port;
+    bool has_load_balance_token;
+    char load_balance_token[64];
+    bool has_drop_request;
+    bool drop_request;
+} grpc_lb_v1_Server;
+
+typedef struct _grpc_lb_v1_InitialLoadBalanceResponse {
+    bool has_load_balancer_delegate;
+    char load_balancer_delegate[64];
+    bool has_client_stats_report_interval;
+    grpc_lb_v1_Duration client_stats_report_interval;
+} grpc_lb_v1_InitialLoadBalanceResponse;
+
+typedef struct _grpc_lb_v1_LoadBalanceRequest {
+    bool has_initial_request;
+    grpc_lb_v1_InitialLoadBalanceRequest initial_request;
+    bool has_client_stats;
+    grpc_lb_v1_ClientStats client_stats;
+} grpc_lb_v1_LoadBalanceRequest;
+
+typedef struct _grpc_lb_v1_ServerList {
+    pb_callback_t servers;
+    bool has_expiration_interval;
+    grpc_lb_v1_Duration expiration_interval;
+} grpc_lb_v1_ServerList;
+
+typedef struct _grpc_lb_v1_LoadBalanceResponse {
+    bool has_initial_response;
+    grpc_lb_v1_InitialLoadBalanceResponse initial_response;
+    bool has_server_list;
+    grpc_lb_v1_ServerList server_list;
+} grpc_lb_v1_LoadBalanceResponse;
+
+/* Default values for struct fields */
+
+/* Initializer values for message structs */
+#define grpc_lb_v1_Duration_init_default         {false, 0, false, 0}
+#define grpc_lb_v1_LoadBalanceRequest_init_default {false, grpc_lb_v1_InitialLoadBalanceRequest_init_default, false, grpc_lb_v1_ClientStats_init_default}
+#define grpc_lb_v1_InitialLoadBalanceRequest_init_default {false, ""}
+#define grpc_lb_v1_ClientStats_init_default      {false, 0, false, 0, false, 0}
+#define grpc_lb_v1_LoadBalanceResponse_init_default {false, grpc_lb_v1_InitialLoadBalanceResponse_init_default, false, grpc_lb_v1_ServerList_init_default}
+#define grpc_lb_v1_InitialLoadBalanceResponse_init_default {false, "", false, grpc_lb_v1_Duration_init_default}
+#define grpc_lb_v1_ServerList_init_default       {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_default}
+#define grpc_lb_v1_Server_init_default           {false, "", false, 0, false, "", false, 0}
+#define grpc_lb_v1_Duration_init_zero            {false, 0, false, 0}
+#define grpc_lb_v1_LoadBalanceRequest_init_zero  {false, grpc_lb_v1_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v1_ClientStats_init_zero}
+#define grpc_lb_v1_InitialLoadBalanceRequest_init_zero {false, ""}
+#define grpc_lb_v1_ClientStats_init_zero         {false, 0, false, 0, false, 0}
+#define grpc_lb_v1_LoadBalanceResponse_init_zero {false, grpc_lb_v1_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v1_ServerList_init_zero}
+#define grpc_lb_v1_InitialLoadBalanceResponse_init_zero {false, "", false, grpc_lb_v1_Duration_init_zero}
+#define grpc_lb_v1_ServerList_init_zero          {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_zero}
+#define grpc_lb_v1_Server_init_zero              {false, "", false, 0, false, "", false, 0}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define grpc_lb_v1_ClientStats_total_requests_tag 1
+#define grpc_lb_v1_ClientStats_client_rpc_errors_tag 2
+#define grpc_lb_v1_ClientStats_dropped_requests_tag 3
+#define grpc_lb_v1_Duration_seconds_tag          1
+#define grpc_lb_v1_Duration_nanos_tag            2
+#define grpc_lb_v1_InitialLoadBalanceRequest_name_tag 1
+#define grpc_lb_v1_Server_ip_address_tag         1
+#define grpc_lb_v1_Server_port_tag               2
+#define grpc_lb_v1_Server_load_balance_token_tag 3
+#define grpc_lb_v1_Server_drop_request_tag       4
+#define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 2
+#define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 3
+#define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1
+#define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2
+#define grpc_lb_v1_ServerList_servers_tag        1
+#define grpc_lb_v1_ServerList_expiration_interval_tag 3
+#define grpc_lb_v1_LoadBalanceResponse_initial_response_tag 1
+#define grpc_lb_v1_LoadBalanceResponse_server_list_tag 2
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t grpc_lb_v1_Duration_fields[3];
+extern const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3];
+extern const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2];
+extern const pb_field_t grpc_lb_v1_ClientStats_fields[4];
+extern const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3];
+extern const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3];
+extern const pb_field_t grpc_lb_v1_ServerList_fields[3];
+extern const pb_field_t grpc_lb_v1_Server_fields[5];
+
+/* Maximum encoded size of messages (where known) */
+#define grpc_lb_v1_Duration_size                 22
+#define grpc_lb_v1_LoadBalanceRequest_size       169
+#define grpc_lb_v1_InitialLoadBalanceRequest_size 131
+#define grpc_lb_v1_ClientStats_size              33
+#define grpc_lb_v1_LoadBalanceResponse_size      (98 + grpc_lb_v1_ServerList_size)
+#define grpc_lb_v1_InitialLoadBalanceResponse_size 90
+#define grpc_lb_v1_Server_size                   127
+
+/* Message IDs (where set with "msgid" option) */
+#ifdef PB_MSGID
+
+#define LOAD_BALANCER_MESSAGES \
+
+
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c
index 0d215cd196..9decf70692 100644
--- a/src/core/ext/lb_policy/pick_first/pick_first.c
+++ b/src/core/ext/lb_policy/pick_first/pick_first.c
@@ -39,7 +39,7 @@
 
 typedef struct pending_pick {
   struct pending_pick *next;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   uint32_t initial_metadata_flags;
   grpc_connected_subchannel **target;
   grpc_closure *on_complete;
@@ -103,8 +103,9 @@ static void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   p->shutdown = 1;
   pp = p->pending_picks;
   p->pending_picks = NULL;
-  grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                              GRPC_CHANNEL_FATAL_FAILURE, "shutdown");
+  grpc_connectivity_state_set(
+      exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
+      GRPC_ERROR_CREATE("Channel shutdown"), "shutdown");
   /* cancel subscription */
   if (selected != NULL) {
     grpc_connected_subchannel_notify_on_state_change(
@@ -118,9 +119,9 @@ static void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while (pp != NULL) {
     pending_pick *next = pp->next;
     *pp->target = NULL;
-    grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                 pp->pollset);
-    grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL);
+    grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                             p->base.interested_parties);
+    grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
     gpr_free(pp);
     pp = next;
   }
@@ -136,10 +137,11 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   while (pp != NULL) {
     pending_pick *next = pp->next;
     if (pp->target == target) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
       *target = NULL;
-      grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
+                          GRPC_ERROR_CREATE("Pick Cancelled"), NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -162,9 +164,10 @@ static void pf_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
-      grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
+                          GRPC_ERROR_CREATE("Pick Cancelled"), NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -196,7 +199,8 @@ static void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                   grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
+                   grpc_polling_entity *pollent,
+                   grpc_metadata_batch *initial_metadata,
                    uint32_t initial_metadata_flags,
                    grpc_connected_subchannel **target,
                    grpc_closure *on_complete) {
@@ -221,10 +225,11 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset);
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+                                           p->base.interested_parties);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
-    pp->pollset = pollset;
+    pp->pollent = pollent;
     pp->target = target;
     pp->initial_metadata_flags = initial_metadata_flags;
     pp->on_complete = on_complete;
@@ -235,7 +240,7 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
 }
 
 static void destroy_subchannels(grpc_exec_ctx *exec_ctx, void *arg,
-                                bool iomgr_success) {
+                                grpc_error *error) {
   pick_first_lb_policy *p = arg;
   size_t i;
   size_t num_subchannels = p->num_subchannels;
@@ -256,12 +261,14 @@ static void destroy_subchannels(grpc_exec_ctx *exec_ctx, void *arg,
 }
 
 static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                    bool iomgr_success) {
+                                    grpc_error *error) {
   pick_first_lb_policy *p = arg;
   grpc_subchannel *selected_subchannel;
   pending_pick *pp;
   grpc_connected_subchannel *selected;
 
+  GRPC_ERROR_REF(error);
+
   gpr_mu_lock(&p->mu);
 
   selected = GET_SELECTED(p);
@@ -269,15 +276,17 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
   if (p->shutdown) {
     gpr_mu_unlock(&p->mu);
     GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pick_first_connectivity");
+    GRPC_ERROR_UNREF(error);
     return;
   } else if (selected != NULL) {
     if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) {
       /* if the selected channel goes bad, we're done */
-      p->checking_connectivity = GRPC_CHANNEL_FATAL_FAILURE;
+      p->checking_connectivity = GRPC_CHANNEL_SHUTDOWN;
     }
     grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                p->checking_connectivity, "selected_changed");
-    if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) {
+                                p->checking_connectivity, GRPC_ERROR_REF(error),
+                                "selected_changed");
+    if (p->checking_connectivity != GRPC_CHANNEL_SHUTDOWN) {
       grpc_connected_subchannel_notify_on_state_change(
           exec_ctx, selected, p->base.interested_parties,
           &p->checking_connectivity, &p->connectivity_changed);
@@ -289,7 +298,8 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
     switch (p->checking_connectivity) {
       case GRPC_CHANNEL_READY:
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                    GRPC_CHANNEL_READY, "connecting_ready");
+                                    GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
+                                    "connecting_ready");
         selected_subchannel = p->subchannels[p->checking_subchannel];
         selected =
             grpc_subchannel_get_connected_subchannel(selected_subchannel);
@@ -298,15 +308,16 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         /* drop the pick list: we are connected now */
         GRPC_LB_POLICY_WEAK_REF(&p->base, "destroy_subchannels");
         gpr_atm_rel_store(&p->selected, (gpr_atm)selected);
-        grpc_exec_ctx_enqueue(
-            exec_ctx, grpc_closure_create(destroy_subchannels, p), true, NULL);
+        grpc_exec_ctx_sched(exec_ctx,
+                            grpc_closure_create(destroy_subchannels, p),
+                            GRPC_ERROR_NONE, NULL);
         /* update any calls that were waiting for a pick */
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
           *pp->target = selected;
-          grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                       pp->pollset);
-          grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL);
+          grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                                   p->base.interested_parties);
+          grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
           gpr_free(pp);
         }
         grpc_connected_subchannel_notify_on_state_change(
@@ -318,12 +329,13 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
             (p->checking_subchannel + 1) % p->num_subchannels;
         if (p->checking_subchannel == 0) {
           /* only trigger transient failure when we've tried all alternatives */
-          grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                      GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                      "connecting_transient_failure");
+          grpc_connectivity_state_set(
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
+              GRPC_ERROR_REF(error), "connecting_transient_failure");
         }
+        GRPC_ERROR_UNREF(error);
         p->checking_connectivity = grpc_subchannel_check_connectivity(
-            p->subchannels[p->checking_subchannel]);
+            p->subchannels[p->checking_subchannel], &error);
         if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) {
           grpc_subchannel_notify_on_state_change(
               exec_ctx, p->subchannels[p->checking_subchannel],
@@ -335,53 +347,60 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         break;
       case GRPC_CHANNEL_CONNECTING:
       case GRPC_CHANNEL_IDLE:
-        grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                    GRPC_CHANNEL_CONNECTING,
-                                    "connecting_changed");
+        grpc_connectivity_state_set(
+            exec_ctx, &p->state_tracker, GRPC_CHANNEL_CONNECTING,
+            GRPC_ERROR_REF(error), "connecting_changed");
         grpc_subchannel_notify_on_state_change(
             exec_ctx, p->subchannels[p->checking_subchannel],
             p->base.interested_parties, &p->checking_connectivity,
             &p->connectivity_changed);
         break;
-      case GRPC_CHANNEL_FATAL_FAILURE:
+      case GRPC_CHANNEL_SHUTDOWN:
         p->num_subchannels--;
         GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel],
                  p->subchannels[p->num_subchannels]);
         GRPC_SUBCHANNEL_UNREF(exec_ctx, p->subchannels[p->num_subchannels],
                               "pick_first");
         if (p->num_subchannels == 0) {
-          grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                      GRPC_CHANNEL_FATAL_FAILURE,
-                                      "no_more_channels");
+          grpc_connectivity_state_set(
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
+              GRPC_ERROR_CREATE_REFERENCING("Pick first exhausted channels",
+                                            &error, 1),
+              "no_more_channels");
           while ((pp = p->pending_picks)) {
             p->pending_picks = pp->next;
             *pp->target = NULL;
-            grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL);
+            grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
+                                NULL);
             gpr_free(pp);
           }
           GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base,
                                     "pick_first_connectivity");
         } else {
-          grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                      GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                      "subchannel_failed");
+          grpc_connectivity_state_set(
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
+              GRPC_ERROR_REF(error), "subchannel_failed");
           p->checking_subchannel %= p->num_subchannels;
+          GRPC_ERROR_UNREF(error);
           p->checking_connectivity = grpc_subchannel_check_connectivity(
-              p->subchannels[p->checking_subchannel]);
+              p->subchannels[p->checking_subchannel], &error);
           goto loop;
         }
     }
   }
 
   gpr_mu_unlock(&p->mu);
+
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_connectivity_state pf_check_connectivity(grpc_exec_ctx *exec_ctx,
-                                                     grpc_lb_policy *pol) {
+                                                     grpc_lb_policy *pol,
+                                                     grpc_error **error) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   grpc_connectivity_state st;
   gpr_mu_lock(&p->mu);
-  st = grpc_connectivity_state_check(&p->state_tracker);
+  st = grpc_connectivity_state_check(&p->state_tracker, error);
   gpr_mu_unlock(&p->mu);
   return st;
 }
@@ -404,7 +423,8 @@ static void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   if (selected) {
     grpc_connected_subchannel_ping(exec_ctx, selected, closure);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("Not connected"),
+                        NULL);
   }
 }
 
diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c
index dcdc0c6285..7bcf608ab9 100644
--- a/src/core/ext/lb_policy/round_robin/round_robin.c
+++ b/src/core/ext/lb_policy/round_robin/round_robin.c
@@ -31,6 +31,34 @@
  *
  */
 
+/** Round Robin Policy.
+ *
+ * This policy keeps:
+ * - A circular list of ready (connected) subchannels, the *readylist*. An empty
+ *   readylist consists solely of its root (dummy) node.
+ * - A pointer to the last element picked from the readylist, the *lastpick*.
+ *   Initially set to point to the readylist's root.
+ *
+ * Behavior:
+ * - When a subchannel connects, it's *prepended* to the readylist's root node.
+ *   Ie, if readylist = A <-> B <-> ROOT <-> C
+ *                      ^                    ^
+ *                      |____________________|
+ *   and subchannel D becomes connected, the addition of D to the readylist
+ *   results in  readylist = A <-> B <-> D <-> ROOT <-> C
+ *                           ^                          ^
+ *                           |__________________________|
+ * - When a subchannel disconnects, it's removed from the readylist. If the
+ *   subchannel being removed was the most recently picked, the *lastpick*
+ *   pointer moves to the removed node's previous element. Note that if the
+ *   readylist only had one element, this is still legal, as the lastpick would
+ *   point to the dummy root node, for an empty readylist.
+ * - Upon picking, *lastpick* is updated to point to the returned (connected)
+ *   subchannel. Note that it's possible that the selected subchannel becomes
+ *   disconnected in the interim between the selection and the actual usage of
+ *   the subchannel by the caller.
+ */
+
 #include <string.h>
 
 #include <grpc/support/alloc.h>
@@ -48,7 +76,7 @@ int grpc_lb_round_robin_trace = 0;
  * Once a pick is available, \a target is updated and \a on_complete called. */
 typedef struct pending_pick {
   struct pending_pick *next;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   uint32_t initial_metadata_flags;
   grpc_connected_subchannel **target;
   grpc_closure *on_complete;
@@ -173,9 +201,7 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p,
     return;
   }
   if (node == p->ready_list_last_pick) {
-    /* If removing the lastly picked node, reset the last pick pointer to the
-     * dummy root of the list */
-    p->ready_list_last_pick = &p->ready_list;
+    p->ready_list_last_pick = p->ready_list_last_pick->prev;
   }
 
   /* removing last item */
@@ -239,11 +265,13 @@ static void rr_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while ((pp = p->pending_picks)) {
     p->pending_picks = pp->next;
     *pp->target = NULL;
-    grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
+                        GRPC_ERROR_CREATE("Channel Shutdown"), NULL);
     gpr_free(pp);
   }
-  grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                              GRPC_CHANNEL_FATAL_FAILURE, "shutdown");
+  grpc_connectivity_state_set(
+      exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
+      GRPC_ERROR_CREATE("Channel Shutdown"), "shutdown");
   for (i = 0; i < p->num_subchannels; i++) {
     subchannel_data *sd = p->subchannels[i];
     grpc_subchannel_notify_on_state_change(exec_ctx, sd->subchannel, NULL, NULL,
@@ -262,10 +290,11 @@ static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   while (pp != NULL) {
     pending_pick *next = pp->next;
     if (pp->target == target) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
       *target = NULL;
-      grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_CANCELLED,
+                          NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -288,10 +317,11 @@ static void rr_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                   pp->pollset);
+      grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                               p->base.interested_parties);
       *pp->target = NULL;
-      grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_CANCELLED,
+                          NULL);
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -307,7 +337,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) {
   p->started_picking = 1;
 
   if (grpc_lb_round_robin_trace) {
-    gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%d", p,
+    gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%" PRIuPTR, p,
             p->num_subchannels);
   }
 
@@ -331,7 +361,8 @@ static void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                   grpc_pollset *pollset, grpc_metadata_batch *initial_metadata,
+                   grpc_polling_entity *pollent,
+                   grpc_metadata_batch *initial_metadata,
                    uint32_t initial_metadata_flags,
                    grpc_connected_subchannel **target,
                    grpc_closure *on_complete) {
@@ -344,8 +375,8 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     *target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
     if (grpc_lb_round_robin_trace) {
       gpr_log(GPR_DEBUG,
-              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)",
-              selected->subchannel, selected);
+              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", *target,
+              selected);
     }
     /* only advance the last picked pointer if the selection was used */
     advance_last_picked_locked(p);
@@ -354,10 +385,11 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset);
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+                                           p->base.interested_parties);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
-    pp->pollset = pollset;
+    pp->pollent = pollent;
     pp->target = target;
     pp->on_complete = on_complete;
     pp->initial_metadata_flags = initial_metadata_flags;
@@ -368,7 +400,7 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
 }
 
 static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                    bool iomgr_success) {
+                                    grpc_error *error) {
   subchannel_data *sd = arg;
   round_robin_lb_policy *p = sd->policy;
   pending_pick *pp;
@@ -376,6 +408,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
 
   int unref = 0;
 
+  GRPC_ERROR_REF(error);
   gpr_mu_lock(&p->mu);
 
   if (p->shutdown) {
@@ -384,7 +417,8 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
     switch (sd->connectivity_state) {
       case GRPC_CHANNEL_READY:
         grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                    GRPC_CHANNEL_READY, "connecting_ready");
+                                    GRPC_CHANNEL_READY, GRPC_ERROR_REF(error),
+                                    "connecting_ready");
         /* add the newly connected subchannel to the list of connected ones.
          * Note that it goes to the "end of the line". */
         sd->ready_list_node = add_connected_sc_locked(p, sd->subchannel);
@@ -406,9 +440,9 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                     "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)",
                     selected->subchannel, selected);
           }
-          grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties,
-                                       pp->pollset);
-          grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL);
+          grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
+                                                   p->base.interested_parties);
+          grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
           gpr_free(pp);
         }
         grpc_subchannel_notify_on_state_change(
@@ -417,9 +451,9 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         break;
       case GRPC_CHANNEL_CONNECTING:
       case GRPC_CHANNEL_IDLE:
-        grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                    sd->connectivity_state,
-                                    "connecting_changed");
+        grpc_connectivity_state_set(
+            exec_ctx, &p->state_tracker, sd->connectivity_state,
+            GRPC_ERROR_REF(error), "connecting_changed");
         grpc_subchannel_notify_on_state_change(
             exec_ctx, sd->subchannel, p->base.interested_parties,
             &sd->connectivity_state, &sd->connectivity_changed_closure);
@@ -435,11 +469,11 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
           remove_disconnected_sc_locked(p, sd->ready_list_node);
           sd->ready_list_node = NULL;
         }
-        grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                    GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                    "connecting_transient_failure");
+        grpc_connectivity_state_set(
+            exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
+            GRPC_ERROR_REF(error), "connecting_transient_failure");
         break;
-      case GRPC_CHANNEL_FATAL_FAILURE:
+      case GRPC_CHANNEL_SHUTDOWN:
         if (sd->ready_list_node != NULL) {
           remove_disconnected_sc_locked(p, sd->ready_list_node);
           sd->ready_list_node = NULL;
@@ -454,19 +488,22 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
 
         unref = 1;
         if (p->num_subchannels == 0) {
-          grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                      GRPC_CHANNEL_FATAL_FAILURE,
-                                      "no_more_channels");
+          grpc_connectivity_state_set(
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
+              GRPC_ERROR_CREATE_REFERENCING("Round Robin Channels Exhausted",
+                                            &error, 1),
+              "no_more_channels");
           while ((pp = p->pending_picks)) {
             p->pending_picks = pp->next;
             *pp->target = NULL;
-            grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL);
+            grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
+                                NULL);
             gpr_free(pp);
           }
         } else {
-          grpc_connectivity_state_set(exec_ctx, &p->state_tracker,
-                                      GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                      "subchannel_failed");
+          grpc_connectivity_state_set(
+              exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
+              GRPC_ERROR_REF(error), "subchannel_failed");
         }
     } /* switch */
   }   /* !unref */
@@ -476,14 +513,17 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
   if (unref) {
     GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "round_robin_connectivity");
   }
+
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_connectivity_state rr_check_connectivity(grpc_exec_ctx *exec_ctx,
-                                                     grpc_lb_policy *pol) {
+                                                     grpc_lb_policy *pol,
+                                                     grpc_error **error) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   grpc_connectivity_state st;
   gpr_mu_lock(&p->mu);
-  st = grpc_connectivity_state_check(&p->state_tracker);
+  st = grpc_connectivity_state_check(&p->state_tracker, error);
   gpr_mu_unlock(&p->mu);
   return st;
 }
@@ -511,7 +551,8 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     grpc_connected_subchannel_ping(exec_ctx, target, closure);
   } else {
     gpr_mu_unlock(&p->mu);
-    grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure,
+                        GRPC_ERROR_CREATE("Round Robin not connected"), NULL);
   }
 }
 
@@ -524,7 +565,7 @@ static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {}
 
 static void round_robin_factory_unref(grpc_lb_policy_factory *factory) {}
 
-static grpc_lb_policy *create_round_robin(grpc_exec_ctx *exec_ctx,
+static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
                                           grpc_lb_policy_factory *factory,
                                           grpc_lb_policy_args *args) {
   GPR_ASSERT(args->addresses != NULL);
@@ -580,7 +621,7 @@ static grpc_lb_policy *create_round_robin(grpc_exec_ctx *exec_ctx,
 }
 
 static const grpc_lb_policy_factory_vtable round_robin_factory_vtable = {
-    round_robin_factory_ref, round_robin_factory_unref, create_round_robin,
+    round_robin_factory_ref, round_robin_factory_unref, round_robin_create,
     "round_robin"};
 
 static grpc_lb_policy_factory round_robin_lb_policy_factory = {
diff --git a/src/core/ext/load_reporting/load_reporting.c b/src/core/ext/load_reporting/load_reporting.c
new file mode 100644
index 0000000000..9e4d32676f
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting.c
@@ -0,0 +1,133 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <limits.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/sync.h>
+
+#include "src/core/ext/load_reporting/load_reporting.h"
+#include "src/core/ext/load_reporting/load_reporting_filter.h"
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_init.h"
+
+struct grpc_load_reporting_config {
+  grpc_load_reporting_fn fn;
+  void *user_data;
+};
+
+grpc_load_reporting_config *grpc_load_reporting_config_create(
+    grpc_load_reporting_fn fn, void *user_data) {
+  GPR_ASSERT(fn != NULL);
+  grpc_load_reporting_config *lrc =
+      gpr_malloc(sizeof(grpc_load_reporting_config));
+  lrc->fn = fn;
+  lrc->user_data = user_data;
+  return lrc;
+}
+
+grpc_load_reporting_config *grpc_load_reporting_config_copy(
+    grpc_load_reporting_config *src) {
+  return grpc_load_reporting_config_create(src->fn, src->user_data);
+}
+
+void grpc_load_reporting_config_destroy(grpc_load_reporting_config *lrc) {
+  gpr_free(lrc);
+}
+
+void grpc_load_reporting_config_call(
+    grpc_load_reporting_config *lrc,
+    const grpc_load_reporting_call_data *call_data) {
+  lrc->fn(call_data, lrc->user_data);
+}
+
+static bool is_load_reporting_enabled(const grpc_channel_args *a) {
+  if (a == NULL) return false;
+  for (size_t i = 0; i < a->num_args; i++) {
+    if (0 == strcmp(a->args[i].key, GRPC_ARG_ENABLE_LOAD_REPORTING)) {
+      return a->args[i].type == GRPC_ARG_POINTER &&
+             a->args[i].value.pointer.p != NULL;
+    }
+  }
+  return false;
+}
+
+static bool maybe_add_load_reporting_filter(grpc_channel_stack_builder *builder,
+                                            void *arg) {
+  const grpc_channel_args *args =
+      grpc_channel_stack_builder_get_channel_arguments(builder);
+  if (is_load_reporting_enabled(args)) {
+    return grpc_channel_stack_builder_prepend_filter(
+        builder, (const grpc_channel_filter *)arg, NULL, NULL);
+  }
+  return true;
+}
+
+static void lrd_arg_destroy(void *p) { grpc_load_reporting_config_destroy(p); }
+
+static void *lrd_arg_copy(void *p) {
+  return grpc_load_reporting_config_copy(p);
+}
+
+static int lrd_arg_cmp(void *a, void *b) {
+  grpc_load_reporting_config *lhs = a;
+  grpc_load_reporting_config *rhs = b;
+  return !(lhs->fn == rhs->fn && lhs->user_data == rhs->user_data);
+}
+
+static const grpc_arg_pointer_vtable lrd_ptr_vtable = {
+    lrd_arg_copy, lrd_arg_destroy, lrd_arg_cmp};
+
+grpc_arg grpc_load_reporting_config_create_arg(
+    grpc_load_reporting_config *lrc) {
+  grpc_arg arg;
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_ARG_ENABLE_LOAD_REPORTING;
+  arg.value.pointer.p = lrc;
+  arg.value.pointer.vtable = &lrd_ptr_vtable;
+  return arg;
+}
+
+/* Plugin registration */
+
+void grpc_load_reporting_plugin_init(void) {
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
+                                   maybe_add_load_reporting_filter,
+                                   (void *)&grpc_load_reporting_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_load_reporting_filter,
+                                   (void *)&grpc_load_reporting_filter);
+}
+
+void grpc_load_reporting_plugin_shutdown() {}
diff --git a/src/core/ext/load_reporting/load_reporting.h b/src/core/ext/load_reporting/load_reporting.h
new file mode 100644
index 0000000000..316cd89bd7
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting.h
@@ -0,0 +1,75 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H
+#define GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H
+
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/surface/call.h"
+
+typedef struct grpc_load_reporting_config grpc_load_reporting_config;
+
+/** Call information to be passed to the provided load reporting function upon
+ * completion of the call */
+typedef struct grpc_load_reporting_call_data {
+  const grpc_call_stats *stats;   /**< Stats for the call */
+  const char *trailing_md_string; /**< LR trailing metadata info */
+} grpc_load_reporting_call_data;
+
+/** Custom function to be called by the load reporting filter. */
+typedef void (*grpc_load_reporting_fn)(
+    const grpc_load_reporting_call_data *call_data, void *user_data);
+
+/** Register \a fn as the function to be invoked by the load reporting filter.
+ * \a fn will be invoked at the beginning and at the end of the call.
+ *
+ * For the first invocation, \a fn's first argument
+ * (grpc_load_reporting_call_data*) will be NULL. \a user_data is always passed
+ * as-is. */
+grpc_load_reporting_config *grpc_load_reporting_config_create(
+    grpc_load_reporting_fn fn, void *user_data);
+
+grpc_load_reporting_config *grpc_load_reporting_config_copy(
+    grpc_load_reporting_config *src);
+
+void grpc_load_reporting_config_destroy(grpc_load_reporting_config *lrc);
+
+/** Invoke the function registered by \a grpc_load_reporting_init. */
+void grpc_load_reporting_config_call(
+    grpc_load_reporting_config *lrc,
+    const grpc_load_reporting_call_data *call_data);
+
+/** Return a \a grpc_arg enabling load reporting */
+grpc_arg grpc_load_reporting_config_create_arg(grpc_load_reporting_config *lrc);
+
+#endif /* GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H */
diff --git a/src/core/ext/load_reporting/load_reporting_filter.c b/src/core/ext/load_reporting/load_reporting_filter.c
new file mode 100644
index 0000000000..f372f88c3a
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting_filter.c
@@ -0,0 +1,151 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <string.h>
+
+#include "src/core/ext/load_reporting/load_reporting.h"
+#include "src/core/ext/load_reporting/load_reporting_filter.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+typedef struct call_data { const char *trailing_md_string; } call_data;
+typedef struct channel_data {
+  gpr_mu mu;
+  grpc_load_reporting_config *lrc;
+} channel_data;
+
+static void invoke_lr_fn_locked(grpc_load_reporting_config *lrc,
+                                grpc_load_reporting_call_data *lr_call_data) {
+  GPR_TIMER_BEGIN("load_reporting_config_fn", 0);
+  grpc_load_reporting_config_call(lrc, lr_call_data);
+  GPR_TIMER_END("load_reporting_config_fn", 0);
+}
+
+/* Constructor for call_data */
+static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                           grpc_call_element_args *args) {
+  call_data *calld = elem->call_data;
+  memset(calld, 0, sizeof(call_data));
+}
+
+/* Destructor for call_data */
+static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats, void *ignored) {
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+
+  grpc_load_reporting_call_data lr_call_data = {stats,
+                                                calld->trailing_md_string};
+
+  gpr_mu_lock(&chand->mu);
+  invoke_lr_fn_locked(chand->lrc, &lr_call_data);
+  gpr_mu_unlock(&chand->mu);
+}
+
+/* Constructor for channel_data */
+static void init_channel_elem(grpc_exec_ctx *exec_ctx,
+                              grpc_channel_element *elem,
+                              grpc_channel_element_args *args) {
+  GPR_ASSERT(!args->is_last);
+
+  channel_data *chand = elem->channel_data;
+  memset(chand, 0, sizeof(channel_data));
+
+  gpr_mu_init(&chand->mu);
+  for (size_t i = 0; i < args->channel_args->num_args; i++) {
+    if (0 == strcmp(args->channel_args->args[i].key,
+                    GRPC_ARG_ENABLE_LOAD_REPORTING)) {
+      grpc_load_reporting_config *arg_lrc =
+          args->channel_args->args[i].value.pointer.p;
+      chand->lrc = grpc_load_reporting_config_copy(arg_lrc);
+      GPR_ASSERT(chand->lrc != NULL);
+      break;
+    }
+  }
+  GPR_ASSERT(chand->lrc != NULL); /* arg actually found */
+
+  gpr_mu_lock(&chand->mu);
+  invoke_lr_fn_locked(chand->lrc, NULL);
+  gpr_mu_unlock(&chand->mu);
+}
+
+/* Destructor for channel data */
+static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_element *elem) {
+  channel_data *chand = elem->channel_data;
+  gpr_mu_destroy(&chand->mu);
+  grpc_load_reporting_config_destroy(chand->lrc);
+}
+
+static grpc_mdelem *lr_trailing_md_filter(void *user_data, grpc_mdelem *md) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+
+  if (md->key == GRPC_MDSTR_LOAD_REPORTING) {
+    calld->trailing_md_string = gpr_strdup(grpc_mdstr_as_c_string(md->value));
+    return NULL;
+  }
+
+  return md;
+}
+
+static void lr_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
+                                         grpc_call_element *elem,
+                                         grpc_transport_stream_op *op) {
+  GPR_TIMER_BEGIN("lr_start_transport_stream_op", 0);
+
+  if (op->send_trailing_metadata) {
+    grpc_metadata_batch_filter(op->send_trailing_metadata,
+                               lr_trailing_md_filter, elem);
+  }
+  grpc_call_next_op(exec_ctx, elem, op);
+
+  GPR_TIMER_END("lr_start_transport_stream_op", 0);
+}
+
+const grpc_channel_filter grpc_load_reporting_filter = {
+    lr_start_transport_stream_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "load_reporting"};
diff --git a/src/core/ext/load_reporting/load_reporting_filter.h b/src/core/ext/load_reporting/load_reporting_filter.h
new file mode 100644
index 0000000000..f69cd6fdc6
--- /dev/null
+++ b/src/core/ext/load_reporting/load_reporting_filter.h
@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_FILTER_H
+#define GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_FILTER_H
+
+#include "src/core/lib/channel/channel_stack.h"
+
+extern const grpc_channel_filter grpc_load_reporting_filter;
+
+#endif /* GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_FILTER_H */
diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c
index 620ba4e2aa..31ac968670 100644
--- a/src/core/ext/resolver/dns/native/dns_resolver.c
+++ b/src/core/ext/resolver/dns/native/dns_resolver.c
@@ -82,6 +82,9 @@ typedef struct {
   grpc_timer retry_timer;
   /** retry backoff state */
   gpr_backoff backoff_state;
+
+  /** currently resolving addresses */
+  grpc_resolved_addresses *addresses;
 } dns_resolver;
 
 static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
@@ -108,7 +111,8 @@ static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
   }
   if (r->next_completion != NULL) {
     *r->target_config = NULL;
-    grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion,
+                        GRPC_ERROR_CREATE("Resolver Shutdown"), NULL);
     r->next_completion = NULL;
   }
   gpr_mu_unlock(&r->mu);
@@ -143,12 +147,12 @@ static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
 }
 
 static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg,
-                               bool success) {
+                               grpc_error *error) {
   dns_resolver *r = arg;
 
   gpr_mu_lock(&r->mu);
   r->have_retry_timer = false;
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     if (!r->resolving) {
       dns_start_resolving_locked(exec_ctx, r);
     }
@@ -159,13 +163,14 @@ static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg,
 }
 
 static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
-                            grpc_resolved_addresses *addresses) {
+                            grpc_error *error) {
   dns_resolver *r = arg;
   grpc_client_config *config = NULL;
   grpc_lb_policy *lb_policy;
   gpr_mu_lock(&r->mu);
   GPR_ASSERT(r->resolving);
   r->resolving = 0;
+  grpc_resolved_addresses *addresses = r->addresses;
   if (addresses != NULL) {
     grpc_lb_policy_args lb_policy_args;
     config = grpc_client_config_create();
@@ -183,11 +188,18 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
     gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
     gpr_timespec next_try = gpr_backoff_step(&r->backoff_state, now);
     gpr_timespec timeout = gpr_time_sub(next_try, now);
-    gpr_log(GPR_DEBUG, "dns resolution failed: retrying in %d.%09d seconds",
-            timeout.tv_sec, timeout.tv_nsec);
+    const char *msg = grpc_error_string(error);
+    gpr_log(GPR_DEBUG, "dns resolution failed: %s", msg);
+    grpc_error_free_string(msg);
     GPR_ASSERT(!r->have_retry_timer);
     r->have_retry_timer = true;
     GRPC_RESOLVER_REF(&r->base, "retry-timer");
+    if (gpr_time_cmp(timeout, gpr_time_0(timeout.clock_type)) <= 0) {
+      gpr_log(GPR_DEBUG, "retrying in %" PRId64 ".%09d seconds", timeout.tv_sec,
+              timeout.tv_nsec);
+    } else {
+      gpr_log(GPR_DEBUG, "retrying immediately");
+    }
     grpc_timer_init(exec_ctx, &r->retry_timer, next_try, dns_on_retry_timer, r,
                     now);
   }
@@ -207,7 +219,9 @@ static void dns_start_resolving_locked(grpc_exec_ctx *exec_ctx,
   GRPC_RESOLVER_REF(&r->base, "dns-resolving");
   GPR_ASSERT(!r->resolving);
   r->resolving = 1;
-  grpc_resolve_address(exec_ctx, r->name, r->default_port, dns_on_resolved, r);
+  r->addresses = NULL;
+  grpc_resolve_address(exec_ctx, r->name, r->default_port,
+                       grpc_closure_create(dns_on_resolved, r), &r->addresses);
 }
 
 static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
@@ -218,7 +232,7 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
     if (r->resolved_config) {
       grpc_client_config_ref(r->resolved_config);
     }
-    grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
     r->published_version = r->resolved_version;
   }
diff --git a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
index a4fa9acf22..1f7cce2f43 100644
--- a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
+++ b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
@@ -92,7 +92,7 @@ static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&r->mu);
   if (r->next_completion != NULL) {
     *r->target_config = NULL;
-    grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
   gpr_mu_unlock(&r->mu);
@@ -133,7 +133,7 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "sockaddr");
     r->published = 1;
     *r->target_config = cfg;
-    grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
 }
diff --git a/src/core/ext/resolver/zookeeper/zookeeper_resolver.c b/src/core/ext/resolver/zookeeper/zookeeper_resolver.c
deleted file mode 100644
index deb4b9b1ef..0000000000
--- a/src/core/ext/resolver/zookeeper/zookeeper_resolver.c
+++ /dev/null
@@ -1,513 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <string.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/string_util.h>
-
-#include <grpc/grpc_zookeeper.h>
-#include <zookeeper/zookeeper.h>
-
-#include "src/core/ext/client_config/lb_policy_registry.h"
-#include "src/core/ext/client_config/resolver_registry.h"
-#include "src/core/lib/iomgr/resolve_address.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/surface/api_trace.h"
-
-/** Zookeeper session expiration time in milliseconds */
-#define GRPC_ZOOKEEPER_SESSION_TIMEOUT 15000
-
-typedef struct {
-  /** base class: must be first */
-  grpc_resolver base;
-  /** refcount */
-  gpr_refcount refs;
-  /** name to resolve */
-  char *name;
-  /** subchannel factory */
-  grpc_client_channel_factory *client_channel_factory;
-  /** load balancing policy name */
-  char *lb_policy_name;
-
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
-  /** are we currently resolving? */
-  int resolving;
-  /** which version of resolved_config have we published? */
-  int published_version;
-  /** which version of resolved_config is current? */
-  int resolved_version;
-  /** pending next completion, or NULL */
-  grpc_closure *next_completion;
-  /** target config address for next completion */
-  grpc_client_config **target_config;
-  /** current (fully resolved) config */
-  grpc_client_config *resolved_config;
-
-  /** zookeeper handle */
-  zhandle_t *zookeeper_handle;
-  /** zookeeper resolved addresses */
-  grpc_resolved_addresses *resolved_addrs;
-  /** total number of addresses to be resolved */
-  int resolved_total;
-  /** number of addresses resolved */
-  int resolved_num;
-} zookeeper_resolver;
-
-static void zookeeper_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
-
-static void zookeeper_start_resolving_locked(zookeeper_resolver *r);
-static void zookeeper_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
-                                               zookeeper_resolver *r);
-
-static void zookeeper_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *r);
-static void zookeeper_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                        grpc_resolver *r);
-static void zookeeper_next(grpc_exec_ctx *exec_ctx, grpc_resolver *r,
-                           grpc_client_config **target_config,
-                           grpc_closure *on_complete);
-
-static const grpc_resolver_vtable zookeeper_resolver_vtable = {
-    zookeeper_destroy, zookeeper_shutdown, zookeeper_channel_saw_error,
-    zookeeper_next};
-
-static void zookeeper_shutdown(grpc_exec_ctx *exec_ctx,
-                               grpc_resolver *resolver) {
-  zookeeper_resolver *r = (zookeeper_resolver *)resolver;
-  grpc_closure *call = NULL;
-  gpr_mu_lock(&r->mu);
-  if (r->next_completion != NULL) {
-    *r->target_config = NULL;
-    call = r->next_completion;
-    r->next_completion = NULL;
-  }
-  zookeeper_close(r->zookeeper_handle);
-  gpr_mu_unlock(&r->mu);
-  if (call != NULL) {
-    call->cb(exec_ctx, call->cb_arg, 1);
-  }
-}
-
-static void zookeeper_channel_saw_error(grpc_exec_ctx *exec_ctx,
-                                        grpc_resolver *resolver) {
-  zookeeper_resolver *r = (zookeeper_resolver *)resolver;
-  gpr_mu_lock(&r->mu);
-  if (r->resolving == 0) {
-    zookeeper_start_resolving_locked(r);
-  }
-  gpr_mu_unlock(&r->mu);
-}
-
-static void zookeeper_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
-                           grpc_client_config **target_config,
-                           grpc_closure *on_complete) {
-  zookeeper_resolver *r = (zookeeper_resolver *)resolver;
-  gpr_mu_lock(&r->mu);
-  GPR_ASSERT(r->next_completion == NULL);
-  r->next_completion = on_complete;
-  r->target_config = target_config;
-  if (r->resolved_version == 0 && r->resolving == 0) {
-    zookeeper_start_resolving_locked(r);
-  } else {
-    zookeeper_maybe_finish_next_locked(exec_ctx, r);
-  }
-  gpr_mu_unlock(&r->mu);
-}
-
-/** Zookeeper global watcher for connection management
-    TODO: better connection management besides logs */
-static void zookeeper_global_watcher(zhandle_t *zookeeper_handle, int type,
-                                     int state, const char *path,
-                                     void *watcher_ctx) {
-  if (type == ZOO_SESSION_EVENT) {
-    if (state == ZOO_EXPIRED_SESSION_STATE) {
-      gpr_log(GPR_ERROR, "Zookeeper session expired");
-    } else if (state == ZOO_AUTH_FAILED_STATE) {
-      gpr_log(GPR_ERROR, "Zookeeper authentication failed");
-    }
-  }
-}
-
-/** Zookeeper watcher triggered by changes to watched nodes
-    Once triggered, it tries to resolve again to get updated addresses */
-static void zookeeper_watcher(zhandle_t *zookeeper_handle, int type, int state,
-                              const char *path, void *watcher_ctx) {
-  if (watcher_ctx != NULL) {
-    zookeeper_resolver *r = (zookeeper_resolver *)watcher_ctx;
-    if (state == ZOO_CONNECTED_STATE) {
-      gpr_mu_lock(&r->mu);
-      if (r->resolving == 0) {
-        zookeeper_start_resolving_locked(r);
-      }
-      gpr_mu_unlock(&r->mu);
-    }
-  }
-}
-
-/** Callback function after getting all resolved addresses
-    Creates a subchannel for each address */
-static void zookeeper_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
-                                  grpc_resolved_addresses *addresses) {
-  zookeeper_resolver *r = arg;
-  grpc_client_config *config = NULL;
-  grpc_lb_policy *lb_policy;
-
-  if (addresses != NULL) {
-    grpc_lb_policy_args lb_policy_args;
-    config = grpc_client_config_create();
-    lb_policy_args.addresses = addresses;
-    lb_policy_args.client_channel_factory = r->client_channel_factory;
-    lb_policy =
-        grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args);
-
-    if (lb_policy != NULL) {
-      grpc_client_config_set_lb_policy(config, lb_policy);
-      GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction");
-    }
-    grpc_resolved_addresses_destroy(addresses);
-  }
-  gpr_mu_lock(&r->mu);
-  GPR_ASSERT(r->resolving == 1);
-  r->resolving = 0;
-  if (r->resolved_config != NULL) {
-    grpc_client_config_unref(exec_ctx, r->resolved_config);
-  }
-  r->resolved_config = config;
-  r->resolved_version++;
-  zookeeper_maybe_finish_next_locked(exec_ctx, r);
-  gpr_mu_unlock(&r->mu);
-
-  GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "zookeeper-resolving");
-}
-
-/** Callback function for each DNS resolved address */
-static void zookeeper_dns_resolved(grpc_exec_ctx *exec_ctx, void *arg,
-                                   grpc_resolved_addresses *addresses) {
-  size_t i;
-  zookeeper_resolver *r = arg;
-  int resolve_done = 0;
-
-  gpr_mu_lock(&r->mu);
-  r->resolved_num++;
-  r->resolved_addrs->addrs =
-      gpr_realloc(r->resolved_addrs->addrs,
-                  sizeof(grpc_resolved_address) *
-                      (r->resolved_addrs->naddrs + addresses->naddrs));
-  for (i = 0; i < addresses->naddrs; i++) {
-    memcpy(r->resolved_addrs->addrs[i + r->resolved_addrs->naddrs].addr,
-           addresses->addrs[i].addr, addresses->addrs[i].len);
-    r->resolved_addrs->addrs[i + r->resolved_addrs->naddrs].len =
-        addresses->addrs[i].len;
-  }
-
-  r->resolved_addrs->naddrs += addresses->naddrs;
-  grpc_resolved_addresses_destroy(addresses);
-
-  /** Wait for all addresses to be resolved */
-  resolve_done = (r->resolved_num == r->resolved_total);
-  gpr_mu_unlock(&r->mu);
-  if (resolve_done) {
-    zookeeper_on_resolved(exec_ctx, r, r->resolved_addrs);
-  }
-}
-
-/** Parses JSON format address of a zookeeper node */
-static char *zookeeper_parse_address(const char *value, size_t value_len) {
-  grpc_json *json;
-  grpc_json *cur;
-  const char *host;
-  const char *port;
-  char *buffer;
-  char *address = NULL;
-
-  buffer = gpr_malloc(value_len);
-  memcpy(buffer, value, value_len);
-  json = grpc_json_parse_string_with_len(buffer, value_len);
-  if (json != NULL) {
-    host = NULL;
-    port = NULL;
-    for (cur = json->child; cur != NULL; cur = cur->next) {
-      if (!strcmp(cur->key, "host")) {
-        host = cur->value;
-        if (port != NULL) {
-          break;
-        }
-      } else if (!strcmp(cur->key, "port")) {
-        port = cur->value;
-        if (host != NULL) {
-          break;
-        }
-      }
-    }
-    if (host != NULL && port != NULL) {
-      gpr_asprintf(&address, "%s:%s", host, port);
-    }
-    grpc_json_destroy(json);
-  }
-  gpr_free(buffer);
-
-  return address;
-}
-
-static void zookeeper_get_children_node_completion(int rc, const char *value,
-                                                   int value_len,
-                                                   const struct Stat *stat,
-                                                   const void *arg) {
-  char *address = NULL;
-  zookeeper_resolver *r = (zookeeper_resolver *)arg;
-  int resolve_done = 0;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-
-  if (rc != 0) {
-    gpr_log(GPR_ERROR, "Error in getting a child node of %s", r->name);
-    grpc_exec_ctx_finish(&exec_ctx);
-    return;
-  }
-
-  address = zookeeper_parse_address(value, (size_t)value_len);
-  if (address != NULL) {
-    /** Further resolves address by DNS */
-    grpc_resolve_address(&exec_ctx, address, NULL, zookeeper_dns_resolved, r);
-    gpr_free(address);
-  } else {
-    gpr_log(GPR_ERROR, "Error in resolving a child node of %s", r->name);
-    gpr_mu_lock(&r->mu);
-    r->resolved_total--;
-    resolve_done = (r->resolved_num == r->resolved_total);
-    gpr_mu_unlock(&r->mu);
-    if (resolve_done) {
-      zookeeper_on_resolved(&exec_ctx, r, r->resolved_addrs);
-    }
-  }
-
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void zookeeper_get_children_completion(
-    int rc, const struct String_vector *children, const void *arg) {
-  char *path;
-  int status;
-  int i;
-  zookeeper_resolver *r = (zookeeper_resolver *)arg;
-
-  if (rc != 0) {
-    gpr_log(GPR_ERROR, "Error in getting zookeeper children of %s", r->name);
-    return;
-  }
-
-  if (children->count == 0) {
-    gpr_log(GPR_ERROR, "Error in resolving zookeeper address %s", r->name);
-    return;
-  }
-
-  r->resolved_addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
-  r->resolved_addrs->addrs = NULL;
-  r->resolved_addrs->naddrs = 0;
-  r->resolved_total = children->count;
-
-  /** TODO: Replace expensive heap allocation with stack
-      if we can get maximum length of zookeeper path */
-  for (i = 0; i < children->count; i++) {
-    gpr_asprintf(&path, "%s/%s", r->name, children->data[i]);
-    status = zoo_awget(r->zookeeper_handle, path, zookeeper_watcher, r,
-                       zookeeper_get_children_node_completion, r);
-    gpr_free(path);
-    if (status != 0) {
-      gpr_log(GPR_ERROR, "Error in getting zookeeper node %s", path);
-    }
-  }
-}
-
-static void zookeeper_get_node_completion(int rc, const char *value,
-                                          int value_len,
-                                          const struct Stat *stat,
-                                          const void *arg) {
-  int status;
-  char *address = NULL;
-  zookeeper_resolver *r = (zookeeper_resolver *)arg;
-  r->resolved_addrs = NULL;
-  r->resolved_total = 0;
-  r->resolved_num = 0;
-
-  if (rc != 0) {
-    gpr_log(GPR_ERROR, "Error in getting zookeeper node %s", r->name);
-    return;
-  }
-
-  /** If zookeeper node of path r->name does not have address
-      (i.e. service node), get its children */
-  address = zookeeper_parse_address(value, (size_t)value_len);
-  if (address != NULL) {
-    r->resolved_addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
-    r->resolved_addrs->addrs = NULL;
-    r->resolved_addrs->naddrs = 0;
-    r->resolved_total = 1;
-    /** Further resolves address by DNS */
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-    grpc_resolve_address(&exec_ctx, address, NULL, zookeeper_dns_resolved, r);
-    gpr_free(address);
-    grpc_exec_ctx_finish(&exec_ctx);
-    return;
-  }
-
-  status = zoo_awget_children(r->zookeeper_handle, r->name, zookeeper_watcher,
-                              r, zookeeper_get_children_completion, r);
-  if (status != 0) {
-    gpr_log(GPR_ERROR, "Error in getting zookeeper children of %s", r->name);
-  }
-}
-
-static void zookeeper_resolve_address(zookeeper_resolver *r) {
-  int status;
-  status = zoo_awget(r->zookeeper_handle, r->name, zookeeper_watcher, r,
-                     zookeeper_get_node_completion, r);
-  if (status != 0) {
-    gpr_log(GPR_ERROR, "Error in getting zookeeper node %s", r->name);
-  }
-}
-
-static void zookeeper_start_resolving_locked(zookeeper_resolver *r) {
-  GRPC_RESOLVER_REF(&r->base, "zookeeper-resolving");
-  GPR_ASSERT(r->resolving == 0);
-  r->resolving = 1;
-  zookeeper_resolve_address(r);
-}
-
-static void zookeeper_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
-                                               zookeeper_resolver *r) {
-  if (r->next_completion != NULL &&
-      r->resolved_version != r->published_version) {
-    *r->target_config = r->resolved_config;
-    if (r->resolved_config != NULL) {
-      grpc_client_config_ref(r->resolved_config);
-    }
-    grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL);
-    r->next_completion = NULL;
-    r->published_version = r->resolved_version;
-  }
-}
-
-static void zookeeper_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) {
-  zookeeper_resolver *r = (zookeeper_resolver *)gr;
-  gpr_mu_destroy(&r->mu);
-  if (r->resolved_config != NULL) {
-    grpc_client_config_unref(exec_ctx, r->resolved_config);
-  }
-  grpc_client_channel_factory_unref(exec_ctx, r->client_channel_factory);
-  gpr_free(r->name);
-  gpr_free(r->lb_policy_name);
-  gpr_free(r);
-}
-
-static grpc_resolver *zookeeper_create(grpc_resolver_args *args,
-                                       const char *lb_policy_name) {
-  zookeeper_resolver *r;
-  size_t length;
-  char *path = args->uri->path;
-
-  if (0 == strcmp(args->uri->authority, "")) {
-    gpr_log(GPR_ERROR, "No authority specified in zookeeper uri");
-    return NULL;
-  }
-
-  /** Removes the trailing slash if exists */
-  length = strlen(path);
-  if (length > 1 && path[length - 1] == '/') {
-    path[length - 1] = 0;
-  }
-
-  r = gpr_malloc(sizeof(zookeeper_resolver));
-  memset(r, 0, sizeof(*r));
-  gpr_ref_init(&r->refs, 1);
-  gpr_mu_init(&r->mu);
-  grpc_resolver_init(&r->base, &zookeeper_resolver_vtable);
-  r->name = gpr_strdup(path);
-
-  r->client_channel_factory = args->client_channel_factory;
-  grpc_client_channel_factory_ref(r->client_channel_factory);
-
-  r->lb_policy_name = gpr_strdup(lb_policy_name);
-
-  /** Initializes zookeeper client */
-  zoo_set_debug_level(ZOO_LOG_LEVEL_WARN);
-  r->zookeeper_handle =
-      zookeeper_init(args->uri->authority, zookeeper_global_watcher,
-                     GRPC_ZOOKEEPER_SESSION_TIMEOUT, 0, 0, 0);
-  if (r->zookeeper_handle == NULL) {
-    gpr_log(GPR_ERROR, "Unable to connect to zookeeper server");
-    return NULL;
-  }
-
-  return &r->base;
-}
-
-/*
- * FACTORY
- */
-
-static void zookeeper_factory_ref(grpc_resolver_factory *factory) {}
-
-static void zookeeper_factory_unref(grpc_resolver_factory *factory) {}
-
-static char *zookeeper_factory_get_default_hostname(
-    grpc_resolver_factory *factory, grpc_uri *uri) {
-  return NULL;
-}
-
-static grpc_resolver *zookeeper_factory_create_resolver(
-    grpc_resolver_factory *factory, grpc_resolver_args *args) {
-  return zookeeper_create(args, "pick_first");
-}
-
-static const grpc_resolver_factory_vtable zookeeper_factory_vtable = {
-    zookeeper_factory_ref, zookeeper_factory_unref,
-    zookeeper_factory_create_resolver, zookeeper_factory_get_default_hostname,
-    "zookeeper"};
-
-static grpc_resolver_factory zookeeper_resolver_factory = {
-    &zookeeper_factory_vtable};
-
-static grpc_resolver_factory *zookeeper_resolver_factory_create() {
-  return &zookeeper_resolver_factory;
-}
-
-static void zookeeper_plugin_init() {
-  grpc_register_resolver_type(zookeeper_resolver_factory_create());
-}
-
-void grpc_zookeeper_register() {
-  GRPC_API_TRACE("grpc_zookeeper_register(void)", 0, ());
-  grpc_register_plugin(zookeeper_plugin_init, NULL);
-}
diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c
index c5d3d8d9cc..85f9efb3b6 100644
--- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c
+++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c
@@ -79,11 +79,11 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
 }
 
 static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
-                                           bool success) {
+                                           grpc_error *error) {
   connector_unref(exec_ctx, arg);
 }
 
-static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   connector *c = arg;
   grpc_closure *notify;
   grpc_endpoint *tcp = c->tcp;
@@ -103,13 +103,13 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL,
                                         0);
     GPR_ASSERT(c->result->transport);
-    c->result->channel_args = c->args.channel_args;
+    c->result->channel_args = grpc_channel_args_copy(c->args.channel_args);
   } else {
     memset(c->result, 0, sizeof(*c->result));
   }
   notify = c->notify;
   c->notify = NULL;
-  notify->cb(exec_ctx, notify->cb_arg, 1);
+  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
 }
 
 static void connector_shutdown(grpc_exec_ctx *exec_ctx, grpc_connector *con) {}
diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
new file mode 100644
index 0000000000..ca435c25ce
--- /dev/null
+++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
@@ -0,0 +1,95 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_SUPPORT_CHANNELS_FROM_FD
+
+#include <fcntl.h>
+
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/transport.h"
+
+grpc_channel *grpc_insecure_channel_create_from_fd(
+    const char *target, int fd, const grpc_channel_args *args) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GRPC_API_TRACE("grpc_insecure_channel_create(target=%p, fd=%d, args=%p)", 3,
+                 (target, fd, args));
+
+  grpc_arg default_authority_arg;
+  default_authority_arg.type = GRPC_ARG_STRING;
+  default_authority_arg.key = GRPC_ARG_DEFAULT_AUTHORITY;
+  default_authority_arg.value.string = "test.authority";
+  grpc_channel_args *final_args =
+      grpc_channel_args_copy_and_add(args, &default_authority_arg, 1);
+
+  int flags = fcntl(fd, F_GETFL, 0);
+  GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0);
+
+  grpc_endpoint *client =
+      grpc_tcp_create(grpc_fd_create(fd, "client"),
+                      GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "fd-client");
+
+  grpc_transport *transport =
+      grpc_create_chttp2_transport(&exec_ctx, final_args, client, 1);
+  GPR_ASSERT(transport);
+  grpc_channel *channel = grpc_channel_create(
+      &exec_ctx, target, final_args, GRPC_CLIENT_DIRECT_CHANNEL, transport);
+  grpc_channel_args_destroy(final_args);
+  grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0);
+
+  grpc_exec_ctx_finish(&exec_ctx);
+
+  return channel != NULL ? channel : grpc_lame_client_channel_create(
+                                         target, GRPC_STATUS_INTERNAL,
+                                         "Failed to create client channel");
+}
+
+#else  // !GPR_SUPPORT_CHANNELS_FROM_FD
+
+grpc_channel *grpc_insecure_channel_create_from_fd(
+    const char *target, int fd, const grpc_channel_args *args) {
+  GPR_ASSERT(0);
+  return NULL;
+}
+
+#endif  // GPR_SUPPORT_CHANNELS_FROM_FD
diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
index 58af6f995a..721ba82d8f 100644
--- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
+++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
@@ -45,9 +45,9 @@
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/tcp_client.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
@@ -90,7 +90,6 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
                                      grpc_auth_context *auth_context) {
   connector *c = arg;
   grpc_closure *notify;
-  grpc_channel_args *args_copy = NULL;
   gpr_mu_lock(&c->mu);
   if (c->connecting_endpoint == NULL) {
     memset(c->result, 0, sizeof(*c->result));
@@ -109,26 +108,23 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL,
                                         0);
     auth_context_arg = grpc_auth_context_to_arg(auth_context);
-    args_copy = grpc_channel_args_copy_and_add(c->args.channel_args,
-                                               &auth_context_arg, 1);
-    c->result->channel_args = args_copy;
+    c->result->channel_args = grpc_channel_args_copy_and_add(
+        c->args.channel_args, &auth_context_arg, 1);
   }
   notify = c->notify;
   c->notify = NULL;
-  /* look at c->args which are connector args. */
-  notify->cb(exec_ctx, notify->cb_arg, 1);
-  if (args_copy != NULL) grpc_channel_args_destroy(args_copy);
+  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_NONE, NULL);
 }
 
 static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
-                                           bool success) {
+                                           grpc_error *error) {
   connector *c = arg;
-  grpc_channel_security_connector_do_handshake(exec_ctx, c->security_connector,
-                                               c->connecting_endpoint,
-                                               on_secure_handshake_done, c);
+  grpc_channel_security_connector_do_handshake(
+      exec_ctx, c->security_connector, c->connecting_endpoint, c->args.deadline,
+      on_secure_handshake_done, c);
 }
 
-static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   connector *c = arg;
   grpc_closure *notify;
   grpc_endpoint *tcp = c->newly_connecting_endpoint;
@@ -147,13 +143,14 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
                           &c->initial_string_sent);
     } else {
       grpc_channel_security_connector_do_handshake(
-          exec_ctx, c->security_connector, tcp, on_secure_handshake_done, c);
+          exec_ctx, c->security_connector, tcp, c->args.deadline,
+          on_secure_handshake_done, c);
     }
   } else {
     memset(c->result, 0, sizeof(*c->result));
     notify = c->notify;
     c->notify = NULL;
-    notify->cb(exec_ctx, notify->cb_arg, 1);
+    grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
   }
 }
 
@@ -175,7 +172,6 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con,
                               grpc_closure *notify) {
   connector *c = (connector *)con;
   GPR_ASSERT(c->notify == NULL);
-  GPR_ASSERT(notify->cb);
   c->notify = notify;
   c->args = *args;
   c->result = result;
diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
index e21fa2a072..e5c987925c 100644
--- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -35,6 +35,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/http_server_filter.h"
@@ -43,14 +44,8 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-static void setup_transport(grpc_exec_ctx *exec_ctx, void *server,
-                            grpc_transport *transport) {
-  grpc_server_setup_transport(exec_ctx, server, transport,
-                              grpc_server_get_channel_args(server));
-}
-
 static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
-                          grpc_endpoint *tcp,
+                          grpc_endpoint *tcp, grpc_pollset *accepting_pollset,
                           grpc_tcp_server_acceptor *acceptor) {
   /*
    * Beware that the call to grpc_create_chttp2_transport() has to happen before
@@ -61,7 +56,8 @@ static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
    */
   grpc_transport *transport = grpc_create_chttp2_transport(
       exec_ctx, grpc_server_get_channel_args(server), tcp, 0);
-  setup_transport(exec_ctx, server, transport);
+  grpc_server_setup_transport(exec_ctx, server, transport, accepting_pollset,
+                              grpc_server_get_channel_args(server));
   grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
 }
 
@@ -79,34 +75,41 @@ static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
                     grpc_closure *destroy_done) {
   grpc_tcp_server *tcp = tcpp;
   grpc_tcp_server_unref(exec_ctx, tcp);
-  grpc_exec_ctx_enqueue(exec_ctx, destroy_done, true, NULL);
+  grpc_exec_ctx_sched(exec_ctx, destroy_done, GRPC_ERROR_NONE, NULL);
 }
 
 int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
   size_t i;
-  unsigned count = 0;
+  size_t count = 0;
   int port_num = -1;
   int port_temp;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_error *err = GRPC_ERROR_NONE;
 
   GRPC_API_TRACE("grpc_server_add_insecure_http2_port(server=%p, addr=%s)", 2,
                  (server, addr));
 
-  resolved = grpc_blocking_resolve_address(addr, "http");
-  if (!resolved) {
+  grpc_error **errors = NULL;
+  err = grpc_blocking_resolve_address(addr, "https", &resolved);
+  if (err != GRPC_ERROR_NONE) {
     goto error;
   }
 
-  tcp = grpc_tcp_server_create(NULL);
-  GPR_ASSERT(tcp);
+  err =
+      grpc_tcp_server_create(NULL, grpc_server_get_channel_args(server), &tcp);
+  if (err != GRPC_ERROR_NONE) {
+    goto error;
+  }
 
-  for (i = 0; i < resolved->naddrs; i++) {
-    port_temp = grpc_tcp_server_add_port(
+  const size_t naddrs = resolved->naddrs;
+  errors = gpr_malloc(sizeof(*errors) * naddrs);
+  for (i = 0; i < naddrs; i++) {
+    errors[i] = grpc_tcp_server_add_port(
         tcp, (struct sockaddr *)&resolved->addrs[i].addr,
-        resolved->addrs[i].len);
-    if (port_temp > 0) {
+        resolved->addrs[i].len, &port_temp);
+    if (errors[i] == GRPC_ERROR_NONE) {
       if (port_num == -1) {
         port_num = port_temp;
       } else {
@@ -116,13 +119,24 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
     }
   }
   if (count == 0) {
-    gpr_log(GPR_ERROR, "No address added out of total %d resolved",
-            resolved->naddrs);
+    char *msg;
+    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
+                 naddrs);
+    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
+    gpr_free(msg);
     goto error;
-  }
-  if (count != resolved->naddrs) {
-    gpr_log(GPR_ERROR, "Only %d addresses added out of total %d resolved",
-            count, resolved->naddrs);
+  } else if (count != naddrs) {
+    char *msg;
+    gpr_asprintf(&msg, "Only %" PRIuPTR
+                       " addresses added out of total %" PRIuPTR " resolved",
+                 count, naddrs);
+    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
+    gpr_free(msg);
+
+    const char *warning_message = grpc_error_string(err);
+    gpr_log(GPR_INFO, "WARNING: %s", warning_message);
+    grpc_error_free_string(warning_message);
+    /* we managed to bind some addresses: continue */
   }
   grpc_resolved_addresses_destroy(resolved);
 
@@ -132,6 +146,7 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
 
 /* Error path: cleanup and return */
 error:
+  GPR_ASSERT(err != GRPC_ERROR_NONE);
   if (resolved) {
     grpc_resolved_addresses_destroy(resolved);
   }
@@ -140,7 +155,18 @@ error:
   }
   port_num = 0;
 
+  const char *msg = grpc_error_string(err);
+  gpr_log(GPR_ERROR, "%s", msg);
+  grpc_error_free_string(msg);
+  GRPC_ERROR_UNREF(err);
+
 done:
   grpc_exec_ctx_finish(&exec_ctx);
+  if (errors != NULL) {
+    for (i = 0; i < naddrs; i++) {
+      GRPC_ERROR_UNREF(errors[i]);
+    }
+  }
+  gpr_free(errors);
   return port_num;
 }
diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
new file mode 100644
index 0000000000..96bf4d6f30
--- /dev/null
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
@@ -0,0 +1,82 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_SUPPORT_CHANNELS_FROM_FD
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/server.h"
+
+void grpc_server_add_insecure_channel_from_fd(grpc_server *server,
+                                              grpc_completion_queue *cq,
+                                              int fd) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+  char *name;
+  gpr_asprintf(&name, "fd:%d", fd);
+
+  grpc_endpoint *server_endpoint = grpc_tcp_create(
+      grpc_fd_create(fd, name), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name);
+
+  gpr_free(name);
+
+  const grpc_channel_args *server_args = grpc_server_get_channel_args(server);
+  grpc_transport *transport = grpc_create_chttp2_transport(
+      &exec_ctx, server_args, server_endpoint, 0 /* is_client */);
+  grpc_endpoint_add_to_pollset(&exec_ctx, server_endpoint, grpc_cq_pollset(cq));
+  grpc_server_setup_transport(&exec_ctx, server, transport, NULL, server_args);
+  grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+#else  // !GPR_SUPPORT_CHANNELS_FROM_FD
+
+void grpc_server_add_insecure_channel_from_fd(grpc_server *server,
+                                              grpc_completion_queue *cq,
+                                              int fd) {
+  GPR_ASSERT(0);
+}
+
+#endif  // GPR_SUPPORT_CHANNELS_FROM_FD
diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index 698b2bef61..c42810e913 100644
--- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -37,6 +37,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
@@ -45,14 +46,14 @@
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/tcp_server.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-typedef struct grpc_server_secure_state {
+typedef struct server_secure_state {
   grpc_server *server;
   grpc_tcp_server *tcp;
   grpc_server_security_connector *sc;
@@ -62,13 +63,16 @@ typedef struct grpc_server_secure_state {
   gpr_refcount refcount;
   grpc_closure destroy_closure;
   grpc_closure *destroy_callback;
-} grpc_server_secure_state;
+} server_secure_state;
 
-static void state_ref(grpc_server_secure_state *state) {
-  gpr_ref(&state->refcount);
-}
+typedef struct server_secure_connect {
+  server_secure_state *state;
+  grpc_pollset *accepting_pollset;
+} server_secure_connect;
+
+static void state_ref(server_secure_state *state) { gpr_ref(&state->refcount); }
 
-static void state_unref(grpc_server_secure_state *state) {
+static void state_unref(server_secure_state *state) {
   if (gpr_unref(&state->refcount)) {
     /* ensure all threads have unlocked */
     gpr_mu_lock(&state->mu);
@@ -80,70 +84,72 @@ static void state_unref(grpc_server_secure_state *state) {
   }
 }
 
-static void setup_transport(grpc_exec_ctx *exec_ctx, void *statep,
-                            grpc_transport *transport,
-                            grpc_auth_context *auth_context) {
-  grpc_server_secure_state *state = statep;
-  grpc_channel_args *args_copy;
-  grpc_arg args_to_add[2];
-  args_to_add[0] = grpc_server_credentials_to_arg(state->creds);
-  args_to_add[1] = grpc_auth_context_to_arg(auth_context);
-  args_copy = grpc_channel_args_copy_and_add(
-      grpc_server_get_channel_args(state->server), args_to_add,
-      GPR_ARRAY_SIZE(args_to_add));
-  grpc_server_setup_transport(exec_ctx, state->server, transport, args_copy);
-  grpc_channel_args_destroy(args_copy);
-}
-
 static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
                                      grpc_security_status status,
                                      grpc_endpoint *secure_endpoint,
                                      grpc_auth_context *auth_context) {
-  grpc_server_secure_state *state = statep;
+  server_secure_connect *state = statep;
   grpc_transport *transport;
   if (status == GRPC_SECURITY_OK) {
     if (secure_endpoint) {
-      gpr_mu_lock(&state->mu);
-      if (!state->is_shutdown) {
+      gpr_mu_lock(&state->state->mu);
+      if (!state->state->is_shutdown) {
         transport = grpc_create_chttp2_transport(
-            exec_ctx, grpc_server_get_channel_args(state->server),
+            exec_ctx, grpc_server_get_channel_args(state->state->server),
             secure_endpoint, 0);
-        setup_transport(exec_ctx, state, transport, auth_context);
+        grpc_channel_args *args_copy;
+        grpc_arg args_to_add[2];
+        args_to_add[0] = grpc_server_credentials_to_arg(state->state->creds);
+        args_to_add[1] = grpc_auth_context_to_arg(auth_context);
+        args_copy = grpc_channel_args_copy_and_add(
+            grpc_server_get_channel_args(state->state->server), args_to_add,
+            GPR_ARRAY_SIZE(args_to_add));
+        grpc_server_setup_transport(exec_ctx, state->state->server, transport,
+                                    state->accepting_pollset, args_copy);
+        grpc_channel_args_destroy(args_copy);
         grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
       } else {
         /* We need to consume this here, because the server may already have
          * gone away. */
         grpc_endpoint_destroy(exec_ctx, secure_endpoint);
       }
-      gpr_mu_unlock(&state->mu);
+      gpr_mu_unlock(&state->state->mu);
     }
   } else {
     gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
   }
-  state_unref(state);
+  state_unref(state->state);
+  gpr_free(state);
 }
 
 static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
+                      grpc_pollset *accepting_pollset,
                       grpc_tcp_server_acceptor *acceptor) {
-  grpc_server_secure_state *state = statep;
-  state_ref(state);
+  server_secure_connect *state = gpr_malloc(sizeof(*state));
+  state->state = statep;
+  state_ref(state->state);
+  state->accepting_pollset = accepting_pollset;
   grpc_server_security_connector_do_handshake(
-      exec_ctx, state->sc, acceptor, tcp, on_secure_handshake_done, state);
+      exec_ctx, state->state->sc, acceptor, tcp,
+      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                   gpr_time_from_seconds(120, GPR_TIMESPAN)),
+      on_secure_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
 static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
                   grpc_pollset **pollsets, size_t pollset_count) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   grpc_tcp_server_start(exec_ctx, state->tcp, pollsets, pollset_count,
                         on_accept, state);
 }
 
-static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
-  grpc_server_secure_state *state = statep;
+static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep,
+                         grpc_error *error) {
+  server_secure_state *state = statep;
   if (state->destroy_callback != NULL) {
     state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg,
-                                success);
+                                GRPC_ERROR_REF(error));
   }
   grpc_server_security_connector_shutdown(exec_ctx, state->sc);
   state_unref(state);
@@ -153,7 +159,7 @@ static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
    callbacks) */
 static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
                     grpc_closure *callback) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   grpc_tcp_server *tcp;
   gpr_mu_lock(&state->mu);
   state->is_shutdown = 1;
@@ -167,14 +173,16 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
                                       grpc_server_credentials *creds) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
-  grpc_server_secure_state *state = NULL;
+  server_secure_state *state = NULL;
   size_t i;
-  unsigned count = 0;
+  size_t count = 0;
   int port_num = -1;
   int port_temp;
   grpc_security_status status = GRPC_SECURITY_ERROR;
   grpc_server_security_connector *sc = NULL;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_error *err = GRPC_ERROR_NONE;
+  grpc_error **errors = NULL;
 
   GRPC_API_TRACE(
       "grpc_server_add_secure_http2_port("
@@ -182,26 +190,35 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
       3, (server, addr, creds));
 
   /* create security context */
-  if (creds == NULL) goto error;
+  if (creds == NULL) {
+    err = GRPC_ERROR_CREATE(
+        "No credentials specified for secure server port (creds==NULL)");
+    goto error;
+  }
   status = grpc_server_credentials_create_security_connector(creds, &sc);
   if (status != GRPC_SECURITY_OK) {
-    gpr_log(GPR_ERROR,
-            "Unable to create secure server with credentials of type %s.",
-            creds->type);
+    char *msg;
+    gpr_asprintf(&msg,
+                 "Unable to create secure server with credentials of type %s.",
+                 creds->type);
+    err = grpc_error_set_int(GRPC_ERROR_CREATE(msg),
+                             GRPC_ERROR_INT_SECURITY_STATUS, status);
+    gpr_free(msg);
     goto error;
   }
   sc->channel_args = grpc_server_get_channel_args(server);
 
   /* resolve address */
-  resolved = grpc_blocking_resolve_address(addr, "https");
-  if (!resolved) {
+  err = grpc_blocking_resolve_address(addr, "https", &resolved);
+  if (err != GRPC_ERROR_NONE) {
     goto error;
   }
   state = gpr_malloc(sizeof(*state));
   memset(state, 0, sizeof(*state));
   grpc_closure_init(&state->destroy_closure, destroy_done, state);
-  tcp = grpc_tcp_server_create(&state->destroy_closure);
-  if (!tcp) {
+  err = grpc_tcp_server_create(&state->destroy_closure,
+                               grpc_server_get_channel_args(server), &tcp);
+  if (err != GRPC_ERROR_NONE) {
     goto error;
   }
 
@@ -213,11 +230,12 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   gpr_mu_init(&state->mu);
   gpr_ref_init(&state->refcount, 1);
 
+  errors = gpr_malloc(sizeof(*errors) * resolved->naddrs);
   for (i = 0; i < resolved->naddrs; i++) {
-    port_temp = grpc_tcp_server_add_port(
+    errors[i] = grpc_tcp_server_add_port(
         tcp, (struct sockaddr *)&resolved->addrs[i].addr,
-        resolved->addrs[i].len);
-    if (port_temp > 0) {
+        resolved->addrs[i].len, &port_temp);
+    if (errors[i] == GRPC_ERROR_NONE) {
       if (port_num == -1) {
         port_num = port_temp;
       } else {
@@ -227,15 +245,31 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
     }
   }
   if (count == 0) {
-    gpr_log(GPR_ERROR, "No address added out of total %d resolved",
-            resolved->naddrs);
+    char *msg;
+    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
+                 resolved->naddrs);
+    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, resolved->naddrs);
+    gpr_free(msg);
     goto error;
+  } else if (count != resolved->naddrs) {
+    char *msg;
+    gpr_asprintf(&msg, "Only %" PRIuPTR
+                       " addresses added out of total %" PRIuPTR " resolved",
+                 count, resolved->naddrs);
+    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, resolved->naddrs);
+    gpr_free(msg);
+
+    const char *warning_message = grpc_error_string(err);
+    gpr_log(GPR_INFO, "WARNING: %s", warning_message);
+    grpc_error_free_string(warning_message);
+    /* we managed to bind some addresses: continue */
+  } else {
+    for (i = 0; i < resolved->naddrs; i++) {
+      GRPC_ERROR_UNREF(errors[i]);
+    }
   }
-  if (count != resolved->naddrs) {
-    gpr_log(GPR_ERROR, "Only %d addresses added out of total %d resolved",
-            count, resolved->naddrs);
-    /* if it's an error, don't we want to goto error; here ? */
-  }
+  gpr_free(errors);
+  errors = NULL;
   grpc_resolved_addresses_destroy(resolved);
 
   /* Register with the server only upon success */
@@ -246,6 +280,13 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
 
 /* Error path: cleanup and return */
 error:
+  GPR_ASSERT(err != GRPC_ERROR_NONE);
+  if (errors != NULL) {
+    for (i = 0; i < resolved->naddrs; i++) {
+      GRPC_ERROR_UNREF(errors[i]);
+    }
+    gpr_free(errors);
+  }
   if (resolved) {
     grpc_resolved_addresses_destroy(resolved);
   }
@@ -260,5 +301,9 @@ error:
     }
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  const char *msg = grpc_error_string(err);
+  GRPC_ERROR_UNREF(err);
+  gpr_log(GPR_ERROR, "%s", msg);
+  grpc_error_free_string(msg);
   return 0;
 }
diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.c b/src/core/ext/transport/chttp2/transport/bin_decoder.c
new file mode 100644
index 0000000000..2d90b01cd8
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/bin_decoder.c
@@ -0,0 +1,232 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/ext/transport/chttp2/transport/bin_decoder.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include "src/core/lib/support/string.h"
+
+static uint8_t decode_table[] = {
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 62,   0x40, 0x40, 0x40, 63,
+    52,   53,   54,   55,   56,   57,   58,   59,   60,   61,   0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0,    1,    2,    3,    4,    5,    6,
+    7,    8,    9,    10,   11,   12,   13,   14,   15,   16,   17,   18,
+    19,   20,   21,   22,   23,   24,   25,   0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 26,   27,   28,   29,   30,   31,   32,   33,   34,   35,   36,
+    37,   38,   39,   40,   41,   42,   43,   44,   45,   46,   47,   48,
+    49,   50,   51,   0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+    0x40, 0x40, 0x40, 0x40};
+
+static const uint8_t tail_xtra[4] = {0, 0, 1, 2};
+
+static bool input_is_valid(uint8_t *input_ptr, size_t length) {
+  size_t i;
+
+  for (i = 0; i < length; ++i) {
+    if ((decode_table[input_ptr[i]] & 0xC0) != 0) {
+      gpr_log(GPR_ERROR,
+              "Base64 decoding failed, invalid character '%c' in base64 "
+              "input.\n",
+              (char)(*input_ptr));
+      return false;
+    }
+  }
+  return true;
+}
+
+#define COMPOSE_OUTPUT_BYTE_0(input_ptr)        \
+  (uint8_t)((decode_table[input_ptr[0]] << 2) | \
+            (decode_table[input_ptr[1]] >> 4))
+
+#define COMPOSE_OUTPUT_BYTE_1(input_ptr)        \
+  (uint8_t)((decode_table[input_ptr[1]] << 4) | \
+            (decode_table[input_ptr[2]] >> 2))
+
+#define COMPOSE_OUTPUT_BYTE_2(input_ptr) \
+  (uint8_t)((decode_table[input_ptr[2]] << 6) | decode_table[input_ptr[3]])
+
+bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx) {
+  size_t input_tail;
+
+  if (ctx->input_cur > ctx->input_end || ctx->output_cur > ctx->output_end) {
+    return false;
+  }
+
+  // Process a block of 4 input characters and 3 output bytes
+  while (ctx->input_end >= ctx->input_cur + 4 &&
+         ctx->output_end >= ctx->output_cur + 3) {
+    if (!input_is_valid(ctx->input_cur, 4)) return false;
+    ctx->output_cur[0] = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+    ctx->output_cur[1] = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+    ctx->output_cur[2] = COMPOSE_OUTPUT_BYTE_2(ctx->input_cur);
+    ctx->output_cur += 3;
+    ctx->input_cur += 4;
+  }
+
+  // Process the tail of input data
+  input_tail = (size_t)(ctx->input_end - ctx->input_cur);
+  if (input_tail == 4) {
+    // Process the input data with pad chars
+    if (ctx->input_cur[3] == '=') {
+      if (ctx->input_cur[2] == '=' && ctx->output_end >= ctx->output_cur + 1) {
+        if (!input_is_valid(ctx->input_cur, 2)) return false;
+        *(ctx->output_cur++) = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+        ctx->input_cur += 4;
+      } else if (ctx->output_end >= ctx->output_cur + 2) {
+        if (!input_is_valid(ctx->input_cur, 3)) return false;
+        *(ctx->output_cur++) = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+        *(ctx->output_cur++) = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+        ;
+        ctx->input_cur += 4;
+      }
+    }
+
+  } else if (ctx->contains_tail && input_tail > 1) {
+    // Process the input data without pad chars, but constains_tail is set
+    if (ctx->output_end >= ctx->output_cur + tail_xtra[input_tail]) {
+      if (!input_is_valid(ctx->input_cur, input_tail)) return false;
+      switch (input_tail) {
+        case 3:
+          ctx->output_cur[1] = COMPOSE_OUTPUT_BYTE_1(ctx->input_cur);
+        case 2:
+          ctx->output_cur[0] = COMPOSE_OUTPUT_BYTE_0(ctx->input_cur);
+      }
+      ctx->output_cur += tail_xtra[input_tail];
+      ctx->input_cur += input_tail;
+    }
+  }
+
+  return true;
+}
+
+gpr_slice grpc_chttp2_base64_decode(gpr_slice input) {
+  size_t input_length = GPR_SLICE_LENGTH(input);
+  size_t output_length = input_length / 4 * 3;
+  struct grpc_base64_decode_context ctx;
+  gpr_slice output;
+
+  if (input_length % 4 != 0) {
+    gpr_log(GPR_ERROR,
+            "Base64 decoding failed, input of "
+            "grpc_chttp2_base64_decode has a length of %d, which is not a "
+            "multiple of 4.\n",
+            (int)input_length);
+    return gpr_empty_slice();
+  }
+
+  if (input_length > 0) {
+    uint8_t *input_end = GPR_SLICE_END_PTR(input);
+    if (*(--input_end) == '=') {
+      output_length--;
+      if (*(--input_end) == '=') {
+        output_length--;
+      }
+    }
+  }
+  output = gpr_slice_malloc(output_length);
+
+  ctx.input_cur = GPR_SLICE_START_PTR(input);
+  ctx.input_end = GPR_SLICE_END_PTR(input);
+  ctx.output_cur = GPR_SLICE_START_PTR(output);
+  ctx.output_end = GPR_SLICE_END_PTR(output);
+  ctx.contains_tail = false;
+
+  if (!grpc_base64_decode_partial(&ctx)) {
+    char *s = gpr_dump_slice(input, GPR_DUMP_ASCII);
+    gpr_log(GPR_ERROR, "Base64 decoding failed, input string:\n%s\n", s);
+    gpr_free(s);
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+  GPR_ASSERT(ctx.output_cur == GPR_SLICE_END_PTR(output));
+  GPR_ASSERT(ctx.input_cur == GPR_SLICE_END_PTR(input));
+  return output;
+}
+
+gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input,
+                                                size_t output_length) {
+  size_t input_length = GPR_SLICE_LENGTH(input);
+  gpr_slice output = gpr_slice_malloc(output_length);
+  struct grpc_base64_decode_context ctx;
+
+  // The length of a base64 string cannot be 4 * n + 1
+  if (input_length % 4 == 1) {
+    gpr_log(GPR_ERROR,
+            "Base64 decoding failed, input of "
+            "grpc_chttp2_base64_decode_with_length has a length of %d, which "
+            "has a tail of 1 byte.\n",
+            (int)input_length);
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+
+  if (output_length > input_length / 4 * 3 + tail_xtra[input_length % 4]) {
+    gpr_log(GPR_ERROR,
+            "Base64 decoding failed, output_length %d is longer "
+            "than the max possible output length %d.\n",
+            (int)output_length,
+            (int)(input_length / 4 * 3 + tail_xtra[input_length % 4]));
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+
+  ctx.input_cur = GPR_SLICE_START_PTR(input);
+  ctx.input_end = GPR_SLICE_END_PTR(input);
+  ctx.output_cur = GPR_SLICE_START_PTR(output);
+  ctx.output_end = GPR_SLICE_END_PTR(output);
+  ctx.contains_tail = true;
+
+  if (!grpc_base64_decode_partial(&ctx)) {
+    char *s = gpr_dump_slice(input, GPR_DUMP_ASCII);
+    gpr_log(GPR_ERROR, "Base64 decoding failed, input string:\n%s\n", s);
+    gpr_free(s);
+    gpr_slice_unref(output);
+    return gpr_empty_slice();
+  }
+  GPR_ASSERT(ctx.output_cur == GPR_SLICE_END_PTR(output));
+  GPR_ASSERT(ctx.input_cur <= GPR_SLICE_END_PTR(input));
+  return output;
+}
diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.h b/src/core/ext/transport/chttp2/transport/bin_decoder.h
new file mode 100644
index 0000000000..b9d40c9b74
--- /dev/null
+++ b/src/core/ext/transport/chttp2/transport/bin_decoder.h
@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H
+
+#include <grpc/support/slice.h>
+#include <stdbool.h>
+
+struct grpc_base64_decode_context {
+  /* input/output: */
+  uint8_t *input_cur;
+  uint8_t *input_end;
+  uint8_t *output_cur;
+  uint8_t *output_end;
+  /* Indicate if the decoder should handle the tail of input data*/
+  bool contains_tail;
+};
+
+/* base64 decode a grpc_base64_decode_context util either input_end is reached
+   or output_end is reached. When input_end is reached, (input_end - input_cur)
+   is less than 4. When output_end is reached, (output_end - output_cur) is less
+   than 3. Returns false if decoding is failed. */
+bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx);
+
+/* base64 decode a slice with pad chars. Returns a new slice, does not take
+   ownership of the input. Returns an empty slice if decoding is failed. */
+gpr_slice grpc_chttp2_base64_decode(gpr_slice input);
+
+/* base64 decode a slice without pad chars, data length is needed. Returns a new
+   slice, does not take ownership of the input. Returns an empty slice if
+   decoding is failed. */
+gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input,
+                                                size_t output_length);
+
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H */
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
index 53633227ac..38e782b9b4 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
@@ -47,6 +47,7 @@
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/ext/transport/chttp2/transport/status_conversion.h"
 #include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
+#include "src/core/lib/http/parser.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
@@ -56,6 +57,8 @@
 #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024)
 #define MAX_WINDOW 0x7fffffffu
 
+#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024)
+
 #define MAX_CLIENT_STREAM_ID 0x7fffffffu
 
 int grpc_http_trace = 0;
@@ -65,8 +68,8 @@ int grpc_flowctl_trace = 0;
   ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \
                                                    writing)))
 
-#define TRANSPORT_FROM_PARSING(tw)                                        \
-  ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \
+#define TRANSPORT_FROM_PARSING(tp)                                        \
+  ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \
                                                    parsing)))
 
 #define TRANSPORT_FROM_GLOBAL(tg)                                         \
@@ -82,19 +85,17 @@ int grpc_flowctl_trace = 0;
 static const grpc_transport_vtable vtable;
 
 /* forward declarations of various callbacks that we'll build closures around */
-static void writing_action(grpc_exec_ctx *exec_ctx, void *t,
-                           bool iomgr_success_ignored);
-static void reading_action(grpc_exec_ctx *exec_ctx, void *t,
-                           bool iomgr_success_ignored);
-static void parsing_action(grpc_exec_ctx *exec_ctx, void *t,
-                           bool iomgr_success_ignored);
+static void writing_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
+static void reading_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
+static void parsing_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
 
 /** Set a transport level setting, and push it to our peer */
 static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id,
                          uint32_t value);
 
 /** Start disconnection chain */
-static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t);
+static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
+                            grpc_error *error);
 
 /** Perform a transport_op */
 static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
@@ -105,13 +106,12 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
 static void cancel_from_api(grpc_exec_ctx *exec_ctx,
                             grpc_chttp2_transport_global *transport_global,
                             grpc_chttp2_stream_global *stream_global,
-                            grpc_status_code status);
+                            grpc_error *error);
 
 static void close_from_api(grpc_exec_ctx *exec_ctx,
                            grpc_chttp2_transport_global *transport_global,
                            grpc_chttp2_stream_global *stream_global,
-                           grpc_status_code status,
-                           gpr_slice *optional_message);
+                           grpc_error *error);
 
 /** Add endpoint from this transport to pollset */
 static void add_to_pollset_locked(grpc_exec_ctx *exec_ctx,
@@ -131,7 +131,7 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
 
 static void connectivity_state_set(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
-    grpc_connectivity_state state, const char *reason);
+    grpc_connectivity_state state, grpc_error *error, const char *reason);
 
 static void check_read_ops(grpc_exec_ctx *exec_ctx,
                            grpc_chttp2_transport_global *transport_global);
@@ -145,7 +145,9 @@ static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
                                                 grpc_chttp2_stream *s,
                                                 void *byte_stream);
 static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
-                                grpc_chttp2_stream_global *stream_global);
+                                grpc_chttp2_transport_global *transport_global,
+                                grpc_chttp2_stream_global *stream_global,
+                                grpc_error *error);
 
 /*******************************************************************************
  * CONSTRUCTION/DESTRUCTION/REFCOUNTING
@@ -188,7 +190,8 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx,
      and maybe they hold resources that need to be freed */
   while (t->global.pings.next != &t->global.pings) {
     grpc_chttp2_outstanding_ping *ping = t->global.pings.next;
-    grpc_exec_ctx_enqueue(exec_ctx, ping->on_recv, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, ping->on_recv,
+                        GRPC_ERROR_CREATE("Transport closed"), NULL);
     ping->next->prev = ping->prev;
     ping->prev->next = ping->next;
     gpr_free(ping);
@@ -257,6 +260,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   t->parsing.is_client = is_client;
   t->parsing.deframe_state =
       is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0;
+  t->parsing.is_first_frame = true;
   t->writing.is_client = is_client;
   grpc_connectivity_state_init(
       &t->channel_callback.state_tracker, GRPC_CHANNEL_READY,
@@ -311,6 +315,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, 0);
   }
   push_setting(t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, DEFAULT_WINDOW);
+  push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
+               DEFAULT_MAX_HEADER_LIST_SIZE);
 
   if (channel_args) {
     for (i = 0; i < channel_args->num_args; i++) {
@@ -378,6 +384,18 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
               &t->writing.hpack_compressor,
               (uint32_t)channel_args->args[i].value.integer);
         }
+      } else if (0 == strcmp(channel_args->args[i].key,
+                             GRPC_ARG_MAX_METADATA_SIZE)) {
+        if (channel_args->args[i].type != GRPC_ARG_INTEGER) {
+          gpr_log(GPR_ERROR, "%s: must be an integer",
+                  GRPC_ARG_MAX_METADATA_SIZE);
+        } else if (channel_args->args[i].value.integer < 0) {
+          gpr_log(GPR_ERROR, "%s: must be non-negative",
+                  GRPC_ARG_MAX_METADATA_SIZE);
+        } else {
+          push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
+                       (uint32_t)channel_args->args[i].value.integer);
+        }
       }
     }
   }
@@ -388,7 +406,7 @@ static void destroy_transport_locked(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_stream *s_ignored,
                                      void *arg_ignored) {
   t->destroying = 1;
-  drop_connection(exec_ctx, t);
+  drop_connection(exec_ctx, t, GRPC_ERROR_CREATE("Transport destroyed"));
 }
 
 static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
@@ -424,12 +442,11 @@ static void destroy_endpoint(grpc_exec_ctx *exec_ctx,
 
 static void close_transport_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_chttp2_transport *t,
-                                   grpc_chttp2_stream *s_ignored,
-                                   void *arg_ignored) {
+                                   grpc_error *error) {
   if (!t->closed) {
     t->closed = 1;
-    connectivity_state_set(exec_ctx, &t->global, GRPC_CHANNEL_FATAL_FAILURE,
-                           "close_transport");
+    connectivity_state_set(exec_ctx, &t->global, GRPC_CHANNEL_SHUTDOWN,
+                           GRPC_ERROR_REF(error), "close_transport");
     if (t->ep) {
       allow_endpoint_shutdown_locked(exec_ctx, t);
     }
@@ -442,6 +459,7 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx,
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing");
     }
   }
+  GRPC_ERROR_UNREF(error);
 }
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
@@ -510,7 +528,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                           [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     *t->accepting_stream = s;
     grpc_chttp2_stream_map_add(&t->parsing_stream_map, s->global.id, s);
-    s->global.in_stream_map = 1;
+    s->global.in_stream_map = true;
   }
 
   grpc_chttp2_run_with_global_lock(exec_ctx, t, s, finish_init_stream_locked,
@@ -530,7 +548,9 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx,
              s->global.id == 0);
   GPR_ASSERT(!s->global.in_stream_map);
   if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) {
-    close_transport_locked(exec_ctx, t, NULL, NULL);
+    close_transport_locked(
+        exec_ctx, t,
+        GRPC_ERROR_CREATE("Last stream closed after sending goaway"));
   }
   if (!t->executor.parsing_active && s->global.id) {
     GPR_ASSERT(grpc_chttp2_stream_map_find(&t->parsing_stream_map,
@@ -569,6 +589,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_incoming_metadata_buffer_destroy(
       &s->global.received_trailing_metadata);
   gpr_slice_buffer_destroy(&s->writing.flow_controlled_buffer);
+  GRPC_ERROR_UNREF(s->global.removal_error);
 
   UNREF_TRANSPORT(exec_ctx, t, "stream");
 
@@ -617,14 +638,15 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_executor_action_header *hdr;
   grpc_chttp2_executor_action_header *next;
 
+  GPR_TIMER_BEGIN("finish_global_actions", 0);
+
   for (;;) {
     if (!t->executor.writing_active && !t->closed &&
-        grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing,
-                                           t->executor.parsing_active)) {
+        grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing)) {
       t->executor.writing_active = 1;
       REF_TRANSPORT(t, "writing");
       prevent_endpoint_shutdown(t);
-      grpc_exec_ctx_enqueue(exec_ctx, &t->writing_action, true, NULL);
+      grpc_exec_ctx_sched(exec_ctx, &t->writing_action, GRPC_ERROR_NONE, NULL);
     }
     check_read_ops(exec_ctx, &t->global);
 
@@ -635,7 +657,9 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
           NULL;
       gpr_mu_unlock(&t->executor.mu);
       while (hdr != NULL) {
+        GPR_TIMER_BEGIN("chttp2:locked_action", 0);
         hdr->action(exec_ctx, t, hdr->stream, hdr->arg);
+        GPR_TIMER_END("chttp2:locked_action", 0);
         next = hdr->next;
         gpr_free(hdr);
         UNREF_TRANSPORT(exec_ctx, t, "pending_action");
@@ -648,6 +672,8 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
     gpr_mu_unlock(&t->executor.mu);
     break;
   }
+
+  GPR_TIMER_END("finish_global_actions", 0);
 }
 
 void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
@@ -657,6 +683,8 @@ void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
                                       void *arg, size_t sizeof_arg) {
   grpc_chttp2_executor_action_header *hdr;
 
+  GPR_TIMER_BEGIN("grpc_chttp2_run_with_global_lock", 0);
+
   REF_TRANSPORT(t, "run_global");
   gpr_mu_lock(&t->executor.mu);
 
@@ -665,7 +693,9 @@ void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
       t->executor.global_active = 1;
       gpr_mu_unlock(&t->executor.mu);
 
+      GPR_TIMER_BEGIN("chttp2:locked_action", 0);
       action(exec_ctx, t, optional_stream, arg);
+      GPR_TIMER_END("chttp2:locked_action", 0);
 
       finish_global_actions(exec_ctx, t);
     } else {
@@ -702,6 +732,8 @@ void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
   }
 
   UNREF_TRANSPORT(exec_ctx, t, "run_global");
+
+  GPR_TIMER_END("grpc_chttp2_run_with_global_lock", 0);
 }
 
 /*******************************************************************************
@@ -735,12 +767,12 @@ static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx,
                                         grpc_chttp2_transport *t,
                                         grpc_chttp2_stream *s_ignored,
                                         void *a) {
-  bool success = (bool)(uintptr_t)a;
+  grpc_error *error = a;
 
   allow_endpoint_shutdown_locked(exec_ctx, t);
 
-  if (!success) {
-    drop_connection(exec_ctx, t);
+  if (error != GRPC_ERROR_NONE) {
+    drop_connection(exec_ctx, t, GRPC_ERROR_REF(error));
   }
 
   grpc_chttp2_cleanup_writing(exec_ctx, &t->global, &t->writing);
@@ -748,7 +780,8 @@ static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_stream_global *stream_global;
   while (grpc_chttp2_list_pop_closed_waiting_for_writing(&t->global,
                                                          &stream_global)) {
-    fail_pending_writes(exec_ctx, stream_global);
+    fail_pending_writes(exec_ctx, &t->global, stream_global,
+                        GRPC_ERROR_REF(error));
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "finish_writes");
   }
 
@@ -761,18 +794,18 @@ static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx,
   }
 
   UNREF_TRANSPORT(exec_ctx, t, "writing");
+  GRPC_ERROR_UNREF(error);
 }
 
 void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx,
-                                   void *transport_writing, bool success) {
+                                   void *transport_writing, grpc_error *error) {
   grpc_chttp2_transport *t = TRANSPORT_FROM_WRITING(transport_writing);
-  grpc_chttp2_run_with_global_lock(exec_ctx, t, NULL,
-                                   terminate_writing_with_lock,
-                                   (void *)(uintptr_t)success, 0);
+  grpc_chttp2_run_with_global_lock(
+      exec_ctx, t, NULL, terminate_writing_with_lock, GRPC_ERROR_REF(error), 0);
 }
 
 static void writing_action(grpc_exec_ctx *exec_ctx, void *gt,
-                           bool iomgr_success_ignored) {
+                           grpc_error *error) {
   grpc_chttp2_transport *t = gt;
   GPR_TIMER_BEGIN("writing_action", 0);
   grpc_chttp2_perform_writes(exec_ctx, &t->writing, t->ep);
@@ -785,11 +818,19 @@ void grpc_chttp2_add_incoming_goaway(
   char *msg = gpr_dump_slice(goaway_text, GPR_DUMP_HEX | GPR_DUMP_ASCII);
   GRPC_CHTTP2_IF_TRACING(
       gpr_log(GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg));
-  gpr_free(msg);
   gpr_slice_unref(goaway_text);
   transport_global->seen_goaway = 1;
-  connectivity_state_set(exec_ctx, transport_global, GRPC_CHANNEL_FATAL_FAILURE,
-                         "got_goaway");
+  /* lie: use transient failure from the transport to indicate goaway has been
+   * received */
+  connectivity_state_set(
+      exec_ctx, transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE,
+      grpc_error_set_str(
+          grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"),
+                             GRPC_ERROR_INT_HTTP2_ERROR,
+                             (intptr_t)goaway_error),
+          GRPC_ERROR_STR_RAW_BYTES, msg),
+      "got_goaway");
+  gpr_free(msg);
 }
 
 static void maybe_start_some_streams(
@@ -818,9 +859,9 @@ static void maybe_start_some_streams(
     transport_global->next_stream_id += 2;
 
     if (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID) {
-      connectivity_state_set(exec_ctx, transport_global,
-                             GRPC_CHANNEL_TRANSIENT_FAILURE,
-                             "no_more_stream_ids");
+      connectivity_state_set(
+          exec_ctx, transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE,
+          GRPC_ERROR_CREATE("Stream IDs exhausted"), "no_more_stream_ids");
     }
 
     stream_global->outgoing_window =
@@ -834,7 +875,7 @@ static void maybe_start_some_streams(
     grpc_chttp2_stream_map_add(
         &TRANSPORT_FROM_GLOBAL(transport_global)->new_stream_map,
         stream_global->id, STREAM_FROM_GLOBAL(stream_global));
-    stream_global->in_stream_map = 1;
+    stream_global->in_stream_map = true;
     transport_global->concurrent_stream_count++;
     grpc_chttp2_become_writable(transport_global, stream_global);
   }
@@ -843,39 +884,47 @@ static void maybe_start_some_streams(
          grpc_chttp2_list_pop_waiting_for_concurrency(transport_global,
                                                       &stream_global)) {
     cancel_from_api(exec_ctx, transport_global, stream_global,
-                    GRPC_STATUS_UNAVAILABLE);
+                    grpc_error_set_int(
+                        GRPC_ERROR_CREATE("Stream IDs exhausted"),
+                        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE));
   }
 }
 
 #define CLOSURE_BARRIER_STATS_BIT (1 << 0)
-#define CLOSURE_BARRIER_FAILURE_BIT (1 << 1)
 #define CLOSURE_BARRIER_FIRST_REF_BIT (1 << 16)
 
 static grpc_closure *add_closure_barrier(grpc_closure *closure) {
-  closure->final_data += CLOSURE_BARRIER_FIRST_REF_BIT;
+  closure->next_data.scratch += CLOSURE_BARRIER_FIRST_REF_BIT;
   return closure;
 }
 
-void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
-                                       grpc_chttp2_stream_global *stream_global,
-                                       grpc_closure **pclosure, int success) {
+void grpc_chttp2_complete_closure_step(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global, grpc_closure **pclosure,
+    grpc_error *error) {
   grpc_closure *closure = *pclosure;
   if (closure == NULL) {
+    GRPC_ERROR_UNREF(error);
     return;
   }
-  closure->final_data -= CLOSURE_BARRIER_FIRST_REF_BIT;
-  if (!success) {
-    closure->final_data |= CLOSURE_BARRIER_FAILURE_BIT;
+  closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT;
+  if (error != GRPC_ERROR_NONE) {
+    if (closure->error == GRPC_ERROR_NONE) {
+      closure->error =
+          GRPC_ERROR_CREATE("Error in HTTP transport completing operation");
+      closure->error = grpc_error_set_str(
+          closure->error, GRPC_ERROR_STR_TARGET_ADDRESS,
+          TRANSPORT_FROM_GLOBAL(transport_global)->peer_string);
+    }
+    closure->error = grpc_error_add_child(closure->error, error);
   }
-  if (closure->final_data < CLOSURE_BARRIER_FIRST_REF_BIT) {
-    if (closure->final_data & CLOSURE_BARRIER_STATS_BIT) {
+  if (closure->next_data.scratch < CLOSURE_BARRIER_FIRST_REF_BIT) {
+    if (closure->next_data.scratch & CLOSURE_BARRIER_STATS_BIT) {
       grpc_transport_move_stats(&stream_global->stats,
                                 stream_global->collecting_stats);
       stream_global->collecting_stats = NULL;
     }
-    grpc_exec_ctx_enqueue(
-        exec_ctx, closure,
-        (closure->final_data & CLOSURE_BARRIER_FAILURE_BIT) == 0, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, closure->error, NULL);
   }
   *pclosure = NULL;
 }
@@ -893,7 +942,7 @@ static int contains_non_ok_status(
   return 0;
 }
 
-static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, bool success) {}
+static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {}
 
 static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_transport *t,
@@ -910,22 +959,23 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
   }
   /* use final_data as a barrier until enqueue time; the inital counter is
      dropped at the end of this function */
-  on_complete->final_data = CLOSURE_BARRIER_FIRST_REF_BIT;
+  on_complete->next_data.scratch = CLOSURE_BARRIER_FIRST_REF_BIT;
+  on_complete->error = GRPC_ERROR_NONE;
 
   if (op->collect_stats != NULL) {
     GPR_ASSERT(stream_global->collecting_stats == NULL);
     stream_global->collecting_stats = op->collect_stats;
-    on_complete->final_data |= CLOSURE_BARRIER_STATS_BIT;
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_STATS_BIT;
   }
 
-  if (op->cancel_with_status != GRPC_STATUS_OK) {
+  if (op->cancel_error != GRPC_ERROR_NONE) {
     cancel_from_api(exec_ctx, transport_global, stream_global,
-                    op->cancel_with_status);
+                    GRPC_ERROR_REF(op->cancel_error));
   }
 
-  if (op->close_with_status != GRPC_STATUS_OK) {
+  if (op->close_error != GRPC_ERROR_NONE) {
     close_from_api(exec_ctx, transport_global, stream_global,
-                   op->close_with_status, op->optional_close_message);
+                   GRPC_ERROR_REF(op->close_error));
   }
 
   if (op->send_initial_metadata != NULL) {
@@ -933,24 +983,44 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     stream_global->send_initial_metadata_finished =
         add_closure_barrier(on_complete);
     stream_global->send_initial_metadata = op->send_initial_metadata;
-    if (contains_non_ok_status(transport_global, op->send_initial_metadata)) {
-      stream_global->seen_error = 1;
-      grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
-    }
-    if (!stream_global->write_closed) {
-      if (transport_global->is_client) {
-        GPR_ASSERT(stream_global->id == 0);
-        grpc_chttp2_list_add_waiting_for_concurrency(transport_global,
-                                                     stream_global);
-        maybe_start_some_streams(exec_ctx, transport_global);
+    const size_t metadata_size =
+        grpc_metadata_batch_size(op->send_initial_metadata);
+    const size_t metadata_peer_limit =
+        transport_global->settings[GRPC_PEER_SETTINGS]
+                                  [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE];
+    if (metadata_size > metadata_peer_limit) {
+      cancel_from_api(
+          exec_ctx, transport_global, stream_global,
+          grpc_error_set_int(
+              grpc_error_set_int(
+                  grpc_error_set_int(
+                      GRPC_ERROR_CREATE("to-be-sent initial metadata size "
+                                        "exceeds peer limit"),
+                      GRPC_ERROR_INT_SIZE, (intptr_t)metadata_size),
+                  GRPC_ERROR_INT_LIMIT, (intptr_t)metadata_peer_limit),
+              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED));
+    } else {
+      if (contains_non_ok_status(transport_global, op->send_initial_metadata)) {
+        stream_global->seen_error = true;
+        grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
+      }
+      if (!stream_global->write_closed) {
+        if (transport_global->is_client) {
+          GPR_ASSERT(stream_global->id == 0);
+          grpc_chttp2_list_add_waiting_for_concurrency(transport_global,
+                                                       stream_global);
+          maybe_start_some_streams(exec_ctx, transport_global);
+        } else {
+          GPR_ASSERT(stream_global->id != 0);
+          grpc_chttp2_become_writable(transport_global, stream_global);
+        }
       } else {
-        GPR_ASSERT(stream_global->id != 0);
-        grpc_chttp2_become_writable(transport_global, stream_global);
+        grpc_chttp2_complete_closure_step(
+            exec_ctx, transport_global, stream_global,
+            &stream_global->send_initial_metadata_finished,
+            GRPC_ERROR_CREATE(
+                "Attempt to send initial metadata after stream was closed"));
       }
-    } else {
-      grpc_chttp2_complete_closure_step(
-          exec_ctx, stream_global,
-          &stream_global->send_initial_metadata_finished, 0);
     }
   }
 
@@ -960,7 +1030,9 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     stream_global->send_message_finished = add_closure_barrier(on_complete);
     if (stream_global->write_closed) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, stream_global, &stream_global->send_message_finished, 0);
+          exec_ctx, transport_global, stream_global,
+          &stream_global->send_message_finished,
+          GRPC_ERROR_CREATE("Attempt to send message after stream was closed"));
     } else {
       stream_global->send_message = op->send_message;
       if (stream_global->id != 0) {
@@ -974,19 +1046,41 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     stream_global->send_trailing_metadata_finished =
         add_closure_barrier(on_complete);
     stream_global->send_trailing_metadata = op->send_trailing_metadata;
-    if (contains_non_ok_status(transport_global, op->send_trailing_metadata)) {
-      stream_global->seen_error = 1;
-      grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
-    }
-    if (stream_global->write_closed) {
-      grpc_chttp2_complete_closure_step(
-          exec_ctx, stream_global,
-          &stream_global->send_trailing_metadata_finished,
-          grpc_metadata_batch_is_empty(op->send_trailing_metadata));
-    } else if (stream_global->id != 0) {
-      /* TODO(ctiller): check if there's flow control for any outstanding
-         bytes before going writable */
-      grpc_chttp2_become_writable(transport_global, stream_global);
+    const size_t metadata_size =
+        grpc_metadata_batch_size(op->send_trailing_metadata);
+    const size_t metadata_peer_limit =
+        transport_global->settings[GRPC_PEER_SETTINGS]
+                                  [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE];
+    if (metadata_size > metadata_peer_limit) {
+      cancel_from_api(
+          exec_ctx, transport_global, stream_global,
+          grpc_error_set_int(
+              grpc_error_set_int(
+                  grpc_error_set_int(
+                      GRPC_ERROR_CREATE("to-be-sent trailing metadata size "
+                                        "exceeds peer limit"),
+                      GRPC_ERROR_INT_SIZE, (intptr_t)metadata_size),
+                  GRPC_ERROR_INT_LIMIT, (intptr_t)metadata_peer_limit),
+              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED));
+    } else {
+      if (contains_non_ok_status(transport_global,
+                                 op->send_trailing_metadata)) {
+        stream_global->seen_error = true;
+        grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
+      }
+      if (stream_global->write_closed) {
+        grpc_chttp2_complete_closure_step(
+            exec_ctx, transport_global, stream_global,
+            &stream_global->send_trailing_metadata_finished,
+            grpc_metadata_batch_is_empty(op->send_trailing_metadata)
+                ? GRPC_ERROR_NONE
+                : GRPC_ERROR_CREATE("Attempt to send trailing metadata after "
+                                    "stream was closed"));
+      } else if (stream_global->id != 0) {
+        /* TODO(ctiller): check if there's flow control for any outstanding
+           bytes before going writable */
+        grpc_chttp2_become_writable(transport_global, stream_global);
+      }
     }
   }
 
@@ -1017,10 +1111,12 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     stream_global->recv_trailing_metadata_finished =
         add_closure_barrier(on_complete);
     stream_global->recv_trailing_metadata = op->recv_trailing_metadata;
+    stream_global->final_metadata_requested = true;
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
 
-  grpc_chttp2_complete_closure_step(exec_ctx, stream_global, &on_complete, 1);
+  grpc_chttp2_complete_closure_step(exec_ctx, transport_global, stream_global,
+                                    &on_complete, GRPC_ERROR_NONE);
 
   GPR_TIMER_END("perform_stream_op_locked", 0);
 }
@@ -1057,7 +1153,7 @@ static void ack_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   for (ping = transport_global->pings.next; ping != &transport_global->pings;
        ping = ping->next) {
     if (0 == memcmp(opaque_8bytes, ping->id, 8)) {
-      grpc_exec_ctx_enqueue(exec_ctx, ping->on_recv, true, NULL);
+      grpc_exec_ctx_sched(exec_ctx, ping->on_recv, GRPC_ERROR_NONE, NULL);
       ping->next->prev = ping->prev;
       ping->prev->next = ping->next;
       gpr_free(ping);
@@ -1079,7 +1175,7 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
                                         grpc_chttp2_stream *s_unused,
                                         void *stream_op) {
   grpc_transport_op *op = stream_op;
-  bool close_transport = op->disconnect;
+  grpc_error *close_transport = op->disconnect_with_error;
 
   /* If there's a set_accept_stream ensure that we're not parsing
      to avoid changing things out from underneath */
@@ -1090,7 +1186,7 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
     return;
   }
 
-  grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
 
   if (op->on_connectivity_state_change != NULL) {
     grpc_connectivity_state_notify_on_state_change(
@@ -1104,7 +1200,9 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
         t->global.last_incoming_stream_id,
         (uint32_t)grpc_chttp2_grpc_status_to_http2_error(op->goaway_status),
         gpr_slice_ref(*op->goaway_message), &t->global.qbuf);
-    close_transport = !grpc_chttp2_has_streams(t);
+    close_transport = grpc_chttp2_has_streams(t)
+                          ? GRPC_ERROR_NONE
+                          : GRPC_ERROR_CREATE("GOAWAY sent");
   }
 
   if (op->set_accept_stream) {
@@ -1125,8 +1223,8 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
     send_ping_locked(t, op->send_ping);
   }
 
-  if (close_transport) {
-    close_transport_locked(exec_ctx, t, NULL, NULL);
+  if (close_transport != GRPC_ERROR_NONE) {
+    close_transport_locked(exec_ctx, t, close_transport);
   }
 }
 
@@ -1149,15 +1247,30 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
       grpc_chttp2_list_pop_check_read_ops(transport_global, &stream_global)) {
     if (stream_global->recv_initial_metadata_ready != NULL &&
         stream_global->published_initial_metadata) {
+      if (stream_global->seen_error) {
+        while ((bs = grpc_chttp2_incoming_frame_queue_pop(
+                    &stream_global->incoming_frames)) != NULL) {
+          incoming_byte_stream_destroy_locked(exec_ctx, NULL, NULL, bs);
+        }
+        if (stream_global->exceeded_metadata_size) {
+          cancel_from_api(
+              exec_ctx, transport_global, stream_global,
+              grpc_error_set_int(
+                  GRPC_ERROR_CREATE(
+                      "received initial metadata size exceeds limit"),
+                  GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED));
+        }
+      }
       grpc_chttp2_incoming_metadata_buffer_publish(
           &stream_global->received_initial_metadata,
           stream_global->recv_initial_metadata);
-      grpc_exec_ctx_enqueue(
-          exec_ctx, stream_global->recv_initial_metadata_ready, true, NULL);
+      grpc_exec_ctx_sched(exec_ctx, stream_global->recv_initial_metadata_ready,
+                          GRPC_ERROR_NONE, NULL);
       stream_global->recv_initial_metadata_ready = NULL;
     }
     if (stream_global->recv_message_ready != NULL) {
-      while (stream_global->seen_error &&
+      while (stream_global->final_metadata_requested &&
+             stream_global->seen_error &&
              (bs = grpc_chttp2_incoming_frame_queue_pop(
                   &stream_global->incoming_frames)) != NULL) {
         incoming_byte_stream_destroy_locked(exec_ctx, NULL, NULL, bs);
@@ -1166,30 +1279,39 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
         *stream_global->recv_message = grpc_chttp2_incoming_frame_queue_pop(
             &stream_global->incoming_frames);
         GPR_ASSERT(*stream_global->recv_message != NULL);
-        grpc_exec_ctx_enqueue(exec_ctx, stream_global->recv_message_ready, true,
-                              NULL);
+        grpc_exec_ctx_sched(exec_ctx, stream_global->recv_message_ready,
+                            GRPC_ERROR_NONE, NULL);
         stream_global->recv_message_ready = NULL;
       } else if (stream_global->published_trailing_metadata) {
         *stream_global->recv_message = NULL;
-        grpc_exec_ctx_enqueue(exec_ctx, stream_global->recv_message_ready, true,
-                              NULL);
+        grpc_exec_ctx_sched(exec_ctx, stream_global->recv_message_ready,
+                            GRPC_ERROR_NONE, NULL);
         stream_global->recv_message_ready = NULL;
       }
     }
     if (stream_global->recv_trailing_metadata_finished != NULL &&
         stream_global->read_closed && stream_global->write_closed) {
-      while (stream_global->seen_error &&
-             (bs = grpc_chttp2_incoming_frame_queue_pop(
-                  &stream_global->incoming_frames)) != NULL) {
-        incoming_byte_stream_destroy_locked(exec_ctx, NULL, NULL, bs);
+      if (stream_global->seen_error) {
+        while ((bs = grpc_chttp2_incoming_frame_queue_pop(
+                    &stream_global->incoming_frames)) != NULL) {
+          incoming_byte_stream_destroy_locked(exec_ctx, NULL, NULL, bs);
+        }
+        if (stream_global->exceeded_metadata_size) {
+          cancel_from_api(
+              exec_ctx, transport_global, stream_global,
+              grpc_error_set_int(
+                  GRPC_ERROR_CREATE(
+                      "received trailing metadata size exceeds limit"),
+                  GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED));
+        }
       }
       if (stream_global->all_incoming_byte_streams_finished) {
         grpc_chttp2_incoming_metadata_buffer_publish(
             &stream_global->received_trailing_metadata,
             stream_global->recv_trailing_metadata);
         grpc_chttp2_complete_closure_step(
-            exec_ctx, stream_global,
-            &stream_global->recv_trailing_metadata_finished, 1);
+            exec_ctx, transport_global, stream_global,
+            &stream_global->recv_trailing_metadata_finished, GRPC_ERROR_NONE);
       }
     }
   }
@@ -1205,7 +1327,7 @@ static void decrement_active_streams_locked(
 }
 
 static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
-                          uint32_t id) {
+                          uint32_t id, grpc_error *error) {
   size_t new_stream_count;
   grpc_chttp2_stream *s =
       grpc_chttp2_stream_map_delete(&t->parsing_stream_map, id);
@@ -1213,19 +1335,22 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     s = grpc_chttp2_stream_map_delete(&t->new_stream_map, id);
   }
   GPR_ASSERT(s);
-  s->global.in_stream_map = 0;
+  s->global.in_stream_map = false;
   if (t->parsing.incoming_stream == &s->parsing) {
     t->parsing.incoming_stream = NULL;
     grpc_chttp2_parsing_become_skip_parser(exec_ctx, &t->parsing);
   }
   if (s->parsing.data_parser.parsing_frame != NULL) {
     grpc_chttp2_incoming_byte_stream_finished(
-        exec_ctx, s->parsing.data_parser.parsing_frame, 0, 0);
+        exec_ctx, s->parsing.data_parser.parsing_frame, GRPC_ERROR_REF(error),
+        0);
     s->parsing.data_parser.parsing_frame = NULL;
   }
 
   if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) {
-    close_transport_locked(exec_ctx, t, NULL, NULL);
+    close_transport_locked(
+        exec_ctx, t, GRPC_ERROR_CREATE_REFERENCING(
+                         "Last stream closed after sending GOAWAY", &error, 1));
   }
   if (grpc_chttp2_list_remove_writable_stream(&t->global, &s->global)) {
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &s->global, "chttp2_writing");
@@ -1238,30 +1363,70 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     t->global.concurrent_stream_count = (uint32_t)new_stream_count;
     maybe_start_some_streams(exec_ctx, &t->global);
   }
+  GRPC_ERROR_UNREF(error);
+}
+
+static void status_codes_from_error(grpc_error *error,
+                                    grpc_chttp2_error_code *http2_error,
+                                    grpc_status_code *grpc_status) {
+  intptr_t ip_http;
+  intptr_t ip_grpc;
+  bool have_http =
+      grpc_error_get_int(error, GRPC_ERROR_INT_HTTP2_ERROR, &ip_http);
+  bool have_grpc =
+      grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &ip_grpc);
+  if (have_http) {
+    *http2_error = (grpc_chttp2_error_code)ip_http;
+  } else if (have_grpc) {
+    *http2_error =
+        grpc_chttp2_grpc_status_to_http2_error((grpc_status_code)ip_grpc);
+  } else {
+    *http2_error = GRPC_CHTTP2_INTERNAL_ERROR;
+  }
+  if (have_grpc) {
+    *grpc_status = (grpc_status_code)ip_grpc;
+  } else if (have_http) {
+    *grpc_status =
+        grpc_chttp2_http2_error_to_grpc_status((grpc_chttp2_error_code)ip_http);
+  } else {
+    *grpc_status = GRPC_STATUS_INTERNAL;
+  }
 }
 
 static void cancel_from_api(grpc_exec_ctx *exec_ctx,
                             grpc_chttp2_transport_global *transport_global,
                             grpc_chttp2_stream_global *stream_global,
-                            grpc_status_code status) {
+                            grpc_error *due_to_error) {
   if (!stream_global->read_closed || !stream_global->write_closed) {
+    grpc_status_code grpc_status;
+    grpc_chttp2_error_code http_error;
+    status_codes_from_error(due_to_error, &http_error, &grpc_status);
+
     if (stream_global->id != 0) {
       gpr_slice_buffer_add(
           &transport_global->qbuf,
-          grpc_chttp2_rst_stream_create(
-              stream_global->id,
-              (uint32_t)grpc_chttp2_grpc_status_to_http2_error(status),
-              &stream_global->stats.outgoing));
+          grpc_chttp2_rst_stream_create(stream_global->id, (uint32_t)http_error,
+                                        &stream_global->stats.outgoing));
+    }
+
+    const char *msg =
+        grpc_error_get_str(due_to_error, GRPC_ERROR_STR_GRPC_MESSAGE);
+    bool free_msg = false;
+    if (msg == NULL) {
+      free_msg = true;
+      msg = grpc_error_string(due_to_error);
     }
-    grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status,
-                            NULL);
+    gpr_slice msg_slice = gpr_slice_from_copied_string(msg);
+    grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global,
+                            grpc_status, &msg_slice);
+    if (free_msg) grpc_error_free_string(msg);
   }
-  if (status != GRPC_STATUS_OK && !stream_global->seen_error) {
-    stream_global->seen_error = 1;
+  if (due_to_error != GRPC_ERROR_NONE && !stream_global->seen_error) {
+    stream_global->seen_error = true;
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
   grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1,
-                                 1);
+                                 1, due_to_error);
 }
 
 void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
@@ -1269,7 +1434,7 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
                              grpc_chttp2_stream_global *stream_global,
                              grpc_status_code status, gpr_slice *slice) {
   if (status != GRPC_STATUS_OK) {
-    stream_global->seen_error = 1;
+    stream_global->seen_error = true;
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
   /* stream_global->recv_trailing_metadata_finished gives us a
@@ -1293,7 +1458,7 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
               GRPC_MDSTR_GRPC_MESSAGE,
               grpc_mdstr_from_slice(gpr_slice_ref(*slice))));
     }
-    stream_global->published_trailing_metadata = 1;
+    stream_global->published_trailing_metadata = true;
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
   if (slice) {
@@ -1302,43 +1467,49 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
 }
 
 static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
-                                grpc_chttp2_stream_global *stream_global) {
+                                grpc_chttp2_transport_global *transport_global,
+                                grpc_chttp2_stream_global *stream_global,
+                                grpc_error *error) {
   grpc_chttp2_complete_closure_step(
-      exec_ctx, stream_global, &stream_global->send_initial_metadata_finished,
-      0);
+      exec_ctx, transport_global, stream_global,
+      &stream_global->send_initial_metadata_finished, GRPC_ERROR_REF(error));
   grpc_chttp2_complete_closure_step(
-      exec_ctx, stream_global, &stream_global->send_trailing_metadata_finished,
-      0);
-  grpc_chttp2_complete_closure_step(exec_ctx, stream_global,
-                                    &stream_global->send_message_finished, 0);
+      exec_ctx, transport_global, stream_global,
+      &stream_global->send_trailing_metadata_finished, GRPC_ERROR_REF(error));
+  grpc_chttp2_complete_closure_step(exec_ctx, transport_global, stream_global,
+                                    &stream_global->send_message_finished,
+                                    error);
 }
 
 void grpc_chttp2_mark_stream_closed(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_stream_global *stream_global, int close_reads,
-    int close_writes) {
+    grpc_chttp2_stream_global *stream_global, int close_reads, int close_writes,
+    grpc_error *error) {
   if (stream_global->read_closed && stream_global->write_closed) {
     /* already closed */
+    GRPC_ERROR_UNREF(error);
     return;
   }
   grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   if (close_reads && !stream_global->read_closed) {
-    stream_global->read_closed = 1;
-    stream_global->published_initial_metadata = 1;
-    stream_global->published_trailing_metadata = 1;
+    stream_global->read_closed = true;
+    stream_global->published_initial_metadata = true;
+    stream_global->published_trailing_metadata = true;
     decrement_active_streams_locked(exec_ctx, transport_global, stream_global);
   }
   if (close_writes && !stream_global->write_closed) {
-    stream_global->write_closed = 1;
+    stream_global->write_closed = true;
     if (TRANSPORT_FROM_GLOBAL(transport_global)->executor.writing_active) {
       GRPC_CHTTP2_STREAM_REF(stream_global, "finish_writes");
       grpc_chttp2_list_add_closed_waiting_for_writing(transport_global,
                                                       stream_global);
     } else {
-      fail_pending_writes(exec_ctx, stream_global);
+      fail_pending_writes(exec_ctx, transport_global, stream_global,
+                          GRPC_ERROR_REF(error));
     }
   }
   if (stream_global->read_closed && stream_global->write_closed) {
+    stream_global->removal_error = GRPC_ERROR_REF(error);
     if (stream_global->id != 0 &&
         TRANSPORT_FROM_GLOBAL(transport_global)->executor.parsing_active) {
       grpc_chttp2_list_add_closed_waiting_for_parsing(transport_global,
@@ -1346,134 +1517,156 @@ void grpc_chttp2_mark_stream_closed(
     } else {
       if (stream_global->id != 0) {
         remove_stream(exec_ctx, TRANSPORT_FROM_GLOBAL(transport_global),
-                      stream_global->id);
+                      stream_global->id, GRPC_ERROR_REF(error));
       }
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2");
     }
   }
+  GRPC_ERROR_UNREF(error);
 }
 
 static void close_from_api(grpc_exec_ctx *exec_ctx,
                            grpc_chttp2_transport_global *transport_global,
                            grpc_chttp2_stream_global *stream_global,
-                           grpc_status_code status,
-                           gpr_slice *optional_message) {
+                           grpc_error *error) {
   gpr_slice hdr;
   gpr_slice status_hdr;
   gpr_slice message_pfx;
   uint8_t *p;
   uint32_t len = 0;
-
-  GPR_ASSERT(status >= 0 && (int)status < 100);
-
-  GPR_ASSERT(stream_global->id != 0);
-
-  /* Hand roll a header block.
-     This is unnecessarily ugly - at some point we should find a more elegant
-     solution.
-     It's complicated by the fact that our send machinery would be dead by the
-     time we got around to sending this, so instead we ignore HPACK compression
-     and just write the uncompressed bytes onto the wire. */
-  status_hdr = gpr_slice_malloc(15 + (status >= 10));
-  p = GPR_SLICE_START_PTR(status_hdr);
-  *p++ = 0x40; /* literal header */
-  *p++ = 11;   /* len(grpc-status) */
-  *p++ = 'g';
-  *p++ = 'r';
-  *p++ = 'p';
-  *p++ = 'c';
-  *p++ = '-';
-  *p++ = 's';
-  *p++ = 't';
-  *p++ = 'a';
-  *p++ = 't';
-  *p++ = 'u';
-  *p++ = 's';
-  if (status < 10) {
-    *p++ = 1;
-    *p++ = (uint8_t)('0' + status);
-  } else {
-    *p++ = 2;
-    *p++ = (uint8_t)('0' + (status / 10));
-    *p++ = (uint8_t)('0' + (status % 10));
-  }
-  GPR_ASSERT(p == GPR_SLICE_END_PTR(status_hdr));
-  len += (uint32_t)GPR_SLICE_LENGTH(status_hdr);
-
-  if (optional_message) {
-    GPR_ASSERT(GPR_SLICE_LENGTH(*optional_message) < 127);
-    message_pfx = gpr_slice_malloc(15);
-    p = GPR_SLICE_START_PTR(message_pfx);
-    *p++ = 0x40;
-    *p++ = 12; /* len(grpc-message) */
+  grpc_status_code grpc_status;
+  grpc_chttp2_error_code http_error;
+  status_codes_from_error(error, &http_error, &grpc_status);
+
+  GPR_ASSERT(grpc_status >= 0 && (int)grpc_status < 100);
+
+  if (stream_global->id != 0 && !transport_global->is_client) {
+    /* Hand roll a header block.
+       This is unnecessarily ugly - at some point we should find a more elegant
+       solution.
+       It's complicated by the fact that our send machinery would be dead by the
+       time we got around to sending this, so instead we ignore HPACK
+       compression
+       and just write the uncompressed bytes onto the wire. */
+    status_hdr = gpr_slice_malloc(15 + (grpc_status >= 10));
+    p = GPR_SLICE_START_PTR(status_hdr);
+    *p++ = 0x40; /* literal header */
+    *p++ = 11;   /* len(grpc-status) */
     *p++ = 'g';
     *p++ = 'r';
     *p++ = 'p';
     *p++ = 'c';
     *p++ = '-';
-    *p++ = 'm';
-    *p++ = 'e';
-    *p++ = 's';
     *p++ = 's';
+    *p++ = 't';
     *p++ = 'a';
-    *p++ = 'g';
-    *p++ = 'e';
-    *p++ = (uint8_t)GPR_SLICE_LENGTH(*optional_message);
-    GPR_ASSERT(p == GPR_SLICE_END_PTR(message_pfx));
-    len += (uint32_t)GPR_SLICE_LENGTH(message_pfx);
-    len += (uint32_t)GPR_SLICE_LENGTH(*optional_message);
-  }
-
-  hdr = gpr_slice_malloc(9);
-  p = GPR_SLICE_START_PTR(hdr);
-  *p++ = (uint8_t)(len >> 16);
-  *p++ = (uint8_t)(len >> 8);
-  *p++ = (uint8_t)(len);
-  *p++ = GRPC_CHTTP2_FRAME_HEADER;
-  *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
-  *p++ = (uint8_t)(stream_global->id >> 24);
-  *p++ = (uint8_t)(stream_global->id >> 16);
-  *p++ = (uint8_t)(stream_global->id >> 8);
-  *p++ = (uint8_t)(stream_global->id);
-  GPR_ASSERT(p == GPR_SLICE_END_PTR(hdr));
-
-  gpr_slice_buffer_add(&transport_global->qbuf, hdr);
-  gpr_slice_buffer_add(&transport_global->qbuf, status_hdr);
-  if (optional_message) {
-    gpr_slice_buffer_add(&transport_global->qbuf, message_pfx);
-    gpr_slice_buffer_add(&transport_global->qbuf,
-                         gpr_slice_ref(*optional_message));
-  }
-
-  gpr_slice_buffer_add(
-      &transport_global->qbuf,
-      grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR,
-                                    &stream_global->stats.outgoing));
-
-  if (optional_message) {
-    gpr_slice_ref(*optional_message);
-  }
-  grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status,
-                          optional_message);
+    *p++ = 't';
+    *p++ = 'u';
+    *p++ = 's';
+    if (grpc_status < 10) {
+      *p++ = 1;
+      *p++ = (uint8_t)('0' + grpc_status);
+    } else {
+      *p++ = 2;
+      *p++ = (uint8_t)('0' + (grpc_status / 10));
+      *p++ = (uint8_t)('0' + (grpc_status % 10));
+    }
+    GPR_ASSERT(p == GPR_SLICE_END_PTR(status_hdr));
+    len += (uint32_t)GPR_SLICE_LENGTH(status_hdr);
+
+    const char *optional_message =
+        grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE);
+
+    if (optional_message != NULL) {
+      size_t msg_len = strlen(optional_message);
+      GPR_ASSERT(msg_len < 127);
+      message_pfx = gpr_slice_malloc(15);
+      p = GPR_SLICE_START_PTR(message_pfx);
+      *p++ = 0x40;
+      *p++ = 12; /* len(grpc-message) */
+      *p++ = 'g';
+      *p++ = 'r';
+      *p++ = 'p';
+      *p++ = 'c';
+      *p++ = '-';
+      *p++ = 'm';
+      *p++ = 'e';
+      *p++ = 's';
+      *p++ = 's';
+      *p++ = 'a';
+      *p++ = 'g';
+      *p++ = 'e';
+      *p++ = (uint8_t)msg_len;
+      GPR_ASSERT(p == GPR_SLICE_END_PTR(message_pfx));
+      len += (uint32_t)GPR_SLICE_LENGTH(message_pfx);
+      len += (uint32_t)msg_len;
+    }
+
+    hdr = gpr_slice_malloc(9);
+    p = GPR_SLICE_START_PTR(hdr);
+    *p++ = (uint8_t)(len >> 16);
+    *p++ = (uint8_t)(len >> 8);
+    *p++ = (uint8_t)(len);
+    *p++ = GRPC_CHTTP2_FRAME_HEADER;
+    *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
+    *p++ = (uint8_t)(stream_global->id >> 24);
+    *p++ = (uint8_t)(stream_global->id >> 16);
+    *p++ = (uint8_t)(stream_global->id >> 8);
+    *p++ = (uint8_t)(stream_global->id);
+    GPR_ASSERT(p == GPR_SLICE_END_PTR(hdr));
+
+    gpr_slice_buffer_add(&transport_global->qbuf, hdr);
+    gpr_slice_buffer_add(&transport_global->qbuf, status_hdr);
+    if (optional_message) {
+      gpr_slice_buffer_add(&transport_global->qbuf, message_pfx);
+      gpr_slice_buffer_add(&transport_global->qbuf,
+                           gpr_slice_from_copied_string(optional_message));
+    }
+    gpr_slice_buffer_add(
+        &transport_global->qbuf,
+        grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR,
+                                      &stream_global->stats.outgoing));
+  }
+
+  const char *msg = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE);
+  bool free_msg = false;
+  if (msg == NULL) {
+    free_msg = true;
+    msg = grpc_error_string(error);
+  }
+  gpr_slice msg_slice = gpr_slice_from_copied_string(msg);
+  grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global,
+                          grpc_status, &msg_slice);
+  if (free_msg) grpc_error_free_string(msg);
+
   grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1,
-                                 1);
+                                 1, error);
 }
 
+typedef struct {
+  grpc_exec_ctx *exec_ctx;
+  grpc_error *error;
+} cancel_stream_cb_args;
+
 static void cancel_stream_cb(grpc_chttp2_transport_global *transport_global,
                              void *user_data,
                              grpc_chttp2_stream_global *stream_global) {
-  cancel_from_api(user_data, transport_global, stream_global,
-                  GRPC_STATUS_UNAVAILABLE);
+  cancel_stream_cb_args *args = user_data;
+  cancel_from_api(args->exec_ctx, transport_global, stream_global,
+                  GRPC_ERROR_REF(args->error));
 }
 
-static void end_all_the_calls(grpc_exec_ctx *exec_ctx,
-                              grpc_chttp2_transport *t) {
-  grpc_chttp2_for_all_streams(&t->global, exec_ctx, cancel_stream_cb);
+static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
+                              grpc_error *error) {
+  cancel_stream_cb_args args = {exec_ctx, error};
+  grpc_chttp2_for_all_streams(&t->global, &args, cancel_stream_cb);
+  GRPC_ERROR_UNREF(error);
 }
 
-static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) {
-  close_transport_locked(exec_ctx, t, NULL, NULL);
-  end_all_the_calls(exec_ctx, t);
+static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
+                            grpc_error *error) {
+  close_transport_locked(exec_ctx, t, GRPC_ERROR_REF(error));
+  end_all_the_calls(exec_ctx, t, error);
 }
 
 /** update window from a settings change */
@@ -1503,20 +1696,22 @@ static void update_global_window(void *args, uint32_t id, void *stream) {
 static void reading_action_locked(grpc_exec_ctx *exec_ctx,
                                   grpc_chttp2_transport *t,
                                   grpc_chttp2_stream *s_unused, void *arg);
-static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg, bool success);
+static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg,
+                           grpc_error *error);
 static void post_reading_action_locked(grpc_exec_ctx *exec_ctx,
                                        grpc_chttp2_transport *t,
                                        grpc_chttp2_stream *s_unused, void *arg);
 static void post_parse_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                               grpc_chttp2_stream *s_unused, void *arg);
 
-static void reading_action(grpc_exec_ctx *exec_ctx, void *tp, bool success) {
+static void reading_action(grpc_exec_ctx *exec_ctx, void *tp,
+                           grpc_error *error) {
   /* Control flow:
      reading_action_locked ->
        (parse_unlocked -> post_parse_locked)? ->
        post_reading_action_locked */
   grpc_chttp2_run_with_global_lock(exec_ctx, tp, NULL, reading_action_locked,
-                                   (void *)(uintptr_t)success, 0);
+                                   GRPC_ERROR_REF(error), 0);
 }
 
 static void reading_action_locked(grpc_exec_ctx *exec_ctx,
@@ -1524,7 +1719,7 @@ static void reading_action_locked(grpc_exec_ctx *exec_ctx,
                                   grpc_chttp2_stream *s_unused, void *arg) {
   grpc_chttp2_transport_global *transport_global = &t->global;
   grpc_chttp2_transport_parsing *transport_parsing = &t->parsing;
-  bool success = (bool)(uintptr_t)arg;
+  grpc_error *error = arg;
 
   GPR_ASSERT(!t->executor.parsing_active);
   if (!t->closed) {
@@ -1533,27 +1728,65 @@ static void reading_action_locked(grpc_exec_ctx *exec_ctx,
     grpc_chttp2_stream_map_move_into(&t->new_stream_map,
                                      &t->parsing_stream_map);
     grpc_chttp2_prepare_to_read(transport_global, transport_parsing);
-    grpc_exec_ctx_enqueue(exec_ctx, &t->parsing_action, success, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &t->parsing_action, error, NULL);
   } else {
     post_reading_action_locked(exec_ctx, t, s_unused, arg);
   }
 }
 
-static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx,
+                                    grpc_chttp2_transport *t) {
+  grpc_http_parser parser;
+  size_t i = 0;
+  grpc_error *error = GRPC_ERROR_NONE;
+  grpc_http_response response;
+  memset(&response, 0, sizeof(response));
+
+  grpc_http_parser_init(&parser, GRPC_HTTP_RESPONSE, &response);
+
+  grpc_error *parse_error = GRPC_ERROR_NONE;
+  for (; i < t->read_buffer.count && parse_error == GRPC_ERROR_NONE; i++) {
+    parse_error = grpc_http_parser_parse(&parser, t->read_buffer.slices[i]);
+  }
+  if (parse_error == GRPC_ERROR_NONE &&
+      (parse_error = grpc_http_parser_eof(&parser)) == GRPC_ERROR_NONE) {
+    error = grpc_error_set_int(
+        GRPC_ERROR_CREATE("Trying to connect an http1.x server"),
+        GRPC_ERROR_INT_HTTP_STATUS, response.status);
+  }
+  GRPC_ERROR_UNREF(parse_error);
+
+  grpc_http_parser_destroy(&parser);
+  grpc_http_response_destroy(&response);
+  return error;
+}
+
+static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg,
+                           grpc_error *error) {
   grpc_chttp2_transport *t = arg;
   GPR_TIMER_BEGIN("reading_action.parse", 0);
   size_t i = 0;
-  for (; i < t->read_buffer.count &&
-         grpc_chttp2_perform_read(exec_ctx, &t->parsing,
-                                  t->read_buffer.slices[i]);
-       i++)
-    ;
+  grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE,
+                           GRPC_ERROR_NONE};
+  for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) {
+    errors[1] = grpc_chttp2_perform_read(exec_ctx, &t->parsing,
+                                         t->read_buffer.slices[i]);
+  };
   if (i != t->read_buffer.count) {
-    success = false;
+    errors[2] = try_http_parsing(exec_ctx, t);
+  }
+  grpc_error *err =
+      errors[0] == GRPC_ERROR_NONE && errors[1] == GRPC_ERROR_NONE &&
+              errors[2] == GRPC_ERROR_NONE
+          ? GRPC_ERROR_NONE
+          : GRPC_ERROR_CREATE_REFERENCING("Failed parsing HTTP/2", errors,
+                                          GPR_ARRAY_SIZE(errors));
+  for (i = 0; i < GPR_ARRAY_SIZE(errors); i++) {
+    GRPC_ERROR_UNREF(errors[i]);
   }
   GPR_TIMER_END("reading_action.parse", 0);
-  grpc_chttp2_run_with_global_lock(exec_ctx, t, NULL, post_parse_locked,
-                                   (void *)(uintptr_t)success, 0);
+  grpc_chttp2_run_with_global_lock(exec_ctx, t, NULL, post_parse_locked, err,
+                                   0);
 }
 
 static void post_parse_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
@@ -1590,7 +1823,8 @@ static void post_parse_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     GPR_ASSERT(stream_global->in_stream_map);
     GPR_ASSERT(stream_global->write_closed);
     GPR_ASSERT(stream_global->read_closed);
-    remove_stream(exec_ctx, t, stream_global->id);
+    remove_stream(exec_ctx, t, stream_global->id,
+                  GRPC_ERROR_REF(stream_global->removal_error));
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2");
   }
 
@@ -1601,10 +1835,17 @@ static void post_reading_action_locked(grpc_exec_ctx *exec_ctx,
                                        grpc_chttp2_transport *t,
                                        grpc_chttp2_stream *s_unused,
                                        void *arg) {
-  bool success = (bool)(uintptr_t)arg;
+  grpc_error *error = arg;
   bool keep_reading = false;
-  if (!success || t->closed) {
-    drop_connection(exec_ctx, t);
+  if (error == GRPC_ERROR_NONE && t->closed) {
+    error = GRPC_ERROR_CREATE("Transport closed");
+  }
+  if (error != GRPC_ERROR_NONE) {
+    if (!grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, NULL)) {
+      error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS,
+                                 GRPC_STATUS_UNAVAILABLE);
+    }
+    drop_connection(exec_ctx, t, GRPC_ERROR_REF(error));
     t->endpoint_reading = 0;
     if (!t->executor.writing_active && t->ep) {
       grpc_endpoint_destroy(exec_ctx, t->ep);
@@ -1618,6 +1859,7 @@ static void post_reading_action_locked(grpc_exec_ctx *exec_ctx,
     prevent_endpoint_shutdown(t);
   }
   gpr_slice_buffer_reset_and_unref(&t->read_buffer);
+  GRPC_ERROR_UNREF(error);
 
   if (keep_reading) {
     grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer, &t->reading_action);
@@ -1634,13 +1876,13 @@ static void post_reading_action_locked(grpc_exec_ctx *exec_ctx,
 
 static void connectivity_state_set(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
-    grpc_connectivity_state state, const char *reason) {
+    grpc_connectivity_state state, grpc_error *error, const char *reason) {
   GRPC_CHTTP2_IF_TRACING(
       gpr_log(GPR_DEBUG, "set connectivity_state=%d", state));
   grpc_connectivity_state_set(
       exec_ctx,
       &TRANSPORT_FROM_GLOBAL(transport_global)->channel_callback.state_tracker,
-      state, reason);
+      state, error, reason);
 }
 
 /*******************************************************************************
@@ -1672,6 +1914,13 @@ static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                                    add_to_pollset_locked, pollset, 0);
 }
 
+static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                            grpc_stream *gs, grpc_pollset_set *pollset_set) {
+  grpc_chttp2_run_with_global_lock(exec_ctx, (grpc_chttp2_transport *)gt,
+                                   (grpc_chttp2_stream *)gs,
+                                   add_to_pollset_set_locked, pollset_set, 0);
+}
+
 /*******************************************************************************
  * BYTE STREAM
  */
@@ -1679,6 +1928,7 @@ static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
 static void incoming_byte_stream_unref(grpc_exec_ctx *exec_ctx,
                                        grpc_chttp2_incoming_byte_stream *bs) {
   if (gpr_unref(&bs->refs)) {
+    GRPC_ERROR_UNREF(bs->error);
     gpr_slice_buffer_destroy(&bs->slices);
     gpr_free(bs);
   }
@@ -1747,9 +1997,10 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx,
   }
   if (bs->slices.count > 0) {
     *arg->slice = gpr_slice_buffer_take_first(&bs->slices);
-    grpc_exec_ctx_enqueue(exec_ctx, arg->on_complete, true, NULL);
-  } else if (bs->failed) {
-    grpc_exec_ctx_enqueue(exec_ctx, arg->on_complete, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, arg->on_complete, GRPC_ERROR_NONE, NULL);
+  } else if (bs->error != GRPC_ERROR_NONE) {
+    grpc_exec_ctx_sched(exec_ctx, arg->on_complete, GRPC_ERROR_REF(bs->error),
+                        NULL);
   } else {
     bs->on_next = arg->on_complete;
     bs->next = arg->slice;
@@ -1806,7 +2057,7 @@ static void incoming_byte_stream_push_locked(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_incoming_byte_stream *bs = arg->byte_stream;
   if (bs->on_next != NULL) {
     *bs->next = arg->slice;
-    grpc_exec_ctx_enqueue(exec_ctx, bs->on_next, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE, NULL);
     bs->on_next = NULL;
   } else {
     gpr_slice_buffer_add(&bs->slices, arg->slice);
@@ -1824,13 +2075,30 @@ void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx,
                                    sizeof(arg));
 }
 
+typedef struct {
+  grpc_chttp2_incoming_byte_stream *bs;
+  grpc_error *error;
+} bs_fail_args;
+
+static bs_fail_args *make_bs_fail_args(grpc_chttp2_incoming_byte_stream *bs,
+                                       grpc_error *error) {
+  bs_fail_args *a = gpr_malloc(sizeof(*a));
+  a->bs = bs;
+  a->error = error;
+  return a;
+}
+
 static void incoming_byte_stream_finished_failed_locked(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s,
     void *argp) {
-  grpc_chttp2_incoming_byte_stream *bs = argp;
-  grpc_exec_ctx_enqueue(exec_ctx, bs->on_next, false, NULL);
+  bs_fail_args *a = argp;
+  grpc_chttp2_incoming_byte_stream *bs = a->bs;
+  grpc_error *error = a->error;
+  gpr_free(a);
+  grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error), NULL);
   bs->on_next = NULL;
-  bs->failed = 1;
+  GRPC_ERROR_UNREF(bs->error);
+  bs->error = error;
   incoming_byte_stream_unref(exec_ctx, bs);
 }
 
@@ -1843,25 +2111,26 @@ static void incoming_byte_stream_finished_ok_locked(grpc_exec_ctx *exec_ctx,
 }
 
 void grpc_chttp2_incoming_byte_stream_finished(
-    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, int success,
-    int from_parsing_thread) {
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs,
+    grpc_error *error, int from_parsing_thread) {
   if (from_parsing_thread) {
-    if (success) {
+    if (error == GRPC_ERROR_NONE) {
       grpc_chttp2_run_with_global_lock(exec_ctx, bs->transport, bs->stream,
                                        incoming_byte_stream_finished_ok_locked,
                                        bs, 0);
     } else {
-      incoming_byte_stream_finished_ok_locked(exec_ctx, bs->transport,
-                                              bs->stream, bs);
-    }
-  } else {
-    if (success) {
       grpc_chttp2_run_with_global_lock(
           exec_ctx, bs->transport, bs->stream,
-          incoming_byte_stream_finished_failed_locked, bs, 0);
+          incoming_byte_stream_finished_failed_locked,
+          make_bs_fail_args(bs, error), 0);
+    }
+  } else {
+    if (error == GRPC_ERROR_NONE) {
+      incoming_byte_stream_finished_ok_locked(exec_ctx, bs->transport,
+                                              bs->stream, bs);
     } else {
-      incoming_byte_stream_finished_failed_locked(exec_ctx, bs->transport,
-                                                  bs->stream, bs);
+      incoming_byte_stream_finished_failed_locked(
+          exec_ctx, bs->transport, bs->stream, make_bs_fail_args(bs, error));
     }
   }
 }
@@ -1884,7 +2153,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create(
   gpr_slice_buffer_init(&incoming_byte_stream->slices);
   incoming_byte_stream->on_next = NULL;
   incoming_byte_stream->is_tail = 1;
-  incoming_byte_stream->failed = 0;
+  incoming_byte_stream->error = GRPC_ERROR_NONE;
   if (add_to_queue->head == NULL) {
     add_to_queue->head = incoming_byte_stream;
   } else {
@@ -1903,10 +2172,13 @@ static char *format_flowctl_context_var(const char *context, const char *var,
                                         int64_t val, uint32_t id,
                                         char **scope) {
   char *underscore_pos;
+  char *buf;
   char *result;
   if (context == NULL) {
     *scope = NULL;
-    gpr_asprintf(&result, "%s(%lld)", var, val);
+    gpr_asprintf(&buf, "%s(%" PRId64 ")", var, val);
+    result = gpr_leftpad(buf, ' ', 40);
+    gpr_free(buf);
     return result;
   }
   underscore_pos = strchr(context, '_');
@@ -1917,7 +2189,9 @@ static char *format_flowctl_context_var(const char *context, const char *var,
     gpr_asprintf(scope, "%s[%d]", tmp, id);
     gpr_free(tmp);
   }
-  gpr_asprintf(&result, "%s.%s(%lld)", underscore_pos + 1, var, val);
+  gpr_asprintf(&buf, "%s.%s(%" PRId64 ")", underscore_pos + 1, var, val);
+  result = gpr_leftpad(buf, ' ', 40);
+  gpr_free(buf);
   return result;
 }
 
@@ -1938,6 +2212,8 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
                                uint32_t stream_id, int64_t val1, int64_t val2) {
   char *scope1;
   char *scope2;
+  char *tmp_phase;
+  char *tmp_scope1;
   char *label1 =
       format_flowctl_context_var(context1, var1, val1, stream_id, &scope1);
   char *label2 =
@@ -1945,14 +2221,18 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
   char *clisvr = is_client ? "client" : "server";
   char *prefix;
 
-  gpr_asprintf(&prefix, "FLOW % 8s: %s % 11s ", phase, clisvr, scope1);
+  tmp_phase = gpr_leftpad(phase, ' ', 8);
+  tmp_scope1 = gpr_leftpad(scope1, ' ', 11);
+  gpr_asprintf(&prefix, "FLOW %s: %s %s ", phase, clisvr, scope1);
+  gpr_free(tmp_phase);
+  gpr_free(tmp_scope1);
 
   switch (op) {
     case GRPC_CHTTP2_FLOWCTL_MOVE:
       GPR_ASSERT(samestr(scope1, scope2));
       if (val2 != 0) {
         gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sMOVE   % 40s <- % 40s giving %d", prefix, label1, label2,
+                "%sMOVE   %s <- %s giving %" PRId64, prefix, label1, label2,
                 val1 + val2);
       }
       break;
@@ -1960,7 +2240,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
       GPR_ASSERT(val2 >= 0);
       if (val2 != 0) {
         gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sCREDIT % 40s by % 40s giving %d", prefix, label1, label2,
+                "%sCREDIT %s by %s giving %" PRId64, prefix, label1, label2,
                 val1 + val2);
       }
       break;
@@ -1968,7 +2248,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
       GPR_ASSERT(val2 >= 0);
       if (val2 != 0) {
         gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "%sDEBIT  % 40s by % 40s giving %d", prefix, label1, label2,
+                "%sDEBIT  %s by %s giving %" PRId64, prefix, label1, label2,
                 val1 - val2);
       }
       break;
@@ -1993,6 +2273,7 @@ static const grpc_transport_vtable vtable = {sizeof(grpc_chttp2_stream),
                                              "chttp2",
                                              init_stream,
                                              set_pollset,
+                                             set_pollset_set,
                                              perform_stream_op,
                                              perform_transport_op,
                                              destroy_stream,
@@ -2013,5 +2294,5 @@ void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_transport *t = (grpc_chttp2_transport *)transport;
   REF_TRANSPORT(t, "reading_action"); /* matches unref inside reading_action */
   gpr_slice_buffer_addn(&t->read_buffer, slices, nslices);
-  reading_action(exec_ctx, t, 1);
+  reading_action(exec_ctx, t, GRPC_ERROR_NONE);
 }
diff --git a/src/core/ext/transport/chttp2/transport/frame.h b/src/core/ext/transport/chttp2/transport/frame.h
index 5c72d91c2a..7776609367 100644
--- a/src/core/ext/transport/chttp2/transport/frame.h
+++ b/src/core/ext/transport/chttp2/transport/frame.h
@@ -37,13 +37,7 @@
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
 
-/* Common definitions for frame handling in the chttp2 transport */
-
-typedef enum {
-  GRPC_CHTTP2_PARSE_OK,
-  GRPC_CHTTP2_STREAM_ERROR,
-  GRPC_CHTTP2_CONNECTION_ERROR
-} grpc_chttp2_parse_error;
+#include "src/core/lib/iomgr/error.h"
 
 /* defined in internal.h */
 typedef struct grpc_chttp2_stream_parsing grpc_chttp2_stream_parsing;
diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c
index 3a6d80e0a3..9046fbc453 100644
--- a/src/core/ext/transport/chttp2/transport/frame_data.c
+++ b/src/core/ext/transport/chttp2/transport/frame_data.c
@@ -37,24 +37,25 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/transport.h"
 
-grpc_chttp2_parse_error grpc_chttp2_data_parser_init(
-    grpc_chttp2_data_parser *parser) {
+grpc_error *grpc_chttp2_data_parser_init(grpc_chttp2_data_parser *parser) {
   parser->state = GRPC_CHTTP2_DATA_FH_0;
   parser->parsing_frame = NULL;
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
 void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_data_parser *parser) {
   grpc_byte_stream *bs;
   if (parser->parsing_frame) {
-    grpc_chttp2_incoming_byte_stream_finished(exec_ctx, parser->parsing_frame,
-                                              0, 1);
+    grpc_chttp2_incoming_byte_stream_finished(
+        exec_ctx, parser->parsing_frame, GRPC_ERROR_CREATE("Parser destroyed"),
+        1);
   }
   while (
       (bs = grpc_chttp2_incoming_frame_queue_pop(&parser->incoming_frames))) {
@@ -62,11 +63,16 @@ void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx,
   }
 }
 
-grpc_chttp2_parse_error grpc_chttp2_data_parser_begin_frame(
-    grpc_chttp2_data_parser *parser, uint8_t flags) {
+grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser,
+                                                uint8_t flags,
+                                                uint32_t stream_id) {
   if (flags & ~GRPC_CHTTP2_DATA_FLAG_END_STREAM) {
-    gpr_log(GPR_ERROR, "unsupported data flags: 0x%02x", flags);
-    return GRPC_CHTTP2_STREAM_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "unsupported data flags: 0x%02x", flags);
+    grpc_error *err = grpc_error_set_int(
+        GRPC_ERROR_CREATE(msg), GRPC_ERROR_INT_STREAM_ID, (intptr_t)stream_id);
+    gpr_free(msg);
+    return err;
   }
 
   if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) {
@@ -75,7 +81,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_begin_frame(
     parser->is_last_frame = 0;
   }
 
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
 void grpc_chttp2_incoming_frame_queue_merge(
@@ -139,7 +145,7 @@ void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf,
   stats->data_bytes += write_bytes;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
+grpc_error *grpc_chttp2_data_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
@@ -149,19 +155,20 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
   grpc_chttp2_data_parser *p = parser;
   uint32_t message_flags;
   grpc_chttp2_incoming_byte_stream *incoming_byte_stream;
+  char *msg;
 
   if (is_last && p->is_last_frame) {
     stream_parsing->received_close = 1;
   }
 
   if (cur == end) {
-    return GRPC_CHTTP2_PARSE_OK;
+    return GRPC_ERROR_NONE;
   }
 
   switch (p->state) {
     case GRPC_CHTTP2_DATA_ERROR:
       p->state = GRPC_CHTTP2_DATA_ERROR;
-      return GRPC_CHTTP2_STREAM_ERROR;
+      return GRPC_ERROR_REF(p->error);
     fh_0:
     case GRPC_CHTTP2_DATA_FH_0:
       stream_parsing->stats.incoming.framing_bytes++;
@@ -174,13 +181,23 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
           p->is_frame_compressed = 1; /* GPR_TRUE */
           break;
         default:
-          gpr_log(GPR_ERROR, "Bad GRPC frame type 0x%02x", p->frame_type);
+          gpr_asprintf(&msg, "Bad GRPC frame type 0x%02x", p->frame_type);
+          p->error = GRPC_ERROR_CREATE(msg);
+          p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID,
+                                        (intptr_t)stream_parsing->id);
+          gpr_free(msg);
+          msg = gpr_dump_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII);
+          p->error =
+              grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES, msg);
+          gpr_free(msg);
+          p->error =
+              grpc_error_set_int(p->error, GRPC_ERROR_INT_OFFSET, cur - beg);
           p->state = GRPC_CHTTP2_DATA_ERROR;
-          return GRPC_CHTTP2_STREAM_ERROR;
+          return GRPC_ERROR_REF(p->error);
       }
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_1;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_1:
@@ -188,7 +205,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       p->frame_size = ((uint32_t)*cur) << 24;
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_2;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_2:
@@ -196,7 +213,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       p->frame_size |= ((uint32_t)*cur) << 16;
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_3;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_3:
@@ -204,7 +221,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       p->frame_size |= ((uint32_t)*cur) << 8;
       if (++cur == end) {
         p->state = GRPC_CHTTP2_DATA_FH_4;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_CHTTP2_DATA_FH_4:
@@ -225,7 +242,7 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
       grpc_chttp2_list_add_parsing_seen_stream(transport_parsing,
                                                stream_parsing);
       if (cur == end) {
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       uint32_t remaining = (uint32_t)(end - cur);
       if (remaining == p->frame_size) {
@@ -233,19 +250,19 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
         grpc_chttp2_incoming_byte_stream_push(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg)));
-        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, 1,
-                                                  1);
+        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame,
+                                                  GRPC_ERROR_NONE, 1);
         p->parsing_frame = NULL;
         p->state = GRPC_CHTTP2_DATA_FH_0;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       } else if (remaining > p->frame_size) {
         stream_parsing->stats.incoming.data_bytes += p->frame_size;
         grpc_chttp2_incoming_byte_stream_push(
             exec_ctx, p->parsing_frame,
             gpr_slice_sub(slice, (size_t)(cur - beg),
                           (size_t)(cur + p->frame_size - beg)));
-        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, 1,
-                                                  1);
+        grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame,
+                                                  GRPC_ERROR_NONE, 1);
         p->parsing_frame = NULL;
         cur += p->frame_size;
         goto fh_0; /* loop */
@@ -256,9 +273,9 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
             gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg)));
         p->frame_size -= remaining;
         stream_parsing->stats.incoming.data_bytes += remaining;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
   }
 
-  GPR_UNREACHABLE_CODE(return GRPC_CHTTP2_CONNECTION_ERROR);
+  GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
 }
diff --git a/src/core/ext/transport/chttp2/transport/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h
index af71f483a2..a21a7942b9 100644
--- a/src/core/ext/transport/chttp2/transport/frame_data.h
+++ b/src/core/ext/transport/chttp2/transport/frame_data.h
@@ -66,6 +66,7 @@ typedef struct {
   uint8_t is_last_frame;
   uint8_t frame_type;
   uint32_t frame_size;
+  grpc_error *error;
 
   int is_frame_compressed;
   grpc_chttp2_incoming_frame_queue incoming_frames;
@@ -79,19 +80,19 @@ grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop(
     grpc_chttp2_incoming_frame_queue *q);
 
 /* initialize per-stream state for data frame parsing */
-grpc_chttp2_parse_error grpc_chttp2_data_parser_init(
-    grpc_chttp2_data_parser *parser);
+grpc_error *grpc_chttp2_data_parser_init(grpc_chttp2_data_parser *parser);
 
 void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_data_parser *parser);
 
 /* start processing a new data frame */
-grpc_chttp2_parse_error grpc_chttp2_data_parser_begin_frame(
-    grpc_chttp2_data_parser *parser, uint8_t flags);
+grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser,
+                                                uint8_t flags,
+                                                uint32_t stream_id);
 
 /* handle a slice of a data frame - is_last indicates the last slice of a
    frame */
-grpc_chttp2_parse_error grpc_chttp2_data_parser_parse(
+grpc_error *grpc_chttp2_data_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.c b/src/core/ext/transport/chttp2/transport/frame_goaway.c
index 69accb7696..299e27ad70 100644
--- a/src/core/ext/transport/chttp2/transport/frame_goaway.c
+++ b/src/core/ext/transport/chttp2/transport/frame_goaway.c
@@ -38,6 +38,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 
 void grpc_chttp2_goaway_parser_init(grpc_chttp2_goaway_parser *p) {
   p->debug_data = NULL;
@@ -47,11 +48,15 @@ void grpc_chttp2_goaway_parser_destroy(grpc_chttp2_goaway_parser *p) {
   gpr_free(p->debug_data);
 }
 
-grpc_chttp2_parse_error grpc_chttp2_goaway_parser_begin_frame(
-    grpc_chttp2_goaway_parser *p, uint32_t length, uint8_t flags) {
+grpc_error *grpc_chttp2_goaway_parser_begin_frame(grpc_chttp2_goaway_parser *p,
+                                                  uint32_t length,
+                                                  uint8_t flags) {
   if (length < 8) {
-    gpr_log(GPR_ERROR, "goaway frame too short (%d bytes)", length);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "goaway frame too short (%d bytes)", length);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
 
   gpr_free(p->debug_data);
@@ -59,10 +64,10 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_begin_frame(
   p->debug_data = gpr_malloc(p->debug_length);
   p->debug_pos = 0;
   p->state = GRPC_CHTTP2_GOAWAY_LSI0;
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
+grpc_error *grpc_chttp2_goaway_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
@@ -75,7 +80,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_LSI0:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_LSI0;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->last_stream_id = ((uint32_t)*cur) << 24;
       ++cur;
@@ -83,7 +88,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_LSI1:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_LSI1;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->last_stream_id |= ((uint32_t)*cur) << 16;
       ++cur;
@@ -91,7 +96,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_LSI2:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_LSI2;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->last_stream_id |= ((uint32_t)*cur) << 8;
       ++cur;
@@ -99,7 +104,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_LSI3:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_LSI3;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->last_stream_id |= ((uint32_t)*cur);
       ++cur;
@@ -107,7 +112,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_ERR0:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_ERR0;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->error_code = ((uint32_t)*cur) << 24;
       ++cur;
@@ -115,7 +120,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_ERR1:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_ERR1;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->error_code |= ((uint32_t)*cur) << 16;
       ++cur;
@@ -123,7 +128,7 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_ERR2:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_ERR2;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->error_code |= ((uint32_t)*cur) << 8;
       ++cur;
@@ -131,13 +136,14 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
     case GRPC_CHTTP2_GOAWAY_ERR3:
       if (cur == end) {
         p->state = GRPC_CHTTP2_GOAWAY_ERR3;
-        return GRPC_CHTTP2_PARSE_OK;
+        return GRPC_ERROR_NONE;
       }
       p->error_code |= ((uint32_t)*cur);
       ++cur;
     /* fallthrough */
     case GRPC_CHTTP2_GOAWAY_DEBUG:
-      memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
+      if (end != cur)
+        memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
       GPR_ASSERT((size_t)(end - cur) < UINT32_MAX - p->debug_pos);
       p->debug_pos += (uint32_t)(end - cur);
       p->state = GRPC_CHTTP2_GOAWAY_DEBUG;
@@ -150,9 +156,9 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
             gpr_slice_new(p->debug_data, p->debug_length, gpr_free);
         p->debug_data = NULL;
       }
-      return GRPC_CHTTP2_PARSE_OK;
+      return GRPC_ERROR_NONE;
   }
-  GPR_UNREACHABLE_CODE(return GRPC_CHTTP2_CONNECTION_ERROR);
+  GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
 }
 
 void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code,
diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.h b/src/core/ext/transport/chttp2/transport/frame_goaway.h
index 7c38b26a39..eb4303405a 100644
--- a/src/core/ext/transport/chttp2/transport/frame_goaway.h
+++ b/src/core/ext/transport/chttp2/transport/frame_goaway.h
@@ -63,9 +63,9 @@ typedef struct {
 
 void grpc_chttp2_goaway_parser_init(grpc_chttp2_goaway_parser *p);
 void grpc_chttp2_goaway_parser_destroy(grpc_chttp2_goaway_parser *p);
-grpc_chttp2_parse_error grpc_chttp2_goaway_parser_begin_frame(
+grpc_error *grpc_chttp2_goaway_parser_begin_frame(
     grpc_chttp2_goaway_parser *parser, uint32_t length, uint8_t flags);
-grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
+grpc_error *grpc_chttp2_goaway_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c
index 7e1815f0fe..1f814ab1bd 100644
--- a/src/core/ext/transport/chttp2/transport/frame_ping.c
+++ b/src/core/ext/transport/chttp2/transport/frame_ping.c
@@ -38,6 +38,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 
 gpr_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes) {
   gpr_slice slice = gpr_slice_malloc(9 + 8);
@@ -57,18 +58,22 @@ gpr_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes) {
   return slice;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_ping_parser_begin_frame(
-    grpc_chttp2_ping_parser *parser, uint32_t length, uint8_t flags) {
+grpc_error *grpc_chttp2_ping_parser_begin_frame(grpc_chttp2_ping_parser *parser,
+                                                uint32_t length,
+                                                uint8_t flags) {
   if (flags & 0xfe || length != 8) {
-    gpr_log(GPR_ERROR, "invalid ping: length=%d, flags=%02x", length, flags);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "invalid ping: length=%d, flags=%02x", length, flags);
+    grpc_error *error = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return error;
   }
   parser->byte = 0;
   parser->is_ack = flags;
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_ping_parser_parse(
+grpc_error *grpc_chttp2_ping_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
@@ -93,5 +98,5 @@ grpc_chttp2_parse_error grpc_chttp2_ping_parser_parse(
     }
   }
 
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.h b/src/core/ext/transport/chttp2/transport/frame_ping.h
index 4f7fcc1305..5a8723421c 100644
--- a/src/core/ext/transport/chttp2/transport/frame_ping.h
+++ b/src/core/ext/transport/chttp2/transport/frame_ping.h
@@ -46,9 +46,9 @@ typedef struct {
 
 gpr_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes);
 
-grpc_chttp2_parse_error grpc_chttp2_ping_parser_begin_frame(
-    grpc_chttp2_ping_parser *parser, uint32_t length, uint8_t flags);
-grpc_chttp2_parse_error grpc_chttp2_ping_parser_parse(
+grpc_error *grpc_chttp2_ping_parser_begin_frame(grpc_chttp2_ping_parser *parser,
+                                                uint32_t length, uint8_t flags);
+grpc_error *grpc_chttp2_ping_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c
index 22467e9ddd..e3a3c9e4a7 100644
--- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c
+++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c
@@ -34,7 +34,9 @@
 #include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h"
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/transport/frame.h"
 
@@ -45,15 +47,20 @@ gpr_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code,
   stats->framing_bytes += frame_size;
   uint8_t *p = GPR_SLICE_START_PTR(slice);
 
+  // Frame size.
   *p++ = 0;
   *p++ = 0;
   *p++ = 4;
+  // Frame type.
   *p++ = GRPC_CHTTP2_FRAME_RST_STREAM;
+  // Flags.
   *p++ = 0;
+  // Stream ID.
   *p++ = (uint8_t)(id >> 24);
   *p++ = (uint8_t)(id >> 16);
   *p++ = (uint8_t)(id >> 8);
   *p++ = (uint8_t)(id);
+  // Error code.
   *p++ = (uint8_t)(code >> 24);
   *p++ = (uint8_t)(code >> 16);
   *p++ = (uint8_t)(code >> 8);
@@ -62,18 +69,21 @@ gpr_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code,
   return slice;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_begin_frame(
+grpc_error *grpc_chttp2_rst_stream_parser_begin_frame(
     grpc_chttp2_rst_stream_parser *parser, uint32_t length, uint8_t flags) {
   if (length != 4) {
-    gpr_log(GPR_ERROR, "invalid rst_stream: length=%d, flags=%02x", length,
-            flags);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "invalid rst_stream: length=%d, flags=%02x", length,
+                 flags);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
   parser->byte = 0;
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_parse(
+grpc_error *grpc_chttp2_rst_stream_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
@@ -92,12 +102,15 @@ grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_parse(
   if (p->byte == 4) {
     GPR_ASSERT(is_last);
     stream_parsing->received_close = 1;
-    stream_parsing->saw_rst_stream = 1;
-    stream_parsing->rst_stream_reason = (((uint32_t)p->reason_bytes[0]) << 24) |
-                                        (((uint32_t)p->reason_bytes[1]) << 16) |
-                                        (((uint32_t)p->reason_bytes[2]) << 8) |
-                                        (((uint32_t)p->reason_bytes[3]));
+    if (stream_parsing->forced_close_error == GRPC_ERROR_NONE) {
+      stream_parsing->forced_close_error = grpc_error_set_int(
+          GRPC_ERROR_CREATE("RST_STREAM"), GRPC_ERROR_INT_HTTP2_ERROR,
+          (intptr_t)((((uint32_t)p->reason_bytes[0]) << 24) |
+                     (((uint32_t)p->reason_bytes[1]) << 16) |
+                     (((uint32_t)p->reason_bytes[2]) << 8) |
+                     (((uint32_t)p->reason_bytes[3]))));
+    }
   }
 
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.h b/src/core/ext/transport/chttp2/transport/frame_rst_stream.h
index 9c1e756a94..11cf94f3ea 100644
--- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.h
+++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.h
@@ -47,9 +47,9 @@ typedef struct {
 gpr_slice grpc_chttp2_rst_stream_create(uint32_t stream_id, uint32_t code,
                                         grpc_transport_one_way_stats *stats);
 
-grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_begin_frame(
+grpc_error *grpc_chttp2_rst_stream_parser_begin_frame(
     grpc_chttp2_rst_stream_parser *parser, uint32_t length, uint8_t flags);
-grpc_chttp2_parse_error grpc_chttp2_rst_stream_parser_parse(
+grpc_error *grpc_chttp2_rst_stream_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c
index a3c1e15f35..04b96c4cd9 100644
--- a/src/core/ext/transport/chttp2/transport/frame_settings.c
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.c
@@ -36,7 +36,9 @@
 
 #include <string.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
@@ -118,7 +120,7 @@ gpr_slice grpc_chttp2_settings_ack_create(void) {
   return output;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_settings_parser_begin_frame(
+grpc_error *grpc_chttp2_settings_parser_begin_frame(
     grpc_chttp2_settings_parser *parser, uint32_t length, uint8_t flags,
     uint32_t *settings) {
   parser->target_settings = settings;
@@ -129,31 +131,29 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_begin_frame(
   if (flags == GRPC_CHTTP2_FLAG_ACK) {
     parser->is_ack = 1;
     if (length != 0) {
-      gpr_log(GPR_ERROR, "non-empty settings ack frame received");
-      return GRPC_CHTTP2_CONNECTION_ERROR;
+      return GRPC_ERROR_CREATE("non-empty settings ack frame received");
     }
-    return GRPC_CHTTP2_PARSE_OK;
+    return GRPC_ERROR_NONE;
   } else if (flags != 0) {
-    gpr_log(GPR_ERROR, "invalid flags on settings frame");
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    return GRPC_ERROR_CREATE("invalid flags on settings frame");
   } else if (length % 6 != 0) {
-    gpr_log(GPR_ERROR, "settings frames must be a multiple of six bytes");
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    return GRPC_ERROR_CREATE("settings frames must be a multiple of six bytes");
   } else {
-    return GRPC_CHTTP2_PARSE_OK;
+    return GRPC_ERROR_NONE;
   }
 }
 
-grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
+grpc_error *grpc_chttp2_settings_parser_parse(
     grpc_exec_ctx *exec_ctx, void *p,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
   grpc_chttp2_settings_parser *parser = p;
   const uint8_t *cur = GPR_SLICE_START_PTR(slice);
   const uint8_t *end = GPR_SLICE_END_PTR(slice);
+  char *msg;
 
   if (parser->is_ack) {
-    return GRPC_CHTTP2_PARSE_OK;
+    return GRPC_ERROR_NONE;
   }
 
   for (;;) {
@@ -168,7 +168,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
             gpr_slice_buffer_add(&transport_parsing->qbuf,
                                  grpc_chttp2_settings_ack_create());
           }
-          return GRPC_CHTTP2_PARSE_OK;
+          return GRPC_ERROR_NONE;
         }
         parser->id = (uint16_t)(((uint16_t)*cur) << 8);
         cur++;
@@ -176,7 +176,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
       case GRPC_CHTTP2_SPS_ID1:
         if (cur == end) {
           parser->state = GRPC_CHTTP2_SPS_ID1;
-          return GRPC_CHTTP2_PARSE_OK;
+          return GRPC_ERROR_NONE;
         }
         parser->id = (uint16_t)(parser->id | (*cur));
         cur++;
@@ -184,7 +184,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
       case GRPC_CHTTP2_SPS_VAL0:
         if (cur == end) {
           parser->state = GRPC_CHTTP2_SPS_VAL0;
-          return GRPC_CHTTP2_PARSE_OK;
+          return GRPC_ERROR_NONE;
         }
         parser->value = ((uint32_t)*cur) << 24;
         cur++;
@@ -192,7 +192,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
       case GRPC_CHTTP2_SPS_VAL1:
         if (cur == end) {
           parser->state = GRPC_CHTTP2_SPS_VAL1;
-          return GRPC_CHTTP2_PARSE_OK;
+          return GRPC_ERROR_NONE;
         }
         parser->value |= ((uint32_t)*cur) << 16;
         cur++;
@@ -200,7 +200,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
       case GRPC_CHTTP2_SPS_VAL2:
         if (cur == end) {
           parser->state = GRPC_CHTTP2_SPS_VAL2;
-          return GRPC_CHTTP2_PARSE_OK;
+          return GRPC_ERROR_NONE;
         }
         parser->value |= ((uint32_t)*cur) << 8;
         cur++;
@@ -208,7 +208,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
       case GRPC_CHTTP2_SPS_VAL3:
         if (cur == end) {
           parser->state = GRPC_CHTTP2_SPS_VAL3;
-          return GRPC_CHTTP2_PARSE_OK;
+          return GRPC_ERROR_NONE;
         } else {
           parser->state = GRPC_CHTTP2_SPS_ID0;
         }
@@ -229,9 +229,11 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
                     transport_parsing->last_incoming_stream_id, sp->error_value,
                     gpr_slice_from_static_string("HTTP2 settings error"),
                     &transport_parsing->qbuf);
-                gpr_log(GPR_ERROR, "invalid value %u passed for %s",
-                        parser->value, sp->name);
-                return GRPC_CHTTP2_CONNECTION_ERROR;
+                gpr_asprintf(&msg, "invalid value %u passed for %s",
+                             parser->value, sp->name);
+                grpc_error *err = GRPC_ERROR_CREATE(msg);
+                gpr_free(msg);
+                return err;
             }
           }
           if (parser->id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE &&
@@ -249,7 +251,7 @@ grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
                     transport_parsing->is_client ? "CLI" : "SVR", parser->id,
                     parser->value);
           }
-        } else {
+        } else if (grpc_http_trace) {
           gpr_log(GPR_ERROR, "CHTTP2: Ignoring unknown setting %d (value %d)",
                   parser->id, parser->value);
         }
diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.h b/src/core/ext/transport/chttp2/transport/frame_settings.h
index d9e30f1ed0..f654c598c8 100644
--- a/src/core/ext/transport/chttp2/transport/frame_settings.h
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.h
@@ -92,10 +92,10 @@ gpr_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new,
 /* Create an ack settings frame */
 gpr_slice grpc_chttp2_settings_ack_create(void);
 
-grpc_chttp2_parse_error grpc_chttp2_settings_parser_begin_frame(
+grpc_error *grpc_chttp2_settings_parser_begin_frame(
     grpc_chttp2_settings_parser *parser, uint32_t length, uint8_t flags,
     uint32_t *settings);
-grpc_chttp2_parse_error grpc_chttp2_settings_parser_parse(
+grpc_error *grpc_chttp2_settings_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c
index 90243418bd..3cf848fd5c 100644
--- a/src/core/ext/transport/chttp2/transport/frame_window_update.c
+++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c
@@ -34,7 +34,9 @@
 #include "src/core/ext/transport/chttp2/transport/frame_window_update.h"
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 
 gpr_slice grpc_chttp2_window_update_create(
     uint32_t id, uint32_t window_update, grpc_transport_one_way_stats *stats) {
@@ -62,19 +64,22 @@ gpr_slice grpc_chttp2_window_update_create(
   return slice;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_window_update_parser_begin_frame(
+grpc_error *grpc_chttp2_window_update_parser_begin_frame(
     grpc_chttp2_window_update_parser *parser, uint32_t length, uint8_t flags) {
   if (flags || length != 4) {
-    gpr_log(GPR_ERROR, "invalid window update: length=%d, flags=%02x", length,
-            flags);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "invalid window update: length=%d, flags=%02x", length,
+                 flags);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
   parser->byte = 0;
   parser->amount = 0;
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
-grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse(
+grpc_error *grpc_chttp2_window_update_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
@@ -96,8 +101,11 @@ grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse(
   if (p->byte == 4) {
     uint32_t received_update = p->amount;
     if (received_update == 0 || (received_update & 0x80000000u)) {
-      gpr_log(GPR_ERROR, "invalid window update bytes: %d", p->amount);
-      return GRPC_CHTTP2_CONNECTION_ERROR;
+      char *msg;
+      gpr_asprintf(&msg, "invalid window update bytes: %d", p->amount);
+      grpc_error *err = GRPC_ERROR_CREATE(msg);
+      gpr_free(msg);
+      return err;
     }
     GPR_ASSERT(is_last);
 
@@ -115,5 +123,5 @@ grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse(
     }
   }
 
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.h b/src/core/ext/transport/chttp2/transport/frame_window_update.h
index d6e87b9329..1bcbbf9247 100644
--- a/src/core/ext/transport/chttp2/transport/frame_window_update.h
+++ b/src/core/ext/transport/chttp2/transport/frame_window_update.h
@@ -48,9 +48,9 @@ typedef struct {
 gpr_slice grpc_chttp2_window_update_create(uint32_t id, uint32_t window_delta,
                                            grpc_transport_one_way_stats *stats);
 
-grpc_chttp2_parse_error grpc_chttp2_window_update_parser_begin_frame(
+grpc_error *grpc_chttp2_window_update_parser_begin_frame(
     grpc_chttp2_window_update_parser *parser, uint32_t length, uint8_t flags);
-grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse(
+grpc_error *grpc_chttp2_window_update_parser_parse(
     grpc_exec_ctx *exec_ctx, void *parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c
index 687936bfd3..522455f7dc 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_parser.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c
@@ -46,6 +46,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
 #include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
@@ -77,63 +78,70 @@ typedef enum {
    a set of indirect jumps, and so not waste stack space. */
 
 /* forward declarations for parsing states */
-static int parse_begin(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                       const uint8_t *end);
-static int parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                       const uint8_t *end);
-static int parse_illegal_op(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                            const uint8_t *end);
-
-static int parse_string_prefix(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+static grpc_error *parse_begin(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
                                const uint8_t *end);
-static int parse_key_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                            const uint8_t *end);
-static int parse_value_string_with_indexed_key(grpc_chttp2_hpack_parser *p,
-                                               const uint8_t *cur,
-                                               const uint8_t *end);
-static int parse_value_string_with_literal_key(grpc_chttp2_hpack_parser *p,
-                                               const uint8_t *cur,
-                                               const uint8_t *end);
-
-static int parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end);
-static int parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end);
-static int parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end);
-static int parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end);
-static int parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end);
-static int parse_value5up(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                          const uint8_t *end);
-
-static int parse_indexed_field(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end);
-static int parse_indexed_field_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_lithdr_incidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end);
-static int parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_lithdr_notidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end);
-static int parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end);
-static int parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end);
-static int parse_max_tbl_size(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                              const uint8_t *end);
-static int parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+static grpc_error *parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                               const uint8_t *end, grpc_error *error);
+static grpc_error *still_parse_error(grpc_chttp2_hpack_parser *p,
+                                     const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_illegal_op(grpc_chttp2_hpack_parser *p,
+                                    const uint8_t *cur, const uint8_t *end);
+
+static grpc_error *parse_string_prefix(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_key_string(grpc_chttp2_hpack_parser *p,
+                                    const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_value_string_with_indexed_key(
+    grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_value_string_with_literal_key(
+    grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end);
+
+static grpc_error *parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end);
+static grpc_error *parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end);
+static grpc_error *parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end);
+static grpc_error *parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end);
+static grpc_error *parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
                                 const uint8_t *end);
+static grpc_error *parse_value5up(grpc_chttp2_hpack_parser *p,
+                                  const uint8_t *cur, const uint8_t *end);
+
+static grpc_error *parse_indexed_field(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_indexed_field_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_lithdr_incidx(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_lithdr_notidx(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end);
+static grpc_error *parse_max_tbl_size(grpc_chttp2_hpack_parser *p,
+                                      const uint8_t *cur, const uint8_t *end);
+static grpc_error *parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p,
+                                        const uint8_t *cur, const uint8_t *end);
 
 /* we translate the first byte of a hpack field into one of these decoding
    cases, then use a lookup table to jump directly to the appropriate parser.
@@ -631,19 +639,18 @@ static const uint8_t inverse_base64[256] = {
 };
 
 /* emission helpers */
-static int on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md,
-                  int add_to_table) {
+static grpc_error *on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md,
+                          int add_to_table) {
   if (add_to_table) {
-    if (!grpc_chttp2_hptbl_add(&p->table, md)) {
-      return 0;
-    }
+    grpc_error *err = grpc_chttp2_hptbl_add(&p->table, md);
+    if (err != GRPC_ERROR_NONE) return err;
   }
   if (p->on_header == NULL) {
     GRPC_MDELEM_UNREF(md);
-    return 0;
+    return GRPC_ERROR_CREATE("on_header callback not set");
   }
   p->on_header(p->on_header_user_data, md);
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 static grpc_mdstr *take_string(grpc_chttp2_hpack_parser *p,
@@ -654,70 +661,70 @@ static grpc_mdstr *take_string(grpc_chttp2_hpack_parser *p,
 }
 
 /* jump to the next state */
-static int parse_next(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                      const uint8_t *end) {
+static grpc_error *parse_next(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                              const uint8_t *end) {
   p->state = *p->next_state++;
   return p->state(p, cur, end);
 }
 
 /* begin parsing a header: all functionality is encoded into lookup tables
    above */
-static int parse_begin(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                       const uint8_t *end) {
+static grpc_error *parse_begin(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                               const uint8_t *end) {
   if (cur == end) {
     p->state = parse_begin;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   return first_byte_action[first_byte_lut[*cur]](p, cur, end);
 }
 
 /* stream dependency and prioritization data: we just skip it */
-static int parse_stream_weight(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *parse_stream_weight(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   if (cur == end) {
     p->state = parse_stream_weight;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   return p->after_prioritization(p, cur + 1, end);
 }
 
-static int parse_stream_dep3(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                             const uint8_t *end) {
+static grpc_error *parse_stream_dep3(grpc_chttp2_hpack_parser *p,
+                                     const uint8_t *cur, const uint8_t *end) {
   if (cur == end) {
     p->state = parse_stream_dep3;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   return parse_stream_weight(p, cur + 1, end);
 }
 
-static int parse_stream_dep2(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                             const uint8_t *end) {
+static grpc_error *parse_stream_dep2(grpc_chttp2_hpack_parser *p,
+                                     const uint8_t *cur, const uint8_t *end) {
   if (cur == end) {
     p->state = parse_stream_dep2;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   return parse_stream_dep3(p, cur + 1, end);
 }
 
-static int parse_stream_dep1(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                             const uint8_t *end) {
+static grpc_error *parse_stream_dep1(grpc_chttp2_hpack_parser *p,
+                                     const uint8_t *cur, const uint8_t *end) {
   if (cur == end) {
     p->state = parse_stream_dep1;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   return parse_stream_dep2(p, cur + 1, end);
 }
 
-static int parse_stream_dep0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                             const uint8_t *end) {
+static grpc_error *parse_stream_dep0(grpc_chttp2_hpack_parser *p,
+                                     const uint8_t *cur, const uint8_t *end) {
   if (cur == end) {
     p->state = parse_stream_dep0;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   return parse_stream_dep1(p, cur + 1, end);
@@ -725,30 +732,34 @@ static int parse_stream_dep0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 
 /* emit an indexed field; for now just logs it to console; jumps to
    begin the next field on completion */
-static int finish_indexed_field(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                                const uint8_t *end) {
+static grpc_error *finish_indexed_field(grpc_chttp2_hpack_parser *p,
+                                        const uint8_t *cur,
+                                        const uint8_t *end) {
   grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   if (md == NULL) {
-    if (grpc_http_trace) {
-      gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index);
-    }
-    return 0;
+    return grpc_error_set_int(
+        grpc_error_set_int(GRPC_ERROR_CREATE("Invalid HPACK index received"),
+                           GRPC_ERROR_INT_INDEX, (intptr_t)p->index),
+        GRPC_ERROR_INT_SIZE, (intptr_t)p->table.num_ents);
   }
   GRPC_MDELEM_REF(md);
-  return on_hdr(p, md, 0) && parse_begin(p, cur, end);
+  grpc_error *err = on_hdr(p, md, 0);
+  if (err != GRPC_ERROR_NONE) return err;
+  return parse_begin(p, cur, end);
 }
 
 /* parse an indexed field with index < 127 */
-static int parse_indexed_field(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *parse_indexed_field(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   p->dynamic_table_update_allowed = 0;
   p->index = (*cur) & 0x7f;
   return finish_indexed_field(p, cur + 1, end);
 }
 
 /* parse an indexed field with index >= 127 */
-static int parse_indexed_field_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_indexed_field_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       finish_indexed_field};
   p->dynamic_table_update_allowed = 0;
@@ -760,28 +771,34 @@ static int parse_indexed_field_x(grpc_chttp2_hpack_parser *p,
 
 /* finish a literal header with incremental indexing: just log, and jump to '
    begin */
-static int finish_lithdr_incidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                                const uint8_t *end) {
+static grpc_error *finish_lithdr_incidx(grpc_chttp2_hpack_parser *p,
+                                        const uint8_t *cur,
+                                        const uint8_t *end) {
   grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   GPR_ASSERT(md != NULL); /* handled in string parsing */
-  return on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key),
-                                                     take_string(p, &p->value)),
-                1) &&
-         parse_begin(p, cur, end);
+  grpc_error *err =
+      on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key),
+                                                  take_string(p, &p->value)),
+             1);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* finish a literal header with incremental indexing with no index */
-static int finish_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
-                                  const uint8_t *cur, const uint8_t *end) {
-  return on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key),
-                                                     take_string(p, &p->value)),
-                1) &&
-         parse_begin(p, cur, end);
+static grpc_error *finish_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
+                                          const uint8_t *cur,
+                                          const uint8_t *end) {
+  grpc_error *err =
+      on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key),
+                                                  take_string(p, &p->value)),
+             1);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* parse a literal header with incremental indexing; index < 63 */
-static int parse_lithdr_incidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *parse_lithdr_incidx(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_value_string_with_indexed_key, finish_lithdr_incidx};
   p->dynamic_table_update_allowed = 0;
@@ -791,8 +808,9 @@ static int parse_lithdr_incidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* parse a literal header with incremental indexing; index >= 63 */
-static int parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_string_prefix, parse_value_string_with_indexed_key,
       finish_lithdr_incidx};
@@ -804,8 +822,9 @@ static int parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
 }
 
 /* parse a literal header with incremental indexing; index = 0 */
-static int parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_key_string, parse_string_prefix,
       parse_value_string_with_literal_key, finish_lithdr_incidx_v};
@@ -815,28 +834,34 @@ static int parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
 }
 
 /* finish a literal header without incremental indexing */
-static int finish_lithdr_notidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                                const uint8_t *end) {
+static grpc_error *finish_lithdr_notidx(grpc_chttp2_hpack_parser *p,
+                                        const uint8_t *cur,
+                                        const uint8_t *end) {
   grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   GPR_ASSERT(md != NULL); /* handled in string parsing */
-  return on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key),
-                                                     take_string(p, &p->value)),
-                0) &&
-         parse_begin(p, cur, end);
+  grpc_error *err =
+      on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key),
+                                                  take_string(p, &p->value)),
+             0);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* finish a literal header without incremental indexing with index = 0 */
-static int finish_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
-                                  const uint8_t *cur, const uint8_t *end) {
-  return on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key),
-                                                     take_string(p, &p->value)),
-                0) &&
-         parse_begin(p, cur, end);
+static grpc_error *finish_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
+                                          const uint8_t *cur,
+                                          const uint8_t *end) {
+  grpc_error *err =
+      on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key),
+                                                  take_string(p, &p->value)),
+             0);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* parse a literal header without incremental indexing; index < 15 */
-static int parse_lithdr_notidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *parse_lithdr_notidx(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_value_string_with_indexed_key, finish_lithdr_notidx};
   p->dynamic_table_update_allowed = 0;
@@ -846,8 +871,9 @@ static int parse_lithdr_notidx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* parse a literal header without incremental indexing; index >= 15 */
-static int parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_string_prefix, parse_value_string_with_indexed_key,
       finish_lithdr_notidx};
@@ -859,8 +885,9 @@ static int parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
 }
 
 /* parse a literal header without incremental indexing; index == 0 */
-static int parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_key_string, parse_string_prefix,
       parse_value_string_with_literal_key, finish_lithdr_notidx_v};
@@ -870,28 +897,34 @@ static int parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
 }
 
 /* finish a literal header that is never indexed */
-static int finish_lithdr_nvridx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                                const uint8_t *end) {
+static grpc_error *finish_lithdr_nvridx(grpc_chttp2_hpack_parser *p,
+                                        const uint8_t *cur,
+                                        const uint8_t *end) {
   grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   GPR_ASSERT(md != NULL); /* handled in string parsing */
-  return on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key),
-                                                     take_string(p, &p->value)),
-                0) &&
-         parse_begin(p, cur, end);
+  grpc_error *err =
+      on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key),
+                                                  take_string(p, &p->value)),
+             0);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* finish a literal header that is never indexed with an extra value */
-static int finish_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
-                                  const uint8_t *cur, const uint8_t *end) {
-  return on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key),
-                                                     take_string(p, &p->value)),
-                0) &&
-         parse_begin(p, cur, end);
+static grpc_error *finish_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
+                                          const uint8_t *cur,
+                                          const uint8_t *end) {
+  grpc_error *err =
+      on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key),
+                                                  take_string(p, &p->value)),
+             0);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* parse a literal header that is never indexed; index < 15 */
-static int parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_value_string_with_indexed_key, finish_lithdr_nvridx};
   p->dynamic_table_update_allowed = 0;
@@ -901,8 +934,9 @@ static int parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* parse a literal header that is never indexed; index >= 15 */
-static int parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_string_prefix, parse_value_string_with_indexed_key,
       finish_lithdr_nvridx};
@@ -914,8 +948,9 @@ static int parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
 }
 
 /* parse a literal header that is never indexed; index == 0 */
-static int parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
-                                 const uint8_t *cur, const uint8_t *end) {
+static grpc_error *parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
+                                         const uint8_t *cur,
+                                         const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       parse_key_string, parse_string_prefix,
       parse_value_string_with_literal_key, finish_lithdr_nvridx_v};
@@ -925,20 +960,25 @@ static int parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
 }
 
 /* finish parsing a max table size change */
-static int finish_max_tbl_size(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *finish_max_tbl_size(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   if (grpc_http_trace) {
     gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index);
   }
-  return grpc_chttp2_hptbl_set_current_table_size(&p->table, p->index) &&
-         parse_begin(p, cur, end);
+  grpc_error *err =
+      grpc_chttp2_hptbl_set_current_table_size(&p->table, p->index);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_begin(p, cur, end);
 }
 
 /* parse a max table size change, max size < 15 */
-static int parse_max_tbl_size(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                              const uint8_t *end) {
+static grpc_error *parse_max_tbl_size(grpc_chttp2_hpack_parser *p,
+                                      const uint8_t *cur, const uint8_t *end) {
   if (p->dynamic_table_update_allowed == 0) {
-    return 0;
+    return parse_error(
+        p, cur, end,
+        GRPC_ERROR_CREATE(
+            "More than two max table size changes in a single frame"));
   }
   p->dynamic_table_update_allowed--;
   p->index = (*cur) & 0x1f;
@@ -946,12 +986,16 @@ static int parse_max_tbl_size(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* parse a max table size change, max size >= 15 */
-static int parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                                const uint8_t *end) {
+static grpc_error *parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p,
+                                        const uint8_t *cur,
+                                        const uint8_t *end) {
   static const grpc_chttp2_hpack_parser_state and_then[] = {
       finish_max_tbl_size};
   if (p->dynamic_table_update_allowed == 0) {
-    return 0;
+    return parse_error(
+        p, cur, end,
+        GRPC_ERROR_CREATE(
+            "More than two max table size changes in a single frame"));
   }
   p->dynamic_table_update_allowed--;
   p->next_state = and_then;
@@ -961,28 +1005,38 @@ static int parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* a parse error: jam the parse state into parse_error, and return error */
-static int parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                       const uint8_t *end) {
-  p->state = parse_error;
-  return 0;
+static grpc_error *parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                               const uint8_t *end, grpc_error *err) {
+  GPR_ASSERT(err != GRPC_ERROR_NONE);
+  if (p->last_error == GRPC_ERROR_NONE) {
+    p->last_error = GRPC_ERROR_REF(err);
+  }
+  p->state = still_parse_error;
+  return err;
+}
+
+static grpc_error *still_parse_error(grpc_chttp2_hpack_parser *p,
+                                     const uint8_t *cur, const uint8_t *end) {
+  return GRPC_ERROR_REF(p->last_error);
 }
 
-static int parse_illegal_op(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                            const uint8_t *end) {
+static grpc_error *parse_illegal_op(grpc_chttp2_hpack_parser *p,
+                                    const uint8_t *cur, const uint8_t *end) {
   GPR_ASSERT(cur != end);
-  if (grpc_http_trace) {
-    gpr_log(GPR_DEBUG, "Illegal hpack op code %d", *cur);
-  }
-  return parse_error(p, cur, end);
+  char *msg;
+  gpr_asprintf(&msg, "Illegal hpack op code %d", *cur);
+  grpc_error *err = GRPC_ERROR_CREATE(msg);
+  gpr_free(msg);
+  return parse_error(p, cur, end, err);
 }
 
 /* parse the 1st byte of a varint into p->parsing.value
    no overflow is possible */
-static int parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end) {
+static grpc_error *parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end) {
   if (cur == end) {
     p->state = parse_value0;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   *p->parsing.value += (*cur) & 0x7f;
@@ -996,11 +1050,11 @@ static int parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 
 /* parse the 2nd byte of a varint into p->parsing.value
    no overflow is possible */
-static int parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end) {
+static grpc_error *parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end) {
   if (cur == end) {
     p->state = parse_value1;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   *p->parsing.value += (((uint32_t)*cur) & 0x7f) << 7;
@@ -1014,11 +1068,11 @@ static int parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 
 /* parse the 3rd byte of a varint into p->parsing.value
    no overflow is possible */
-static int parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end) {
+static grpc_error *parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end) {
   if (cur == end) {
     p->state = parse_value2;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   *p->parsing.value += (((uint32_t)*cur) & 0x7f) << 14;
@@ -1032,11 +1086,11 @@ static int parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 
 /* parse the 4th byte of a varint into p->parsing.value
    no overflow is possible */
-static int parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end) {
+static grpc_error *parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end) {
   if (cur == end) {
     p->state = parse_value3;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   *p->parsing.value += (((uint32_t)*cur) & 0x7f) << 21;
@@ -1050,15 +1104,16 @@ static int parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 
 /* parse the 5th byte of a varint into p->parsing.value
    depending on the byte, we may overflow, and care must be taken */
-static int parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end) {
+static grpc_error *parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end) {
   uint8_t c;
   uint32_t cur_value;
   uint32_t add_value;
+  char *msg;
 
   if (cur == end) {
     p->state = parse_value4;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   c = (*cur) & 0x7f;
@@ -1081,48 +1136,49 @@ static int parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
   }
 
 error:
-  if (grpc_http_trace) {
-    gpr_log(GPR_ERROR,
-            "integer overflow in hpack integer decoding: have 0x%08x, "
-            "got byte 0x%02x on byte 5",
-            *p->parsing.value, *cur);
-  }
-  return parse_error(p, cur, end);
+  gpr_asprintf(&msg,
+               "integer overflow in hpack integer decoding: have 0x%08x, "
+               "got byte 0x%02x on byte 5",
+               *p->parsing.value, *cur);
+  grpc_error *err = GRPC_ERROR_CREATE(msg);
+  gpr_free(msg);
+  return parse_error(p, cur, end, err);
 }
 
 /* parse any trailing bytes in a varint: it's possible to append an arbitrary
    number of 0x80's and not affect the value - a zero will terminate - and
    anything else will overflow */
-static int parse_value5up(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                          const uint8_t *end) {
+static grpc_error *parse_value5up(grpc_chttp2_hpack_parser *p,
+                                  const uint8_t *cur, const uint8_t *end) {
   while (cur != end && *cur == 0x80) {
     ++cur;
   }
 
   if (cur == end) {
     p->state = parse_value5up;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   if (*cur == 0) {
     return parse_next(p, cur + 1, end);
   }
 
-  if (grpc_http_trace) {
-    gpr_log(GPR_ERROR,
-            "integer overflow in hpack integer decoding: have 0x%08x, "
-            "got byte 0x%02x sometime after byte 5",
-            *p->parsing.value, *cur);
-  }
-  return parse_error(p, cur, end);
+  char *msg;
+  gpr_asprintf(&msg,
+               "integer overflow in hpack integer decoding: have 0x%08x, "
+               "got byte 0x%02x sometime after byte 5",
+               *p->parsing.value, *cur);
+  grpc_error *err = GRPC_ERROR_CREATE(msg);
+  gpr_free(msg);
+  return parse_error(p, cur, end, err);
 }
 
 /* parse a string prefix */
-static int parse_string_prefix(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                               const uint8_t *end) {
+static grpc_error *parse_string_prefix(grpc_chttp2_hpack_parser *p,
+                                       const uint8_t *cur, const uint8_t *end) {
   if (cur == end) {
     p->state = parse_string_prefix;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   p->strlen = (*cur) & 0x7f;
@@ -1138,6 +1194,7 @@ static int parse_string_prefix(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 /* append some bytes to a string */
 static void append_bytes(grpc_chttp2_hpack_parser_string *str,
                          const uint8_t *data, size_t length) {
+  if (length == 0) return;
   if (length + str->length > str->capacity) {
     GPR_ASSERT(str->length + length <= UINT32_MAX);
     str->capacity = (uint32_t)(str->length + length);
@@ -1148,25 +1205,26 @@ static void append_bytes(grpc_chttp2_hpack_parser_string *str,
   str->length += (uint32_t)length;
 }
 
-static int append_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                         const uint8_t *end) {
+static grpc_error *append_string(grpc_chttp2_hpack_parser *p,
+                                 const uint8_t *cur, const uint8_t *end) {
   grpc_chttp2_hpack_parser_string *str = p->parsing.str;
   uint32_t bits;
   uint8_t decoded[3];
   switch ((binary_state)p->binary) {
     case NOT_BINARY:
       append_bytes(str, cur, (size_t)(end - cur));
-      return 1;
+      return GRPC_ERROR_NONE;
     b64_byte0:
     case B64_BYTE0:
       if (cur == end) {
         p->binary = B64_BYTE0;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
       bits = inverse_base64[*cur];
       ++cur;
       if (bits == 255)
-        return 0;
+        return parse_error(p, cur, end,
+                           GRPC_ERROR_CREATE("Illegal base64 character"));
       else if (bits == 64)
         goto b64_byte0;
       p->base64_buffer = bits << 18;
@@ -1175,12 +1233,13 @@ static int append_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
     case B64_BYTE1:
       if (cur == end) {
         p->binary = B64_BYTE1;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
       bits = inverse_base64[*cur];
       ++cur;
       if (bits == 255)
-        return 0;
+        return parse_error(p, cur, end,
+                           GRPC_ERROR_CREATE("Illegal base64 character"));
       else if (bits == 64)
         goto b64_byte1;
       p->base64_buffer |= bits << 12;
@@ -1189,12 +1248,13 @@ static int append_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
     case B64_BYTE2:
       if (cur == end) {
         p->binary = B64_BYTE2;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
       bits = inverse_base64[*cur];
       ++cur;
       if (bits == 255)
-        return 0;
+        return parse_error(p, cur, end,
+                           GRPC_ERROR_CREATE("Illegal base64 character"));
       else if (bits == 64)
         goto b64_byte2;
       p->base64_buffer |= bits << 6;
@@ -1203,12 +1263,13 @@ static int append_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
     case B64_BYTE3:
       if (cur == end) {
         p->binary = B64_BYTE3;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
       bits = inverse_base64[*cur];
       ++cur;
       if (bits == 255)
-        return 0;
+        return parse_error(p, cur, end,
+                           GRPC_ERROR_CREATE("Illegal base64 character"));
       else if (bits == 64)
         goto b64_byte3;
       p->base64_buffer |= bits;
@@ -1219,11 +1280,13 @@ static int append_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
       append_bytes(str, decoded, 3);
       goto b64_byte0;
   }
-  GPR_UNREACHABLE_CODE(return 1);
+  GPR_UNREACHABLE_CODE(return parse_error(
+      p, cur, end, GRPC_ERROR_CREATE("Should never reach here")));
 }
 
 /* append a null terminator to a string */
-static int finish_str(grpc_chttp2_hpack_parser *p) {
+static grpc_error *finish_str(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                              const uint8_t *end) {
   uint8_t terminator = 0;
   uint8_t decoded[2];
   uint32_t bits;
@@ -1234,14 +1297,18 @@ static int finish_str(grpc_chttp2_hpack_parser *p) {
     case B64_BYTE0:
       break;
     case B64_BYTE1:
-      gpr_log(GPR_ERROR, "illegal base64 encoding");
-      return 0; /* illegal encoding */
+      return parse_error(
+          p, cur, end,
+          GRPC_ERROR_CREATE("illegal base64 encoding")); /* illegal encoding */
     case B64_BYTE2:
       bits = p->base64_buffer;
       if (bits & 0xffff) {
-        gpr_log(GPR_ERROR, "trailing bits in base64 encoding: 0x%04x",
-                bits & 0xffff);
-        return 0;
+        char *msg;
+        gpr_asprintf(&msg, "trailing bits in base64 encoding: 0x%04x",
+                     bits & 0xffff);
+        grpc_error *err = GRPC_ERROR_CREATE(msg);
+        gpr_free(msg);
+        return parse_error(p, cur, end, err);
       }
       decoded[0] = (uint8_t)(bits >> 16);
       append_bytes(str, decoded, 1);
@@ -1249,9 +1316,12 @@ static int finish_str(grpc_chttp2_hpack_parser *p) {
     case B64_BYTE3:
       bits = p->base64_buffer;
       if (bits & 0xff) {
-        gpr_log(GPR_ERROR, "trailing bits in base64 encoding: 0x%02x",
-                bits & 0xff);
-        return 0;
+        char *msg;
+        gpr_asprintf(&msg, "trailing bits in base64 encoding: 0x%02x",
+                     bits & 0xff);
+        grpc_error *err = GRPC_ERROR_CREATE(msg);
+        gpr_free(msg);
+        return parse_error(p, cur, end, err);
       }
       decoded[0] = (uint8_t)(bits >> 16);
       decoded[1] = (uint8_t)(bits >> 8);
@@ -1260,38 +1330,42 @@ static int finish_str(grpc_chttp2_hpack_parser *p) {
   }
   append_bytes(str, &terminator, 1);
   p->parsing.str->length--; /* don't actually count the null terminator */
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 /* decode a nibble from a huffman encoded stream */
-static int huff_nibble(grpc_chttp2_hpack_parser *p, uint8_t nibble) {
+static grpc_error *huff_nibble(grpc_chttp2_hpack_parser *p, uint8_t nibble) {
   int16_t emit = emit_sub_tbl[16 * emit_tbl[p->huff_state] + nibble];
   int16_t next = next_sub_tbl[16 * next_tbl[p->huff_state] + nibble];
   if (emit != -1) {
     if (emit >= 0 && emit < 256) {
       uint8_t c = (uint8_t)emit;
-      if (!append_string(p, &c, (&c) + 1)) return 0;
+      grpc_error *err = append_string(p, &c, (&c) + 1);
+      if (err != GRPC_ERROR_NONE) return err;
     } else {
       assert(emit == 256);
     }
   }
   p->huff_state = next;
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 /* decode full bytes from a huffman encoded stream */
-static int add_huff_bytes(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                          const uint8_t *end) {
+static grpc_error *add_huff_bytes(grpc_chttp2_hpack_parser *p,
+                                  const uint8_t *cur, const uint8_t *end) {
   for (; cur != end; ++cur) {
-    if (!huff_nibble(p, *cur >> 4) || !huff_nibble(p, *cur & 0xf)) return 0;
+    grpc_error *err = huff_nibble(p, *cur >> 4);
+    if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+    err = huff_nibble(p, *cur & 0xf);
+    if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   }
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 /* decode some string bytes based on the current decoding mode
    (huffman or not) */
-static int add_str_bytes(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                         const uint8_t *end) {
+static grpc_error *add_str_bytes(grpc_chttp2_hpack_parser *p,
+                                 const uint8_t *cur, const uint8_t *end) {
   if (p->huff) {
     return add_huff_bytes(p, cur, end);
   } else {
@@ -1300,26 +1374,31 @@ static int add_str_bytes(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* parse a string - tries to do large chunks at a time */
-static int parse_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                        const uint8_t *end) {
+static grpc_error *parse_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
+                                const uint8_t *end) {
   size_t remaining = p->strlen - p->strgot;
   size_t given = (size_t)(end - cur);
   if (remaining <= given) {
-    return add_str_bytes(p, cur, cur + remaining) && finish_str(p) &&
-           parse_next(p, cur + remaining, end);
+    grpc_error *err = add_str_bytes(p, cur, cur + remaining);
+    if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+    err = finish_str(p, cur + remaining, end);
+    if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+    return parse_next(p, cur + remaining, end);
   } else {
-    if (!add_str_bytes(p, cur, cur + given)) return 0;
+    grpc_error *err = add_str_bytes(p, cur, cur + given);
+    if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
     GPR_ASSERT(given <= UINT32_MAX - p->strgot);
     p->strgot += (uint32_t)given;
     p->state = parse_string;
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 }
 
 /* begin parsing a string - performs setup, calls parse_string */
-static int begin_parse_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                              const uint8_t *end, uint8_t binary,
-                              grpc_chttp2_hpack_parser_string *str) {
+static grpc_error *begin_parse_string(grpc_chttp2_hpack_parser *p,
+                                      const uint8_t *cur, const uint8_t *end,
+                                      uint8_t binary,
+                                      grpc_chttp2_hpack_parser_string *str) {
   p->strgot = 0;
   str->length = 0;
   p->parsing.str = str;
@@ -1329,58 +1408,50 @@ static int begin_parse_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 }
 
 /* parse the key string */
-static int parse_key_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                            const uint8_t *end) {
+static grpc_error *parse_key_string(grpc_chttp2_hpack_parser *p,
+                                    const uint8_t *cur, const uint8_t *end) {
   return begin_parse_string(p, cur, end, NOT_BINARY, &p->key);
 }
 
 /* check if a key represents a binary header or not */
-typedef enum { BINARY_HEADER, PLAINTEXT_HEADER, ERROR_HEADER } is_binary_header;
 
-static is_binary_header is_binary_literal_header(grpc_chttp2_hpack_parser *p) {
-  return grpc_is_binary_header(p->key.str, p->key.length) ? BINARY_HEADER
-                                                          : PLAINTEXT_HEADER;
+static bool is_binary_literal_header(grpc_chttp2_hpack_parser *p) {
+  return grpc_is_binary_header(p->key.str, p->key.length);
 }
 
-static is_binary_header is_binary_indexed_header(grpc_chttp2_hpack_parser *p) {
+static grpc_error *is_binary_indexed_header(grpc_chttp2_hpack_parser *p,
+                                            bool *is) {
   grpc_mdelem *elem = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   if (!elem) {
-    if (grpc_http_trace) {
-      gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index);
-    }
-    return ERROR_HEADER;
+    return grpc_error_set_int(
+        grpc_error_set_int(GRPC_ERROR_CREATE("Invalid HPACK index received"),
+                           GRPC_ERROR_INT_INDEX, (intptr_t)p->index),
+        GRPC_ERROR_INT_SIZE, (intptr_t)p->table.num_ents);
   }
-  return grpc_is_binary_header(
-             (const char *)GPR_SLICE_START_PTR(elem->key->slice),
-             GPR_SLICE_LENGTH(elem->key->slice))
-             ? BINARY_HEADER
-             : PLAINTEXT_HEADER;
+  *is =
+      grpc_is_binary_header((const char *)GPR_SLICE_START_PTR(elem->key->slice),
+                            GPR_SLICE_LENGTH(elem->key->slice));
+  return GRPC_ERROR_NONE;
 }
 
 /* parse the value string */
-static int parse_value_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
-                              const uint8_t *end, is_binary_header type) {
-  switch (type) {
-    case BINARY_HEADER:
-      return begin_parse_string(p, cur, end, B64_BYTE0, &p->value);
-    case PLAINTEXT_HEADER:
-      return begin_parse_string(p, cur, end, NOT_BINARY, &p->value);
-    case ERROR_HEADER:
-      return 0;
-  }
-  /* Add code to prevent return without value error */
-  GPR_UNREACHABLE_CODE(return 0);
+static grpc_error *parse_value_string(grpc_chttp2_hpack_parser *p,
+                                      const uint8_t *cur, const uint8_t *end,
+                                      bool is_binary) {
+  return begin_parse_string(p, cur, end, is_binary ? B64_BYTE0 : NOT_BINARY,
+                            &p->value);
 }
 
-static int parse_value_string_with_indexed_key(grpc_chttp2_hpack_parser *p,
-                                               const uint8_t *cur,
-                                               const uint8_t *end) {
-  return parse_value_string(p, cur, end, is_binary_indexed_header(p));
+static grpc_error *parse_value_string_with_indexed_key(
+    grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) {
+  bool is_binary = false;
+  grpc_error *err = is_binary_indexed_header(p, &is_binary);
+  if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
+  return parse_value_string(p, cur, end, is_binary);
 }
 
-static int parse_value_string_with_literal_key(grpc_chttp2_hpack_parser *p,
-                                               const uint8_t *cur,
-                                               const uint8_t *end) {
+static grpc_error *parse_value_string_with_literal_key(
+    grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) {
   return parse_value_string(p, cur, end, is_binary_literal_header(p));
 }
 
@@ -1397,6 +1468,7 @@ void grpc_chttp2_hpack_parser_init(grpc_chttp2_hpack_parser *p) {
   p->value.capacity = 0;
   p->value.length = 0;
   p->dynamic_table_update_allowed = 2;
+  p->last_error = GRPC_ERROR_NONE;
   grpc_chttp2_hptbl_init(&p->table);
 }
 
@@ -1407,12 +1479,14 @@ void grpc_chttp2_hpack_parser_set_has_priority(grpc_chttp2_hpack_parser *p) {
 
 void grpc_chttp2_hpack_parser_destroy(grpc_chttp2_hpack_parser *p) {
   grpc_chttp2_hptbl_destroy(&p->table);
+  GRPC_ERROR_UNREF(p->last_error);
   gpr_free(p->key.str);
   gpr_free(p->value.str);
 }
 
-int grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p,
-                                   const uint8_t *beg, const uint8_t *end) {
+grpc_error *grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p,
+                                           const uint8_t *beg,
+                                           const uint8_t *end) {
   /* TODO(ctiller): limit the distance of end from beg, and perform multiple
      steps in the event of a large chunk of data to limit
      stack space usage when no tail call optimization is
@@ -1420,7 +1494,7 @@ int grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p,
   return p->state(p, beg, end);
 }
 
-grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
+grpc_error *grpc_chttp2_header_parser_parse(
     grpc_exec_ctx *exec_ctx, void *hpack_parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
@@ -1429,22 +1503,26 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
   if (stream_parsing != NULL) {
     stream_parsing->stats.incoming.header_bytes += GPR_SLICE_LENGTH(slice);
   }
-  if (!grpc_chttp2_hpack_parser_parse(parser, GPR_SLICE_START_PTR(slice),
-                                      GPR_SLICE_END_PTR(slice))) {
+  grpc_error *error = grpc_chttp2_hpack_parser_parse(
+      parser, GPR_SLICE_START_PTR(slice), GPR_SLICE_END_PTR(slice));
+  if (error != GRPC_ERROR_NONE) {
     GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    return error;
   }
   if (is_last) {
     if (parser->is_boundary && parser->state != parse_begin) {
-      gpr_log(GPR_ERROR,
-              "end of header frame not aligned with a hpack record boundary");
       GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0);
-      return GRPC_CHTTP2_CONNECTION_ERROR;
+      return GRPC_ERROR_CREATE(
+          "end of header frame not aligned with a hpack record boundary");
     }
     /* need to check for null stream: this can occur if we receive an invalid
        stream id on a header */
     if (stream_parsing != NULL) {
       if (parser->is_boundary) {
+        if (stream_parsing->header_frames_received ==
+            GPR_ARRAY_SIZE(stream_parsing->got_metadata_on_parse)) {
+          return GRPC_ERROR_CREATE("Too many trailer frames");
+        }
         stream_parsing
             ->got_metadata_on_parse[stream_parsing->header_frames_received] = 1;
         stream_parsing->header_frames_received++;
@@ -1462,5 +1540,5 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
     parser->dynamic_table_update_allowed = 2;
   }
   GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0);
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h
index 855d6c5d52..78eb38db5e 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_parser.h
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h
@@ -44,9 +44,8 @@
 
 typedef struct grpc_chttp2_hpack_parser grpc_chttp2_hpack_parser;
 
-typedef int (*grpc_chttp2_hpack_parser_state)(grpc_chttp2_hpack_parser *p,
-                                              const uint8_t *beg,
-                                              const uint8_t *end);
+typedef grpc_error *(*grpc_chttp2_hpack_parser_state)(
+    grpc_chttp2_hpack_parser *p, const uint8_t *beg, const uint8_t *end);
 
 typedef struct {
   char *str;
@@ -59,6 +58,8 @@ struct grpc_chttp2_hpack_parser {
   void (*on_header)(void *user_data, grpc_mdelem *md);
   void *on_header_user_data;
 
+  grpc_error *last_error;
+
   /* current parse state - or a function that implements it */
   grpc_chttp2_hpack_parser_state state;
   /* future states dependent on the opening op code */
@@ -103,12 +104,13 @@ void grpc_chttp2_hpack_parser_destroy(grpc_chttp2_hpack_parser *p);
 void grpc_chttp2_hpack_parser_set_has_priority(grpc_chttp2_hpack_parser *p);
 
 /* returns 1 on success, 0 on error */
-int grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p,
-                                   const uint8_t *beg, const uint8_t *end);
+grpc_error *grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p,
+                                           const uint8_t *beg,
+                                           const uint8_t *end);
 
 /* wraps grpc_chttp2_hpack_parser_parse to provide a frame level parser for
    the transport */
-grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
+grpc_error *grpc_chttp2_header_parser_parse(
     grpc_exec_ctx *exec_ctx, void *hpack_parser,
     grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c
index 295f31c44f..2b73ec969e 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_table.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_table.c
@@ -38,6 +38,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 
 #include "src/core/lib/support/murmur_hash.h"
 
@@ -262,18 +263,19 @@ void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl *tbl,
   tbl->max_bytes = max_bytes;
 }
 
-int grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl,
-                                             uint32_t bytes) {
+grpc_error *grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl,
+                                                     uint32_t bytes) {
   if (tbl->current_table_bytes == bytes) {
-    return 1;
+    return GRPC_ERROR_NONE;
   }
   if (bytes > tbl->max_bytes) {
-    if (grpc_http_trace) {
-      gpr_log(GPR_ERROR,
-              "Attempt to make hpack table %d bytes when max is %d bytes",
-              bytes, tbl->max_bytes);
-    }
-    return 0;
+    char *msg;
+    gpr_asprintf(&msg,
+                 "Attempt to make hpack table %d bytes when max is %d bytes",
+                 bytes, tbl->max_bytes);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
   if (grpc_http_trace) {
     gpr_log(GPR_DEBUG, "Update hpack parser table size to %d", bytes);
@@ -291,23 +293,25 @@ int grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl,
       rebuild_ents(tbl, new_cap);
     }
   }
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
-int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) {
+grpc_error *grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) {
   /* determine how many bytes of buffer this entry represents */
   size_t elem_bytes = GPR_SLICE_LENGTH(md->key->slice) +
                       GPR_SLICE_LENGTH(md->value->slice) +
                       GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD;
 
   if (tbl->current_table_bytes > tbl->max_bytes) {
-    if (grpc_http_trace) {
-      gpr_log(GPR_ERROR,
-              "HPACK max table size reduced to %d but not reflected by hpack "
-              "stream (still at %d)",
-              tbl->max_bytes, tbl->current_table_bytes);
-    }
-    return 0;
+    char *msg;
+    gpr_asprintf(
+        &msg,
+        "HPACK max table size reduced to %d but not reflected by hpack "
+        "stream (still at %d)",
+        tbl->max_bytes, tbl->current_table_bytes);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
 
   /* we can't add elements bigger than the max table size */
@@ -324,7 +328,7 @@ int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) {
     while (tbl->num_ents) {
       evict1(tbl);
     }
-    return 1;
+    return GRPC_ERROR_NONE;
   }
 
   /* evict entries to ensure no overflow */
@@ -339,7 +343,7 @@ int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) {
   /* update accounting values */
   tbl->num_ents++;
   tbl->mem_used += (uint32_t)elem_bytes;
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find(
diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h
index 074fea36d8..45bd9255bf 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_table.h
+++ b/src/core/ext/transport/chttp2/transport/hpack_table.h
@@ -36,6 +36,7 @@
 
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
+#include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/transport/metadata.h"
 
 /* HPACK header table */
@@ -87,15 +88,15 @@ void grpc_chttp2_hptbl_init(grpc_chttp2_hptbl *tbl);
 void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl *tbl);
 void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl *tbl,
                                      uint32_t max_bytes);
-int grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl,
-                                             uint32_t bytes);
+grpc_error *grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl,
+                                                     uint32_t bytes);
 
 /* lookup a table entry based on its hpack index */
 grpc_mdelem *grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl *tbl,
                                       uint32_t index);
 /* add a table entry to the index */
-int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl,
-                          grpc_mdelem *md) GRPC_MUST_USE_RESULT;
+grpc_error *grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl,
+                                  grpc_mdelem *md) GRPC_MUST_USE_RESULT;
 /* Find a key/value pair in the table... returns the index in the table of the
    most similar entry, or 0 if the value was not found */
 typedef struct {
diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c
index db21744f0c..3e463a7995 100644
--- a/src/core/ext/transport/chttp2/transport/incoming_metadata.c
+++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c
@@ -65,6 +65,7 @@ void grpc_chttp2_incoming_metadata_buffer_add(
         gpr_realloc(buffer->elems, sizeof(*buffer->elems) * buffer->capacity);
   }
   buffer->elems[buffer->count++].md = elem;
+  buffer->size += GRPC_MDELEM_LENGTH(elem);
 }
 
 void grpc_chttp2_incoming_metadata_buffer_set_deadline(
diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.h b/src/core/ext/transport/chttp2/transport/incoming_metadata.h
index 17ecf8e181..df4343b93e 100644
--- a/src/core/ext/transport/chttp2/transport/incoming_metadata.h
+++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.h
@@ -42,6 +42,7 @@ typedef struct {
   size_t capacity;
   gpr_timespec deadline;
   int published;
+  size_t size;  // total size of metadata
 } grpc_chttp2_incoming_metadata_buffer;
 
 /** assumes everything initially zeroed */
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 8c4fa2d34a..b5180c6fc8 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -156,7 +156,7 @@ struct grpc_chttp2_incoming_byte_stream {
   grpc_byte_stream base;
   gpr_refcount refs;
   struct grpc_chttp2_incoming_byte_stream *next_message;
-  int failed;
+  grpc_error *error;
 
   grpc_chttp2_transport *transport;
   grpc_chttp2_stream *stream;
@@ -265,6 +265,7 @@ struct grpc_chttp2_transport_parsing {
   uint8_t incoming_frame_type;
   uint8_t incoming_frame_flags;
   uint8_t header_eof;
+  bool is_first_frame;
   uint32_t expect_continuation_stream_id;
   uint32_t incoming_frame_size;
   uint32_t incoming_stream_id;
@@ -275,10 +276,10 @@ struct grpc_chttp2_transport_parsing {
   /* active parser */
   void *parser_data;
   grpc_chttp2_stream_parsing *incoming_stream;
-  grpc_chttp2_parse_error (*parser)(
-      grpc_exec_ctx *exec_ctx, void *parser_user_data,
-      grpc_chttp2_transport_parsing *transport_parsing,
-      grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last);
+  grpc_error *(*parser)(grpc_exec_ctx *exec_ctx, void *parser_user_data,
+                        grpc_chttp2_transport_parsing *transport_parsing,
+                        grpc_chttp2_stream_parsing *stream_parsing,
+                        gpr_slice slice, int is_last);
 
   /* received settings */
   uint32_t settings[GRPC_CHTTP2_NUM_SETTINGS];
@@ -422,23 +423,25 @@ typedef struct {
   /** number of streams that are currently being read */
   gpr_refcount active_streams;
 
-  /** when the application requests writes be closed, the write_closed is
-      'queued'; when the close is flow controlled into the send path, we are
-      'sending' it; when the write has been performed it is 'sent' */
+  /** Is this stream closed for writing. */
   bool write_closed;
-  /** is this stream reading half-closed (boolean) */
+  /** Is this stream reading half-closed. */
   bool read_closed;
-  /** are all published incoming byte streams closed */
+  /** Are all published incoming byte streams closed. */
   bool all_incoming_byte_streams_finished;
-  /** is this stream in the stream map? (boolean) */
+  /** Is this stream in the stream map. */
   bool in_stream_map;
-  /** has this stream seen an error? if 1, then pending incoming frames
-      can be thrown away */
+  /** Has this stream seen an error.
+      If true, then pending incoming frames can be thrown away. */
   bool seen_error;
+  bool exceeded_metadata_size;
+
+  /** the error that resulted in this stream being removed */
+  grpc_error *removal_error;
 
   bool published_initial_metadata;
   bool published_trailing_metadata;
-  bool faked_trailing_metadata;
+  bool final_metadata_requested;
 
   grpc_chttp2_incoming_metadata_buffer received_initial_metadata;
   grpc_chttp2_incoming_metadata_buffer received_trailing_metadata;
@@ -470,24 +473,23 @@ typedef struct {
 } grpc_chttp2_stream_writing;
 
 struct grpc_chttp2_stream_parsing {
+  /** saw some stream level error */
+  grpc_error *forced_close_error;
   /** HTTP2 stream id for this stream, or zero if one has not been assigned */
   uint32_t id;
   /** has this stream received a close */
   uint8_t received_close;
-  /** saw a rst_stream */
-  uint8_t saw_rst_stream;
   /** how many header frames have we received? */
   uint8_t header_frames_received;
   /** which metadata did we get (on this parse) */
   uint8_t got_metadata_on_parse[2];
   /** should we raise the seen_error flag in transport_global */
-  uint8_t seen_error;
+  bool seen_error;
+  bool exceeded_metadata_size;
   /** window available for peer to send to us */
   int64_t incoming_window;
   /** parsing state for data frames */
   grpc_chttp2_data_parser data_parser;
-  /** reason give to rst_stream */
-  uint32_t rst_stream_reason;
   /** amount of window given */
   int64_t outgoing_window;
   /** number of bytes received - reset at end of parse thread execution */
@@ -524,13 +526,12 @@ struct grpc_chttp2_stream {
     are required, and schedule them if so */
 int grpc_chttp2_unlocking_check_writes(grpc_exec_ctx *exec_ctx,
                                        grpc_chttp2_transport_global *global,
-                                       grpc_chttp2_transport_writing *writing,
-                                       int is_parsing);
+                                       grpc_chttp2_transport_writing *writing);
 void grpc_chttp2_perform_writes(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing,
     grpc_endpoint *endpoint);
 void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx,
-                                   void *transport_writing, bool success);
+                                   void *transport_writing, grpc_error *error);
 void grpc_chttp2_cleanup_writing(grpc_exec_ctx *exec_ctx,
                                  grpc_chttp2_transport_global *global,
                                  grpc_chttp2_transport_writing *writing);
@@ -539,9 +540,9 @@ void grpc_chttp2_prepare_to_read(grpc_chttp2_transport_global *global,
                                  grpc_chttp2_transport_parsing *parsing);
 /** Process one slice of incoming data; return 1 if the connection is still
     viable after reading, or 0 if the connection should be torn down */
-int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
-                             grpc_chttp2_transport_parsing *transport_parsing,
-                             gpr_slice slice);
+grpc_error *grpc_chttp2_perform_read(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
+    gpr_slice slice);
 void grpc_chttp2_publish_reads(grpc_exec_ctx *exec_ctx,
                                grpc_chttp2_transport_global *global,
                                grpc_chttp2_transport_parsing *parsing);
@@ -671,9 +672,10 @@ void grpc_chttp2_for_all_streams(
 void grpc_chttp2_parsing_become_skip_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
 
-void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
-                                       grpc_chttp2_stream_global *stream_global,
-                                       grpc_closure **pclosure, int success);
+void grpc_chttp2_complete_closure_step(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
+    grpc_chttp2_stream_global *stream_global, grpc_closure **pclosure,
+    grpc_error *error);
 
 void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
                                       grpc_chttp2_transport *transport,
@@ -776,8 +778,8 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
                              grpc_status_code status, gpr_slice *details);
 void grpc_chttp2_mark_stream_closed(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_stream_global *stream_global, int close_reads,
-    int close_writes);
+    grpc_chttp2_stream_global *stream_global, int close_reads, int close_writes,
+    grpc_error *error);
 void grpc_chttp2_start_writing(grpc_exec_ctx *exec_ctx,
                                grpc_chttp2_transport_global *transport_global);
 
@@ -809,8 +811,8 @@ void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx,
                                            grpc_chttp2_incoming_byte_stream *bs,
                                            gpr_slice slice);
 void grpc_chttp2_incoming_byte_stream_finished(
-    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, int success,
-    int from_parsing_thread);
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs,
+    grpc_error *error, int from_parsing_thread);
 
 void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx,
                           grpc_chttp2_transport_parsing *parsing,
diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c
index 2995066e51..991d7729af 100644
--- a/src/core/ext/transport/chttp2/transport/parsing.c
+++ b/src/core/ext/transport/chttp2/transport/parsing.c
@@ -45,30 +45,34 @@
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-static int init_frame_parser(grpc_exec_ctx *exec_ctx,
-                             grpc_chttp2_transport_parsing *transport_parsing);
-static int init_header_frame_parser(
+#define TRANSPORT_FROM_PARSING(tp)                                        \
+  ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \
+                                                   parsing)))
+
+static grpc_error *init_frame_parser(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
+static grpc_error *init_header_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
     int is_continuation);
-static int init_data_frame_parser(
+static grpc_error *init_data_frame_parser(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
+static grpc_error *init_rst_stream_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
-static int init_rst_stream_parser(
+static grpc_error *init_settings_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
-static int init_settings_frame_parser(
+static grpc_error *init_window_update_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
-static int init_window_update_frame_parser(
+static grpc_error *init_ping_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
-static int init_ping_parser(grpc_exec_ctx *exec_ctx,
-                            grpc_chttp2_transport_parsing *transport_parsing);
-static int init_goaway_parser(grpc_exec_ctx *exec_ctx,
-                              grpc_chttp2_transport_parsing *transport_parsing);
-static int init_skip_frame_parser(
+static grpc_error *init_goaway_parser(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing);
+static grpc_error *init_skip_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
     int is_header);
 
-static int parse_frame_slice(grpc_exec_ctx *exec_ctx,
-                             grpc_chttp2_transport_parsing *transport_parsing,
-                             gpr_slice slice, int is_last);
+static grpc_error *parse_frame_slice(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
+    gpr_slice slice, int is_last);
 
 void grpc_chttp2_prepare_to_read(
     grpc_chttp2_transport_global *transport_global,
@@ -170,7 +174,9 @@ void grpc_chttp2_publish_reads(
   while (grpc_chttp2_list_pop_parsing_seen_stream(
       transport_global, transport_parsing, &stream_global, &stream_parsing)) {
     if (stream_parsing->seen_error) {
-      stream_global->seen_error = 1;
+      stream_global->seen_error = true;
+      stream_global->exceeded_metadata_size =
+          stream_parsing->exceeded_metadata_size;
       grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
     }
 
@@ -224,38 +230,42 @@ void grpc_chttp2_publish_reads(
       grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
     }
 
-    if (stream_parsing->saw_rst_stream) {
-      if (stream_parsing->rst_stream_reason != GRPC_CHTTP2_NO_ERROR) {
-        grpc_status_code status_code = grpc_chttp2_http2_error_to_grpc_status(
-            (grpc_chttp2_error_code)stream_parsing->rst_stream_reason);
-        char *status_details;
-        gpr_slice slice_details;
-        gpr_asprintf(&status_details, "Received RST_STREAM err=%d",
-                     stream_parsing->rst_stream_reason);
-        slice_details = gpr_slice_from_copied_string(status_details);
-        gpr_free(status_details);
+    if (stream_parsing->forced_close_error != GRPC_ERROR_NONE) {
+      intptr_t reason;
+      bool has_reason = grpc_error_get_int(stream_parsing->forced_close_error,
+                                           GRPC_ERROR_INT_HTTP2_ERROR, &reason);
+      if (has_reason && reason != GRPC_CHTTP2_NO_ERROR) {
+        grpc_status_code status_code =
+            has_reason ? grpc_chttp2_http2_error_to_grpc_status(
+                             (grpc_chttp2_error_code)reason)
+                       : GRPC_STATUS_INTERNAL;
+        const char *status_details =
+            grpc_error_string(stream_parsing->forced_close_error);
+        gpr_slice slice_details = gpr_slice_from_copied_string(status_details);
+        grpc_error_free_string(status_details);
         grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global,
                                 status_code, &slice_details);
       }
       grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
-                                     1, 1);
+                                     1, 1, stream_parsing->forced_close_error);
     }
 
     if (stream_parsing->received_close) {
       grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
-                                     1, 0);
+                                     1, 0, GRPC_ERROR_NONE);
     }
   }
 }
 
-int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
-                             grpc_chttp2_transport_parsing *transport_parsing,
-                             gpr_slice slice) {
+grpc_error *grpc_chttp2_perform_read(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
+    gpr_slice slice) {
   uint8_t *beg = GPR_SLICE_START_PTR(slice);
   uint8_t *end = GPR_SLICE_END_PTR(slice);
   uint8_t *cur = beg;
+  grpc_error *err;
 
-  if (cur == end) return 1;
+  if (cur == end) return GRPC_ERROR_NONE;
 
   switch (transport_parsing->deframe_state) {
     case GRPC_DTS_CLIENT_PREFIX_0:
@@ -285,21 +295,25 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       while (cur != end && transport_parsing->deframe_state != GRPC_DTS_FH_0) {
         if (*cur != GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing
                                                           ->deframe_state]) {
-          gpr_log(GPR_INFO,
-                  "Connect string mismatch: expected '%c' (%d) got '%c' (%d) "
-                  "at byte %d",
-                  GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing
-                                                        ->deframe_state],
-                  (int)(uint8_t)GRPC_CHTTP2_CLIENT_CONNECT_STRING
-                      [transport_parsing->deframe_state],
-                  *cur, (int)*cur, transport_parsing->deframe_state);
-          return 0;
+          char *msg;
+          gpr_asprintf(
+              &msg,
+              "Connect string mismatch: expected '%c' (%d) got '%c' (%d) "
+              "at byte %d",
+              GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing
+                                                    ->deframe_state],
+              (int)(uint8_t)GRPC_CHTTP2_CLIENT_CONNECT_STRING
+                  [transport_parsing->deframe_state],
+              *cur, (int)*cur, transport_parsing->deframe_state);
+          err = GRPC_ERROR_CREATE(msg);
+          gpr_free(msg);
+          return err;
         }
         ++cur;
         ++transport_parsing->deframe_state;
       }
       if (cur == end) {
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     dts_fh_0:
@@ -308,7 +322,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_frame_size = ((uint32_t)*cur) << 16;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_1;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_1:
@@ -316,7 +330,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_frame_size |= ((uint32_t)*cur) << 8;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_2;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_2:
@@ -324,7 +338,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_frame_size |= *cur;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_3;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_3:
@@ -332,7 +346,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_frame_type = *cur;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_4;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_4:
@@ -340,7 +354,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_frame_flags = *cur;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_5;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_5:
@@ -348,7 +362,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_stream_id = (((uint32_t)*cur) & 0x7f) << 24;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_6;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_6:
@@ -356,7 +370,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_stream_id |= ((uint32_t)*cur) << 16;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_7;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_7:
@@ -364,15 +378,16 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
       transport_parsing->incoming_stream_id |= ((uint32_t)*cur) << 8;
       if (++cur == end) {
         transport_parsing->deframe_state = GRPC_DTS_FH_8;
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FH_8:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_stream_id |= ((uint32_t)*cur);
       transport_parsing->deframe_state = GRPC_DTS_FRAME;
-      if (!init_frame_parser(exec_ctx, transport_parsing)) {
-        return 0;
+      err = init_frame_parser(exec_ctx, transport_parsing);
+      if (err != GRPC_ERROR_NONE) {
+        return err;
       }
       if (transport_parsing->incoming_stream_id != 0 &&
           transport_parsing->incoming_stream_id >
@@ -381,62 +396,69 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
             transport_parsing->incoming_stream_id;
       }
       if (transport_parsing->incoming_frame_size == 0) {
-        if (!parse_frame_slice(exec_ctx, transport_parsing, gpr_empty_slice(),
-                               1)) {
-          return 0;
+        err = parse_frame_slice(exec_ctx, transport_parsing, gpr_empty_slice(),
+                                1);
+        if (err != GRPC_ERROR_NONE) {
+          return err;
         }
         transport_parsing->incoming_stream = NULL;
         if (++cur == end) {
           transport_parsing->deframe_state = GRPC_DTS_FH_0;
-          return 1;
+          return GRPC_ERROR_NONE;
         }
         goto dts_fh_0; /* loop */
       } else if (transport_parsing->incoming_frame_size >
                  transport_parsing->max_frame_size) {
-        gpr_log(GPR_DEBUG, "Frame size %d is larger than max frame size %d",
-                transport_parsing->incoming_frame_size,
-                transport_parsing->max_frame_size);
-        return 0;
+        char *msg;
+        gpr_asprintf(&msg, "Frame size %d is larger than max frame size %d",
+                     transport_parsing->incoming_frame_size,
+                     transport_parsing->max_frame_size);
+        err = GRPC_ERROR_CREATE(msg);
+        gpr_free(msg);
+        return err;
       }
       if (++cur == end) {
-        return 1;
+        return GRPC_ERROR_NONE;
       }
     /* fallthrough */
     case GRPC_DTS_FRAME:
       GPR_ASSERT(cur < end);
       if ((uint32_t)(end - cur) == transport_parsing->incoming_frame_size) {
-        if (!parse_frame_slice(exec_ctx, transport_parsing,
-                               gpr_slice_sub_no_ref(slice, (size_t)(cur - beg),
-                                                    (size_t)(end - beg)),
-                               1)) {
-          return 0;
+        err = parse_frame_slice(exec_ctx, transport_parsing,
+                                gpr_slice_sub_no_ref(slice, (size_t)(cur - beg),
+                                                     (size_t)(end - beg)),
+                                1);
+        if (err != GRPC_ERROR_NONE) {
+          return err;
         }
         transport_parsing->deframe_state = GRPC_DTS_FH_0;
         transport_parsing->incoming_stream = NULL;
-        return 1;
+        return GRPC_ERROR_NONE;
       } else if ((uint32_t)(end - cur) >
                  transport_parsing->incoming_frame_size) {
         size_t cur_offset = (size_t)(cur - beg);
-        if (!parse_frame_slice(
-                exec_ctx, transport_parsing,
-                gpr_slice_sub_no_ref(
-                    slice, cur_offset,
-                    cur_offset + transport_parsing->incoming_frame_size),
-                1)) {
-          return 0;
+        err = parse_frame_slice(
+            exec_ctx, transport_parsing,
+            gpr_slice_sub_no_ref(
+                slice, cur_offset,
+                cur_offset + transport_parsing->incoming_frame_size),
+            1);
+        if (err != GRPC_ERROR_NONE) {
+          return err;
         }
         cur += transport_parsing->incoming_frame_size;
         transport_parsing->incoming_stream = NULL;
         goto dts_fh_0; /* loop */
       } else {
-        if (!parse_frame_slice(exec_ctx, transport_parsing,
-                               gpr_slice_sub_no_ref(slice, (size_t)(cur - beg),
-                                                    (size_t)(end - beg)),
-                               0)) {
-          return 0;
+        err = parse_frame_slice(exec_ctx, transport_parsing,
+                                gpr_slice_sub_no_ref(slice, (size_t)(cur - beg),
+                                                     (size_t)(end - beg)),
+                                0);
+        if (err != GRPC_ERROR_NONE) {
+          return err;
         }
         transport_parsing->incoming_frame_size -= (uint32_t)(end - cur);
-        return 1;
+        return GRPC_ERROR_NONE;
       }
       GPR_UNREACHABLE_CODE(return 0);
   }
@@ -444,23 +466,41 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx,
   GPR_UNREACHABLE_CODE(return 0);
 }
 
-static int init_frame_parser(grpc_exec_ctx *exec_ctx,
-                             grpc_chttp2_transport_parsing *transport_parsing) {
+static grpc_error *init_frame_parser(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
+  if (transport_parsing->is_first_frame &&
+      transport_parsing->incoming_frame_type != GRPC_CHTTP2_FRAME_SETTINGS) {
+    char *msg;
+    gpr_asprintf(
+        &msg, "Expected SETTINGS frame as the first frame, got frame type %d",
+        transport_parsing->incoming_frame_type);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
+  }
+  transport_parsing->is_first_frame = false;
   if (transport_parsing->expect_continuation_stream_id != 0) {
     if (transport_parsing->incoming_frame_type !=
         GRPC_CHTTP2_FRAME_CONTINUATION) {
-      gpr_log(GPR_ERROR, "Expected CONTINUATION frame, got frame type %02x",
-              transport_parsing->incoming_frame_type);
-      return 0;
+      char *msg;
+      gpr_asprintf(&msg, "Expected CONTINUATION frame, got frame type %02x",
+                   transport_parsing->incoming_frame_type);
+      grpc_error *err = GRPC_ERROR_CREATE(msg);
+      gpr_free(msg);
+      return err;
     }
     if (transport_parsing->expect_continuation_stream_id !=
         transport_parsing->incoming_stream_id) {
-      gpr_log(GPR_ERROR,
-              "Expected CONTINUATION frame for grpc_chttp2_stream %08x, got "
-              "grpc_chttp2_stream %08x",
-              transport_parsing->expect_continuation_stream_id,
-              transport_parsing->incoming_stream_id);
-      return 0;
+      char *msg;
+      gpr_asprintf(
+          &msg,
+          "Expected CONTINUATION frame for grpc_chttp2_stream %08x, got "
+          "grpc_chttp2_stream %08x",
+          transport_parsing->expect_continuation_stream_id,
+          transport_parsing->incoming_stream_id);
+      grpc_error *err = GRPC_ERROR_CREATE(msg);
+      gpr_free(msg);
+      return err;
     }
     return init_header_frame_parser(exec_ctx, transport_parsing, 1);
   }
@@ -470,8 +510,7 @@ static int init_frame_parser(grpc_exec_ctx *exec_ctx,
     case GRPC_CHTTP2_FRAME_HEADER:
       return init_header_frame_parser(exec_ctx, transport_parsing, 0);
     case GRPC_CHTTP2_FRAME_CONTINUATION:
-      gpr_log(GPR_ERROR, "Unexpected CONTINUATION frame");
-      return 0;
+      return GRPC_ERROR_CREATE("Unexpected CONTINUATION frame");
     case GRPC_CHTTP2_FRAME_RST_STREAM:
       return init_rst_stream_parser(exec_ctx, transport_parsing);
     case GRPC_CHTTP2_FRAME_SETTINGS:
@@ -483,22 +522,24 @@ static int init_frame_parser(grpc_exec_ctx *exec_ctx,
     case GRPC_CHTTP2_FRAME_GOAWAY:
       return init_goaway_parser(exec_ctx, transport_parsing);
     default:
-      gpr_log(GPR_ERROR, "Unknown frame type %02x",
-              transport_parsing->incoming_frame_type);
+      if (grpc_http_trace) {
+        gpr_log(GPR_ERROR, "Unknown frame type %02x",
+                transport_parsing->incoming_frame_type);
+      }
       return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
   }
 }
 
-static grpc_chttp2_parse_error skip_parser(
-    grpc_exec_ctx *exec_ctx, void *parser,
-    grpc_chttp2_transport_parsing *transport_parsing,
-    grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) {
-  return GRPC_CHTTP2_PARSE_OK;
+static grpc_error *skip_parser(grpc_exec_ctx *exec_ctx, void *parser,
+                               grpc_chttp2_transport_parsing *transport_parsing,
+                               grpc_chttp2_stream_parsing *stream_parsing,
+                               gpr_slice slice, int is_last) {
+  return GRPC_ERROR_NONE;
 }
 
 static void skip_header(void *tp, grpc_mdelem *md) { GRPC_MDELEM_UNREF(md); }
 
-static int init_skip_frame_parser(
+static grpc_error *init_skip_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
     int is_header) {
   if (is_header) {
@@ -513,7 +554,7 @@ static int init_skip_frame_parser(
   } else {
     transport_parsing->parser = skip_parser;
   }
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 void grpc_chttp2_parsing_become_skip_parser(
@@ -523,22 +564,28 @@ void grpc_chttp2_parsing_become_skip_parser(
       transport_parsing->parser == grpc_chttp2_header_parser_parse);
 }
 
-static grpc_chttp2_parse_error update_incoming_window(
+static grpc_error *update_incoming_window(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
     grpc_chttp2_stream_parsing *stream_parsing) {
   uint32_t incoming_frame_size = transport_parsing->incoming_frame_size;
   if (incoming_frame_size > transport_parsing->incoming_window) {
-    gpr_log(GPR_ERROR, "frame of size %d overflows incoming window of %d",
-            transport_parsing->incoming_frame_size,
-            transport_parsing->incoming_window);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64,
+                 transport_parsing->incoming_frame_size,
+                 transport_parsing->incoming_window);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
 
   if (incoming_frame_size > stream_parsing->incoming_window) {
-    gpr_log(GPR_ERROR, "frame of size %d overflows incoming window of %d",
-            transport_parsing->incoming_frame_size,
-            stream_parsing->incoming_window);
-    return GRPC_CHTTP2_CONNECTION_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64,
+                 transport_parsing->incoming_frame_size,
+                 stream_parsing->incoming_window);
+    grpc_error *err = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return err;
   }
 
   GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", transport_parsing, incoming_window,
@@ -549,15 +596,15 @@ static grpc_chttp2_parse_error update_incoming_window(
 
   grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing);
 
-  return GRPC_CHTTP2_PARSE_OK;
+  return GRPC_ERROR_NONE;
 }
 
-static int init_data_frame_parser(
+static grpc_error *init_data_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
   grpc_chttp2_stream_parsing *stream_parsing =
       grpc_chttp2_parsing_lookup_stream(transport_parsing,
                                         transport_parsing->incoming_stream_id);
-  grpc_chttp2_parse_error err = GRPC_CHTTP2_PARSE_OK;
+  grpc_error *err = GRPC_ERROR_NONE;
   if (stream_parsing == NULL) {
     return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
   }
@@ -565,33 +612,32 @@ static int init_data_frame_parser(
   if (stream_parsing->received_close) {
     return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
   }
-  if (err == GRPC_CHTTP2_PARSE_OK) {
+  if (err == GRPC_ERROR_NONE) {
     err = update_incoming_window(exec_ctx, transport_parsing, stream_parsing);
   }
-  if (err == GRPC_CHTTP2_PARSE_OK) {
+  if (err == GRPC_ERROR_NONE) {
     err = grpc_chttp2_data_parser_begin_frame(
-        &stream_parsing->data_parser, transport_parsing->incoming_frame_flags);
-  }
-  switch (err) {
-    case GRPC_CHTTP2_PARSE_OK:
-      transport_parsing->incoming_stream = stream_parsing;
-      transport_parsing->parser = grpc_chttp2_data_parser_parse;
-      transport_parsing->parser_data = &stream_parsing->data_parser;
-      return 1;
-    case GRPC_CHTTP2_STREAM_ERROR:
-      stream_parsing->received_close = 1;
-      stream_parsing->saw_rst_stream = 1;
-      stream_parsing->rst_stream_reason = GRPC_CHTTP2_PROTOCOL_ERROR;
-      gpr_slice_buffer_add(
-          &transport_parsing->qbuf,
-          grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id,
-                                        GRPC_CHTTP2_PROTOCOL_ERROR,
-                                        &stream_parsing->stats.outgoing));
-      return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
-    case GRPC_CHTTP2_CONNECTION_ERROR:
-      return 0;
+        &stream_parsing->data_parser, transport_parsing->incoming_frame_flags,
+        stream_parsing->id);
+  }
+  if (err == GRPC_ERROR_NONE) {
+    transport_parsing->incoming_stream = stream_parsing;
+    transport_parsing->parser = grpc_chttp2_data_parser_parse;
+    transport_parsing->parser_data = &stream_parsing->data_parser;
+    return GRPC_ERROR_NONE;
+  } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) {
+    /* handle stream errors by closing the stream */
+    stream_parsing->received_close = 1;
+    stream_parsing->forced_close_error = err;
+    gpr_slice_buffer_add(
+        &transport_parsing->qbuf,
+        grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id,
+                                      GRPC_CHTTP2_PROTOCOL_ERROR,
+                                      &stream_parsing->stats.outgoing));
+    return init_skip_frame_parser(exec_ctx, transport_parsing, 0);
+  } else {
+    return err;
   }
-  GPR_UNREACHABLE_CODE(return 0);
 }
 
 static void free_timeout(void *p) { gpr_free(p); }
@@ -612,7 +658,7 @@ static void on_initial_header(void *tp, grpc_mdelem *md) {
 
   if (md->key == GRPC_MDSTR_GRPC_STATUS && md != GRPC_MDELEM_GRPC_STATUS_0) {
     /* TODO(ctiller): check for a status like " 0" */
-    stream_parsing->seen_error = 1;
+    stream_parsing->seen_error = true;
   }
 
   if (md->key == GRPC_MDSTR_GRPC_TIMEOUT) {
@@ -633,8 +679,27 @@ static void on_initial_header(void *tp, grpc_mdelem *md) {
         gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), *cached_timeout));
     GRPC_MDELEM_UNREF(md);
   } else {
-    grpc_chttp2_incoming_metadata_buffer_add(
-        &stream_parsing->metadata_buffer[0], md);
+    const size_t new_size =
+        stream_parsing->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md);
+    grpc_chttp2_transport_global *transport_global =
+        &TRANSPORT_FROM_PARSING(transport_parsing)->global;
+    const size_t metadata_size_limit =
+        transport_global->settings[GRPC_LOCAL_SETTINGS]
+                                  [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE];
+    if (new_size > metadata_size_limit) {
+      if (!stream_parsing->exceeded_metadata_size) {
+        gpr_log(GPR_DEBUG,
+                "received initial metadata size exceeds limit (%" PRIuPTR
+                " vs. %" PRIuPTR ")",
+                new_size, metadata_size_limit);
+        stream_parsing->seen_error = true;
+        stream_parsing->exceeded_metadata_size = true;
+      }
+      GRPC_MDELEM_UNREF(md);
+    } else {
+      grpc_chttp2_incoming_metadata_buffer_add(
+          &stream_parsing->metadata_buffer[0], md);
+    }
   }
 
   grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing);
@@ -658,18 +723,37 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) {
 
   if (md->key == GRPC_MDSTR_GRPC_STATUS && md != GRPC_MDELEM_GRPC_STATUS_0) {
     /* TODO(ctiller): check for a status like " 0" */
-    stream_parsing->seen_error = 1;
+    stream_parsing->seen_error = true;
+  }
+
+  const size_t new_size =
+      stream_parsing->metadata_buffer[1].size + GRPC_MDELEM_LENGTH(md);
+  grpc_chttp2_transport_global *transport_global =
+      &TRANSPORT_FROM_PARSING(transport_parsing)->global;
+  const size_t metadata_size_limit =
+      transport_global->settings[GRPC_LOCAL_SETTINGS]
+                                [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE];
+  if (new_size > metadata_size_limit) {
+    if (!stream_parsing->exceeded_metadata_size) {
+      gpr_log(GPR_DEBUG,
+              "received trailing metadata size exceeds limit (%" PRIuPTR
+              " vs. %" PRIuPTR ")",
+              new_size, metadata_size_limit);
+      stream_parsing->seen_error = true;
+      stream_parsing->exceeded_metadata_size = true;
+    }
+    GRPC_MDELEM_UNREF(md);
+  } else {
+    grpc_chttp2_incoming_metadata_buffer_add(
+        &stream_parsing->metadata_buffer[1], md);
   }
 
-  grpc_chttp2_incoming_metadata_buffer_add(&stream_parsing->metadata_buffer[1],
-                                           md);
-
   grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing);
 
   GPR_TIMER_END("on_trailing_header", 0);
 }
 
-static int init_header_frame_parser(
+static grpc_error *init_header_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
     int is_continuation) {
   uint8_t is_eoh = (transport_parsing->incoming_frame_flags &
@@ -764,15 +848,16 @@ static int init_header_frame_parser(
                            GRPC_CHTTP2_FLAG_HAS_PRIORITY)) {
     grpc_chttp2_hpack_parser_set_has_priority(&transport_parsing->hpack_parser);
   }
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
-static int init_window_update_frame_parser(
+static grpc_error *init_window_update_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
-  int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_window_update_parser_begin_frame(
-                                       &transport_parsing->simple.window_update,
-                                       transport_parsing->incoming_frame_size,
-                                       transport_parsing->incoming_frame_flags);
+  grpc_error *err = grpc_chttp2_window_update_parser_begin_frame(
+      &transport_parsing->simple.window_update,
+      transport_parsing->incoming_frame_size,
+      transport_parsing->incoming_frame_flags);
+  if (err != GRPC_ERROR_NONE) return err;
   if (transport_parsing->incoming_stream_id != 0) {
     grpc_chttp2_stream_parsing *stream_parsing =
         transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream(
@@ -784,26 +869,27 @@ static int init_window_update_frame_parser(
   }
   transport_parsing->parser = grpc_chttp2_window_update_parser_parse;
   transport_parsing->parser_data = &transport_parsing->simple.window_update;
-  return ok;
+  return GRPC_ERROR_NONE;
 }
 
-static int init_ping_parser(grpc_exec_ctx *exec_ctx,
-                            grpc_chttp2_transport_parsing *transport_parsing) {
-  int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_ping_parser_begin_frame(
-                                       &transport_parsing->simple.ping,
-                                       transport_parsing->incoming_frame_size,
-                                       transport_parsing->incoming_frame_flags);
+static grpc_error *init_ping_parser(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
+  grpc_error *err = grpc_chttp2_ping_parser_begin_frame(
+      &transport_parsing->simple.ping, transport_parsing->incoming_frame_size,
+      transport_parsing->incoming_frame_flags);
+  if (err != GRPC_ERROR_NONE) return err;
   transport_parsing->parser = grpc_chttp2_ping_parser_parse;
   transport_parsing->parser_data = &transport_parsing->simple.ping;
-  return ok;
+  return GRPC_ERROR_NONE;
 }
 
-static int init_rst_stream_parser(
+static grpc_error *init_rst_stream_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
-  int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_rst_stream_parser_begin_frame(
-                                       &transport_parsing->simple.rst_stream,
-                                       transport_parsing->incoming_frame_size,
-                                       transport_parsing->incoming_frame_flags);
+  grpc_error *err = grpc_chttp2_rst_stream_parser_begin_frame(
+      &transport_parsing->simple.rst_stream,
+      transport_parsing->incoming_frame_size,
+      transport_parsing->incoming_frame_flags);
+  if (err != GRPC_ERROR_NONE) return err;
   grpc_chttp2_stream_parsing *stream_parsing =
       transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream(
           transport_parsing, transport_parsing->incoming_stream_id);
@@ -813,37 +899,32 @@ static int init_rst_stream_parser(
   stream_parsing->stats.incoming.framing_bytes += 9;
   transport_parsing->parser = grpc_chttp2_rst_stream_parser_parse;
   transport_parsing->parser_data = &transport_parsing->simple.rst_stream;
-  return ok;
+  return GRPC_ERROR_NONE;
 }
 
-static int init_goaway_parser(
+static grpc_error *init_goaway_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
-  int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_goaway_parser_begin_frame(
-                                       &transport_parsing->goaway_parser,
-                                       transport_parsing->incoming_frame_size,
-                                       transport_parsing->incoming_frame_flags);
+  grpc_error *err = grpc_chttp2_goaway_parser_begin_frame(
+      &transport_parsing->goaway_parser, transport_parsing->incoming_frame_size,
+      transport_parsing->incoming_frame_flags);
+  if (err != GRPC_ERROR_NONE) return err;
   transport_parsing->parser = grpc_chttp2_goaway_parser_parse;
   transport_parsing->parser_data = &transport_parsing->goaway_parser;
-  return ok;
+  return GRPC_ERROR_NONE;
 }
 
-static int init_settings_frame_parser(
+static grpc_error *init_settings_frame_parser(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) {
-  int ok;
-
   if (transport_parsing->incoming_stream_id != 0) {
-    gpr_log(GPR_ERROR, "settings frame received for grpc_chttp2_stream %d",
-            transport_parsing->incoming_stream_id);
-    return 0;
+    return GRPC_ERROR_CREATE("Settings frame received for grpc_chttp2_stream");
   }
 
-  ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_settings_parser_begin_frame(
-                                   &transport_parsing->simple.settings,
-                                   transport_parsing->incoming_frame_size,
-                                   transport_parsing->incoming_frame_flags,
-                                   transport_parsing->settings);
-  if (!ok) {
-    return 0;
+  grpc_error *err = grpc_chttp2_settings_parser_begin_frame(
+      &transport_parsing->simple.settings,
+      transport_parsing->incoming_frame_size,
+      transport_parsing->incoming_frame_flags, transport_parsing->settings);
+  if (err != GRPC_ERROR_NONE) {
+    return err;
   }
   if (transport_parsing->incoming_frame_flags & GRPC_CHTTP2_FLAG_ACK) {
     transport_parsing->settings_ack_received = 1;
@@ -857,7 +938,7 @@ static int init_settings_frame_parser(
   }
   transport_parsing->parser = grpc_chttp2_settings_parser_parse;
   transport_parsing->parser_data = &transport_parsing->simple.settings;
-  return ok;
+  return GRPC_ERROR_NONE;
 }
 
 /*
@@ -866,34 +947,37 @@ static int is_window_update_legal(int64_t window_update, int64_t window) {
 }
 */
 
-static int parse_frame_slice(grpc_exec_ctx *exec_ctx,
-                             grpc_chttp2_transport_parsing *transport_parsing,
-                             gpr_slice slice, int is_last) {
+static grpc_error *parse_frame_slice(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing,
+    gpr_slice slice, int is_last) {
   grpc_chttp2_stream_parsing *stream_parsing =
       transport_parsing->incoming_stream;
-  switch (transport_parsing->parser(exec_ctx, transport_parsing->parser_data,
-                                    transport_parsing, stream_parsing, slice,
-                                    is_last)) {
-    case GRPC_CHTTP2_PARSE_OK:
-      if (stream_parsing) {
-        grpc_chttp2_list_add_parsing_seen_stream(transport_parsing,
-                                                 stream_parsing);
-      }
-      return 1;
-    case GRPC_CHTTP2_STREAM_ERROR:
-      grpc_chttp2_parsing_become_skip_parser(exec_ctx, transport_parsing);
-      if (stream_parsing) {
-        stream_parsing->saw_rst_stream = 1;
-        stream_parsing->rst_stream_reason = GRPC_CHTTP2_PROTOCOL_ERROR;
-        gpr_slice_buffer_add(
-            &transport_parsing->qbuf,
-            grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id,
-                                          GRPC_CHTTP2_PROTOCOL_ERROR,
-                                          &stream_parsing->stats.outgoing));
-      }
-      return 1;
-    case GRPC_CHTTP2_CONNECTION_ERROR:
-      return 0;
+  grpc_error *err = transport_parsing->parser(
+      exec_ctx, transport_parsing->parser_data, transport_parsing,
+      stream_parsing, slice, is_last);
+  if (err == GRPC_ERROR_NONE) {
+    if (stream_parsing) {
+      grpc_chttp2_list_add_parsing_seen_stream(transport_parsing,
+                                               stream_parsing);
+    }
+    return GRPC_ERROR_NONE;
+  } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) {
+    if (grpc_http_trace) {
+      const char *msg = grpc_error_string(err);
+      gpr_log(GPR_ERROR, "%s", msg);
+      grpc_error_free_string(msg);
+    }
+    grpc_chttp2_parsing_become_skip_parser(exec_ctx, transport_parsing);
+    if (stream_parsing) {
+      stream_parsing->forced_close_error = err;
+      gpr_slice_buffer_add(
+          &transport_parsing->qbuf,
+          grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id,
+                                        GRPC_CHTTP2_PROTOCOL_ERROR,
+                                        &stream_parsing->stats.outgoing));
+    } else {
+      GRPC_ERROR_UNREF(err);
+    }
   }
-  GPR_UNREACHABLE_CODE(return 0);
+  return err;
 }
diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c
index a8fb463939..b19f5f068d 100644
--- a/src/core/ext/transport/chttp2/transport/writing.c
+++ b/src/core/ext/transport/chttp2/transport/writing.c
@@ -45,7 +45,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
 
 int grpc_chttp2_unlocking_check_writes(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
-    grpc_chttp2_transport_writing *transport_writing, int is_parsing) {
+    grpc_chttp2_transport_writing *transport_writing) {
   grpc_chttp2_stream_global *stream_global;
   grpc_chttp2_stream_writing *stream_writing;
 
@@ -61,7 +61,7 @@ int grpc_chttp2_unlocking_check_writes(
                                 [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]);
 
   if (transport_global->dirtied_local_settings &&
-      !transport_global->sent_local_settings && !is_parsing) {
+      !transport_global->sent_local_settings) {
     gpr_slice_buffer_add(
         &transport_writing->outbuf,
         grpc_chttp2_settings_create(
@@ -187,7 +187,8 @@ void grpc_chttp2_perform_writes(
     grpc_endpoint_write(exec_ctx, endpoint, &transport_writing->outbuf,
                         &transport_writing->done_cb);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, &transport_writing->done_cb, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &transport_writing->done_cb, GRPC_ERROR_NONE,
+                        NULL);
   }
 }
 
@@ -334,25 +335,27 @@ void grpc_chttp2_cleanup_writing(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
     if (stream_writing->sent_initial_metadata) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, stream_global,
-          &stream_global->send_initial_metadata_finished, 1);
+          exec_ctx, transport_global, stream_global,
+          &stream_global->send_initial_metadata_finished, GRPC_ERROR_NONE);
     }
     grpc_transport_move_one_way_stats(&stream_writing->stats,
                                       &stream_global->stats.outgoing);
     if (stream_writing->sent_message) {
       GPR_ASSERT(stream_writing->send_message == NULL);
       grpc_chttp2_complete_closure_step(
-          exec_ctx, stream_global, &stream_global->send_message_finished, 1);
+          exec_ctx, transport_global, stream_global,
+          &stream_global->send_message_finished, GRPC_ERROR_NONE);
       stream_writing->sent_message = 0;
     }
     if (stream_writing->sent_trailing_metadata) {
       grpc_chttp2_complete_closure_step(
-          exec_ctx, stream_global,
-          &stream_global->send_trailing_metadata_finished, 1);
+          exec_ctx, transport_global, stream_global,
+          &stream_global->send_trailing_metadata_finished, GRPC_ERROR_NONE);
     }
     if (stream_writing->sent_trailing_metadata) {
       grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
-                                     !transport_global->is_client, 1);
+                                     !transport_global->is_client, 1,
+                                     GRPC_ERROR_NONE);
     }
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing");
   }
diff --git a/src/ruby/ext/grpc/rb_signal.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c
index a9e512374b..df1acddcc0 100644
--- a/src/ruby/ext/grpc/rb_signal.c
+++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c
@@ -31,40 +31,39 @@
  *
  */
 
-#include <ruby/ruby.h>
-#include <signal.h>
-#include <stdbool.h>
+#include <grpc/impl/codegen/port_platform.h>
 
-#include <grpc/support/log.h>
+#include <stdio.h>
+#include <string.h>
 
-#include "rb_grpc.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 
-static void (*old_sigint_handler)(int);
-static void (*old_sigterm_handler)(int);
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/transport_impl.h"
 
-static volatile bool signal_received = false;
+// Cronet transport object
+typedef struct cronet_transport {
+  grpc_transport base;  // must be first element in this structure
+  void *engine;
+  char *host;
+} cronet_transport;
 
-/* This has to be handled at the C level instead of Ruby, because Ruby signal
- * handlers are constrained to run in the main interpreter thread. If that main
- * thread is blocked on grpc_completion_queue_pluck, the signal handlers will
- * never run */
-static void handle_signal(int signum) {
-  signal_received = true;
-  if (signum == SIGINT) {
-    old_sigint_handler(signum);
-  } else if (signum == SIGTERM) {
-    old_sigterm_handler(signum);
-  }
-}
+extern grpc_transport_vtable grpc_cronet_vtable;
 
-static VALUE grpc_rb_signal_received(VALUE self) {
-  (void)self;
-  return signal_received ? Qtrue : Qfalse;
-}
+GRPCAPI grpc_channel *grpc_cronet_secure_channel_create(
+    void *engine, const char *target, const grpc_channel_args *args,
+    void *reserved) {
+  cronet_transport *ct = gpr_malloc(sizeof(cronet_transport));
+  ct->base.vtable = &grpc_cronet_vtable;
+  ct->engine = engine;
+  ct->host = gpr_malloc(strlen(target) + 1);
+  strcpy(ct->host, target);
+  gpr_log(GPR_DEBUG,
+          "grpc_create_cronet_transport: cronet_engine = %p, target=%s", engine,
+          ct->host);
 
-void Init_grpc_signals() {
-  old_sigint_handler = signal(SIGINT, handle_signal);
-  old_sigterm_handler = signal(SIGTERM, handle_signal);
-  rb_define_singleton_method(grpc_rb_mGrpcCore, "signal_received?",
-                             grpc_rb_signal_received, 0);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  return grpc_channel_create(&exec_ctx, target, args,
+                             GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct);
 }
diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c
new file mode 100644
index 0000000000..687026c9fd
--- /dev/null
+++ b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* This file has empty implementation of all the functions exposed by the cronet
+library, so we can build it in all environments */
+
+#include <stdbool.h>
+
+#include <grpc/support/log.h>
+
+#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h"
+
+#ifdef GRPC_COMPILE_WITH_CRONET
+/* link with the real CRONET library in the build system */
+#else
+/* Dummy implementation of cronet API just to test for build-ability */
+cronet_bidirectional_stream* cronet_bidirectional_stream_create(
+    cronet_engine* engine, void* annotation,
+    cronet_bidirectional_stream_callback* callback) {
+  GPR_ASSERT(0);
+  return NULL;
+}
+
+int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream) {
+  GPR_ASSERT(0);
+  return 0;
+}
+
+int cronet_bidirectional_stream_start(
+    cronet_bidirectional_stream* stream, const char* url, int priority,
+    const char* method, const cronet_bidirectional_stream_header_array* headers,
+    bool end_of_stream) {
+  GPR_ASSERT(0);
+  return 0;
+}
+
+int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream,
+                                     char* buffer, int capacity) {
+  GPR_ASSERT(0);
+  return 0;
+}
+
+int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream,
+                                      const char* buffer, int count,
+                                      bool end_of_stream) {
+  GPR_ASSERT(0);
+  return 0;
+}
+
+int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) {
+  GPR_ASSERT(0);
+  return 0;
+}
+
+#endif /* GRPC_COMPILE_WITH_CRONET */
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c
new file mode 100644
index 0000000000..25d8aca250
--- /dev/null
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.c
@@ -0,0 +1,662 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <string.h>
+
+#include <grpc/impl/codegen/port_platform.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
+#include <grpc/support/slice_buffer.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/lib/transport/transport_impl.h"
+#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h"
+
+#define GRPC_HEADER_SIZE_IN_BYTES 5
+
+// Global flag that gets set with GRPC_TRACE env variable
+int grpc_cronet_trace = 1;
+
+// Cronet transport object
+struct grpc_cronet_transport {
+  grpc_transport base; /* must be first element in this structure */
+  cronet_engine *engine;
+  char *host;
+};
+
+typedef struct grpc_cronet_transport grpc_cronet_transport;
+
+enum send_state {
+  CRONET_SEND_IDLE = 0,
+  CRONET_REQ_STARTED,
+  CRONET_SEND_HEADER,
+  CRONET_WRITE,
+  CRONET_WRITE_COMPLETED,
+};
+
+enum recv_state {
+  CRONET_RECV_IDLE = 0,
+  CRONET_RECV_READ_LENGTH,
+  CRONET_RECV_READ_DATA,
+  CRONET_RECV_CLOSED,
+};
+
+static const char *recv_state_name[] = {
+    "CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,",
+    "CRONET_RECV_CLOSED"};
+
+// Enum that identifies calling function.
+enum e_caller {
+  PERFORM_STREAM_OP,
+  ON_READ_COMPLETE,
+  ON_RESPONSE_HEADERS_RECEIVED,
+  ON_RESPONSE_TRAILERS_RECEIVED
+};
+
+enum callback_id {
+  CB_SEND_INITIAL_METADATA = 0,
+  CB_SEND_MESSAGE,
+  CB_SEND_TRAILING_METADATA,
+  CB_RECV_MESSAGE,
+  CB_RECV_INITIAL_METADATA,
+  CB_RECV_TRAILING_METADATA,
+  CB_NUM_CALLBACKS
+};
+
+struct stream_obj {
+  // we store received bytes here as they trickle in.
+  gpr_slice_buffer write_slice_buffer;
+  cronet_bidirectional_stream *cbs;
+  gpr_slice slice;
+  gpr_slice_buffer read_slice_buffer;
+  struct grpc_slice_buffer_stream sbs;
+  char *read_buffer;
+  int remaining_read_bytes;
+  int total_read_bytes;
+
+  char *write_buffer;
+  size_t write_buffer_size;
+
+  // Hold the URL
+  char *url;
+
+  bool response_headers_received;
+  bool read_requested;
+  bool response_trailers_received;
+  bool read_closed;
+
+  // Recv message stuff
+  grpc_byte_buffer **recv_message;
+  // Initial metadata stuff
+  grpc_metadata_batch *recv_initial_metadata;
+  // Trailing metadata stuff
+  grpc_metadata_batch *recv_trailing_metadata;
+  grpc_chttp2_incoming_metadata_buffer imb;
+
+  // This mutex protects receive state machine execution
+  gpr_mu recv_mu;
+  // we can queue up up to 2 callbacks for each OP
+  grpc_closure *callback_list[CB_NUM_CALLBACKS][2];
+
+  // storage for header
+  cronet_bidirectional_stream_header *headers;
+  uint32_t num_headers;
+  cronet_bidirectional_stream_header_array header_array;
+  // state tracking
+  enum recv_state cronet_recv_state;
+  enum send_state cronet_send_state;
+};
+
+typedef struct stream_obj stream_obj;
+
+static void next_send_step(stream_obj *s);
+static void next_recv_step(stream_obj *s, enum e_caller caller);
+
+static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                                   grpc_stream *gs, grpc_pollset *pollset) {}
+
+static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx,
+                                       grpc_transport *gt, grpc_stream *gs,
+                                       grpc_pollset_set *pollset_set) {}
+
+static void enqueue_callbacks(grpc_closure *callback_list[]) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  if (callback_list[0]) {
+    grpc_exec_ctx_sched(&exec_ctx, callback_list[0], GRPC_ERROR_NONE, NULL);
+    callback_list[0] = NULL;
+  }
+  if (callback_list[1]) {
+    grpc_exec_ctx_sched(&exec_ctx, callback_list[1], GRPC_ERROR_NONE, NULL);
+    callback_list[1] = NULL;
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void on_canceled(cronet_bidirectional_stream *stream) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "on_canceled %p", stream);
+  }
+}
+
+static void on_failed(cronet_bidirectional_stream *stream, int net_error) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error);
+  }
+}
+
+static void on_succeeded(cronet_bidirectional_stream *stream) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "on_succeeded %p", stream);
+  }
+}
+
+static void on_response_trailers_received(
+    cronet_bidirectional_stream *stream,
+    const cronet_bidirectional_stream_header_array *trailers) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "R: on_response_trailers_received");
+  }
+  stream_obj *s = (stream_obj *)stream->annotation;
+
+  memset(&s->imb, 0, sizeof(s->imb));
+  grpc_chttp2_incoming_metadata_buffer_init(&s->imb);
+  unsigned int i = 0;
+  for (i = 0; i < trailers->count; i++) {
+    grpc_chttp2_incoming_metadata_buffer_add(
+        &s->imb, grpc_mdelem_from_metadata_strings(
+                     grpc_mdstr_from_string(trailers->headers[i].key),
+                     grpc_mdstr_from_string(trailers->headers[i].value)));
+  }
+  s->response_trailers_received = true;
+  next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED);
+}
+
+static void on_write_completed(cronet_bidirectional_stream *stream,
+                               const char *data) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "W: on_write_completed");
+  }
+  stream_obj *s = (stream_obj *)stream->annotation;
+  enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]);
+  s->cronet_send_state = CRONET_WRITE_COMPLETED;
+  next_send_step(s);
+}
+
+static void process_recv_message(stream_obj *s, const uint8_t *recv_data) {
+  gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes);
+  uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice);
+  if (s->total_read_bytes > 0) {
+    // Only copy if there is non-zero number of bytes
+    memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
+    gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  }
+  grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0);
+  *s->recv_message = (grpc_byte_buffer *)&s->sbs;
+}
+
+static int parse_grpc_header(const uint8_t *data) {
+  const uint8_t *p = data + 1;
+  int length = 0;
+  length |= ((uint8_t)*p++) << 24;
+  length |= ((uint8_t)*p++) << 16;
+  length |= ((uint8_t)*p++) << 8;
+  length |= ((uint8_t)*p++);
+  return length;
+}
+
+static void on_read_completed(cronet_bidirectional_stream *stream, char *data,
+                              int count) {
+  stream_obj *s = (stream_obj *)stream->annotation;
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d",
+            count, s->total_read_bytes, s->remaining_read_bytes);
+  }
+  if (count > 0) {
+    GPR_ASSERT(s->recv_message);
+    s->remaining_read_bytes -= count;
+    next_recv_step(s, ON_READ_COMPLETE);
+  } else {
+    s->read_closed = true;
+    next_recv_step(s, ON_READ_COMPLETE);
+  }
+}
+
+static void on_response_headers_received(
+    cronet_bidirectional_stream *stream,
+    const cronet_bidirectional_stream_header_array *headers,
+    const char *negotiated_protocol) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "R: on_response_headers_received");
+  }
+  stream_obj *s = (stream_obj *)stream->annotation;
+  enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]);
+  s->response_headers_received = true;
+  next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED);
+}
+
+static void on_request_headers_sent(cronet_bidirectional_stream *stream) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "W: on_request_headers_sent");
+  }
+  stream_obj *s = (stream_obj *)stream->annotation;
+  enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]);
+  s->cronet_send_state = CRONET_SEND_HEADER;
+  next_send_step(s);
+}
+
+// Callback function pointers (invoked by cronet in response to events)
+static cronet_bidirectional_stream_callback callbacks = {
+    on_request_headers_sent,
+    on_response_headers_received,
+    on_read_completed,
+    on_write_completed,
+    on_response_trailers_received,
+    on_succeeded,
+    on_failed,
+    on_canceled};
+
+static void invoke_closing_callback(stream_obj *s) {
+  grpc_chttp2_incoming_metadata_buffer_publish(&s->imb,
+                                               s->recv_trailing_metadata);
+  if (s->callback_list[CB_RECV_TRAILING_METADATA]) {
+    enqueue_callbacks(s->callback_list[CB_RECV_TRAILING_METADATA]);
+  }
+}
+
+static void set_recv_state(stream_obj *s, enum recv_state state) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "next_state = %s", recv_state_name[state]);
+  }
+  s->cronet_recv_state = state;
+}
+
+// This is invoked from perform_stream_op, and all on_xxxx callbacks.
+static void next_recv_step(stream_obj *s, enum e_caller caller) {
+  gpr_mu_lock(&s->recv_mu);
+  switch (s->cronet_recv_state) {
+    case CRONET_RECV_IDLE:
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE");
+      }
+      if (caller == PERFORM_STREAM_OP ||
+          caller == ON_RESPONSE_HEADERS_RECEIVED) {
+        if (s->read_closed && s->response_trailers_received) {
+          invoke_closing_callback(s);
+          set_recv_state(s, CRONET_RECV_CLOSED);
+        } else if (s->response_headers_received == true &&
+                   s->read_requested == true) {
+          set_recv_state(s, CRONET_RECV_READ_LENGTH);
+          s->total_read_bytes = s->remaining_read_bytes =
+              GRPC_HEADER_SIZE_IN_BYTES;
+          GPR_ASSERT(s->read_buffer);
+          if (grpc_cronet_trace) {
+            gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");
+          }
+          cronet_bidirectional_stream_read(s->cbs, s->read_buffer,
+                                           s->remaining_read_bytes);
+        }
+      }
+      break;
+    case CRONET_RECV_READ_LENGTH:
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_LENGTH");
+      }
+      if (caller == ON_READ_COMPLETE) {
+        if (s->read_closed) {
+          invoke_closing_callback(s);
+          enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]);
+          set_recv_state(s, CRONET_RECV_CLOSED);
+        } else {
+          GPR_ASSERT(s->remaining_read_bytes == 0);
+          set_recv_state(s, CRONET_RECV_READ_DATA);
+          s->total_read_bytes = s->remaining_read_bytes =
+              parse_grpc_header((const uint8_t *)s->read_buffer);
+          s->read_buffer =
+              gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes);
+          GPR_ASSERT(s->read_buffer);
+          if (grpc_cronet_trace) {
+            gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");
+          }
+          if (s->remaining_read_bytes > 0) {
+            cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
+                                             s->remaining_read_bytes);
+          } else {
+            // Calling the closing callback directly since this is a 0 byte read
+            // for an empty message.
+            process_recv_message(s, NULL);
+            enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]);
+            invoke_closing_callback(s);
+            set_recv_state(s, CRONET_RECV_CLOSED);
+          }
+        }
+      }
+      break;
+    case CRONET_RECV_READ_DATA:
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA");
+      }
+      if (caller == ON_READ_COMPLETE) {
+        if (s->remaining_read_bytes > 0) {
+          int offset = s->total_read_bytes - s->remaining_read_bytes;
+          GPR_ASSERT(s->read_buffer);
+          if (grpc_cronet_trace) {
+            gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");
+          }
+          cronet_bidirectional_stream_read(
+              s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes);
+        } else {
+          gpr_slice_buffer_init(&s->read_slice_buffer);
+          uint8_t *p = (uint8_t *)s->read_buffer;
+          process_recv_message(s, p);
+          set_recv_state(s, CRONET_RECV_IDLE);
+          enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]);
+        }
+      }
+      break;
+    case CRONET_RECV_CLOSED:
+      break;
+    default:
+      GPR_ASSERT(0);  // Should not reach here
+      break;
+  }
+  gpr_mu_unlock(&s->recv_mu);
+}
+
+// This function takes the data from s->write_slice_buffer and assembles into
+// a contiguous byte stream with 5 byte gRPC header prepended.
+static void create_grpc_frame(stream_obj *s) {
+  gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slice_buffer);
+  uint8_t *raw_data = GPR_SLICE_START_PTR(slice);
+  size_t length = GPR_SLICE_LENGTH(slice);
+  s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES;
+  s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size);
+  uint8_t *p = (uint8_t *)s->write_buffer;
+  // Append 5 byte header
+  *p++ = 0;
+  *p++ = (uint8_t)(length >> 24);
+  *p++ = (uint8_t)(length >> 16);
+  *p++ = (uint8_t)(length >> 8);
+  *p++ = (uint8_t)(length);
+  // append actual data
+  memcpy(p, raw_data, length);
+}
+
+static void do_write(stream_obj *s) {
+  gpr_slice_buffer *sb = &s->write_slice_buffer;
+  GPR_ASSERT(sb->count <= 1);
+  if (sb->count > 0) {
+    create_grpc_frame(s);
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write");
+    }
+    cronet_bidirectional_stream_write(s->cbs, s->write_buffer,
+                                      (int)s->write_buffer_size, false);
+  }
+}
+
+//
+static void next_send_step(stream_obj *s) {
+  switch (s->cronet_send_state) {
+    case CRONET_SEND_IDLE:
+      GPR_ASSERT(
+          s->cbs);  // cronet_bidirectional_stream is not initialized yet.
+      s->cronet_send_state = CRONET_REQ_STARTED;
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url);
+      }
+      cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST",
+                                        &s->header_array, false);
+      // we no longer need the memory that was allocated earlier.
+      gpr_free(s->header_array.headers);
+      break;
+    case CRONET_SEND_HEADER:
+      do_write(s);
+      s->cronet_send_state = CRONET_WRITE;
+      break;
+    case CRONET_WRITE_COMPLETED:
+      do_write(s);
+      break;
+    default:
+      GPR_ASSERT(0);
+      break;
+  }
+}
+
+static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head,
+                                               const char *host,
+                                               stream_obj *s) {
+  grpc_linked_mdelem *curr = head;
+  // Walk the linked list and get number of header fields
+  uint32_t num_headers_available = 0;
+  while (curr != NULL) {
+    curr = curr->next;
+    num_headers_available++;
+  }
+  // Allocate enough memory
+  s->headers = (cronet_bidirectional_stream_header *)gpr_malloc(
+      sizeof(cronet_bidirectional_stream_header) * num_headers_available);
+
+  // Walk the linked list again, this time copying the header fields.
+  // s->num_headers
+  // can be less than num_headers_available, as some headers are not used for
+  // cronet
+  curr = head;
+  s->num_headers = 0;
+  while (s->num_headers < num_headers_available) {
+    grpc_mdelem *mdelem = curr->md;
+    curr = curr->next;
+    const char *key = grpc_mdstr_as_c_string(mdelem->key);
+    const char *value = grpc_mdstr_as_c_string(mdelem->value);
+    if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 ||
+        strcmp(key, ":authority") == 0) {
+      // Cronet populates these fields on its own.
+      continue;
+    }
+    if (strcmp(key, ":path") == 0) {
+      // Create URL by appending :path value to the hostname
+      gpr_asprintf(&s->url, "https://%s%s", host, value);
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "extracted URL = %s", s->url);
+      }
+      continue;
+    }
+    s->headers[s->num_headers].key = key;
+    s->headers[s->num_headers].value = value;
+    s->num_headers++;
+    if (curr == NULL) {
+      break;
+    }
+  }
+}
+
+static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                              grpc_stream *gs, grpc_transport_stream_op *op) {
+  grpc_cronet_transport *ct = (grpc_cronet_transport *)gt;
+  GPR_ASSERT(ct->engine);
+  stream_obj *s = (stream_obj *)gs;
+  if (op->recv_trailing_metadata) {
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG,
+              "perform_stream_op - recv_trailing_metadata: on_complete=%p",
+              op->on_complete);
+    }
+    s->recv_trailing_metadata = op->recv_trailing_metadata;
+    GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]);
+    s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete;
+  }
+  if (op->recv_message) {
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p",
+              op->on_complete);
+    }
+    s->recv_message = (grpc_byte_buffer **)op->recv_message;
+    GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]);
+    GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]);
+    s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready;
+    s->callback_list[CB_RECV_MESSAGE][1] = op->on_complete;
+    s->read_requested = true;
+    next_recv_step(s, PERFORM_STREAM_OP);
+  }
+  if (op->recv_initial_metadata) {
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p",
+              op->on_complete);
+    }
+    s->recv_initial_metadata = op->recv_initial_metadata;
+    GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]);
+    GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]);
+    s->callback_list[CB_RECV_INITIAL_METADATA][0] =
+        op->recv_initial_metadata_ready;
+    s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete;
+  }
+  if (op->send_initial_metadata) {
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG,
+              "perform_stream_op - send_initial_metadata: on_complete=%p",
+              op->on_complete);
+    }
+    s->num_headers = 0;
+    convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head,
+                                       ct->host, s);
+    s->header_array.count = s->num_headers;
+    s->header_array.capacity = s->num_headers;
+    s->header_array.headers = s->headers;
+    GPR_ASSERT(!s->callback_list[CB_SEND_INITIAL_METADATA][0]);
+    s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete;
+  }
+  if (op->send_message) {
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p",
+              op->on_complete);
+    }
+    grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice,
+                          op->send_message->length, NULL);
+    // Check that compression flag is not ON. We don't support compression yet.
+    // TODO (makdharma): add compression support
+    GPR_ASSERT(op->send_message->flags == 0);
+    gpr_slice_buffer_add(&s->write_slice_buffer, s->slice);
+    if (s->cbs == NULL) {
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create");
+      }
+      s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks);
+      GPR_ASSERT(s->cbs);
+      s->read_closed = false;
+      s->response_trailers_received = false;
+      s->response_headers_received = false;
+      s->cronet_send_state = CRONET_SEND_IDLE;
+      s->cronet_recv_state = CRONET_RECV_IDLE;
+    }
+    GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]);
+    s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete;
+    next_send_step(s);
+  }
+  if (op->send_trailing_metadata) {
+    if (grpc_cronet_trace) {
+      gpr_log(GPR_DEBUG,
+              "perform_stream_op - send_trailing_metadata: on_complete=%p",
+              op->on_complete);
+    }
+    GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]);
+    s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete;
+    if (s->cbs) {
+      // Send an "empty" write to the far end to signal that we're done.
+      // This will induce the server to send down trailers.
+      if (grpc_cronet_trace) {
+        gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write");
+      }
+      cronet_bidirectional_stream_write(s->cbs, "abc", 0, true);
+    } else {
+      // We never created a stream. This was probably an empty request.
+      invoke_closing_callback(s);
+    }
+  }
+}
+
+static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                       grpc_stream *gs, grpc_stream_refcount *refcount,
+                       const void *server_data) {
+  stream_obj *s = (stream_obj *)gs;
+  memset(s->callback_list, 0, sizeof(s->callback_list));
+  s->cbs = NULL;
+  gpr_mu_init(&s->recv_mu);
+  s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES);
+  s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES);
+  gpr_slice_buffer_init(&s->write_slice_buffer);
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "cronet_transport - init_stream");
+  }
+  return 0;
+}
+
+static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                           grpc_stream *gs, void *and_free_memory) {
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "Destroy stream");
+  }
+  stream_obj *s = (stream_obj *)gs;
+  s->cbs = NULL;
+  gpr_free(s->read_buffer);
+  gpr_free(s->write_buffer);
+  gpr_free(s->url);
+  gpr_mu_destroy(&s->recv_mu);
+  if (and_free_memory) {
+    gpr_free(and_free_memory);
+  }
+}
+
+static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
+  grpc_cronet_transport *ct = (grpc_cronet_transport *)gt;
+  gpr_free(ct->host);
+  if (grpc_cronet_trace) {
+    gpr_log(GPR_DEBUG, "Destroy transport");
+  }
+}
+
+const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj),
+                                                  "cronet_http",
+                                                  init_stream,
+                                                  set_pollset_do_nothing,
+                                                  set_pollset_set_do_nothing,
+                                                  perform_stream_op,
+                                                  NULL,
+                                                  destroy_stream,
+                                                  destroy_transport,
+                                                  NULL};
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index 28d2d78d00..79ceeb66b3 100644
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -35,6 +35,7 @@
 #include <grpc/grpc.h>
 #include "src/core/lib/support/string.h"
 
+#include <grpc/compression.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
@@ -132,7 +133,8 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
   for (size_t i = 0; i < a->num_args; i++) {
     args[i] = &a->args[i];
   }
-  qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
+  if (a->num_args > 1)
+    qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
 
   grpc_channel_args *b = gpr_malloc(sizeof(grpc_channel_args));
   b->num_args = a->num_args;
@@ -147,6 +149,7 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
 
 void grpc_channel_args_destroy(grpc_channel_args *a) {
   size_t i;
+  if (!a) return;
   for (i = 0; i < a->num_args; i++) {
     switch (a->args[i].type) {
       case GRPC_ARG_STRING:
@@ -170,7 +173,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
   if (a == NULL) return 0;
   for (i = 0; i < a->num_args; ++i) {
     if (a->args[i].type == GRPC_ARG_INTEGER &&
-        !strcmp(GRPC_COMPRESSION_ALGORITHM_ARG, a->args[i].key)) {
+        !strcmp(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, a->args[i].key)) {
       return (grpc_compression_algorithm)a->args[i].value.integer;
       break;
     }
@@ -180,9 +183,10 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
 
 grpc_channel_args *grpc_channel_args_set_compression_algorithm(
     grpc_channel_args *a, grpc_compression_algorithm algorithm) {
+  GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT);
   grpc_arg tmp;
   tmp.type = GRPC_ARG_INTEGER;
-  tmp.key = GRPC_COMPRESSION_ALGORITHM_ARG;
+  tmp.key = GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM;
   tmp.value.integer = algorithm;
   return grpc_channel_args_copy_and_add(a, &tmp, 1);
 }
@@ -196,9 +200,11 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a,
     size_t i;
     for (i = 0; i < a->num_args; ++i) {
       if (a->args[i].type == GRPC_ARG_INTEGER &&
-          !strcmp(GRPC_COMPRESSION_ALGORITHM_STATE_ARG, a->args[i].key)) {
+          !strcmp(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET,
+                  a->args[i].key)) {
         *states_arg = &a->args[i].value.integer;
-        return 1; /* GPR_TRUE */
+        **states_arg |= 0x1; /* forcefully enable support for no compression */
+        return 1;
       }
     }
   }
@@ -207,27 +213,35 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a,
 
 grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
     grpc_channel_args **a, grpc_compression_algorithm algorithm, int state) {
-  int *states_arg;
+  int *states_arg = NULL;
   grpc_channel_args *result = *a;
   const int states_arg_found =
       find_compression_algorithm_states_bitset(*a, &states_arg);
 
-  if (states_arg_found) {
+  if (grpc_channel_args_get_compression_algorithm(*a) == algorithm &&
+      state == 0) {
+    char *algo_name = NULL;
+    GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algo_name) != 0);
+    gpr_log(GPR_ERROR,
+            "Tried to disable default compression algorithm '%s'. The "
+            "operation has been ignored.",
+            algo_name);
+  } else if (states_arg_found) {
     if (state != 0) {
       GPR_BITSET((unsigned *)states_arg, algorithm);
-    } else {
+    } else if (algorithm != GRPC_COMPRESS_NONE) {
       GPR_BITCLEAR((unsigned *)states_arg, algorithm);
     }
   } else {
     /* create a new arg */
     grpc_arg tmp;
     tmp.type = GRPC_ARG_INTEGER;
-    tmp.key = GRPC_COMPRESSION_ALGORITHM_STATE_ARG;
+    tmp.key = GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET;
     /* all enabled by default */
     tmp.value.integer = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
     if (state != 0) {
       GPR_BITSET((unsigned *)&tmp.value.integer, algorithm);
-    } else {
+    } else if (algorithm != GRPC_COMPRESS_NONE) {
       GPR_BITCLEAR((unsigned *)&tmp.value.integer, algorithm);
     }
     result = grpc_channel_args_copy_and_add(*a, &tmp, 1);
@@ -237,11 +251,11 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
   return result;
 }
 
-int grpc_channel_args_compression_algorithm_get_states(
+uint32_t grpc_channel_args_compression_algorithm_get_states(
     const grpc_channel_args *a) {
   int *states_arg;
   if (find_compression_algorithm_states_bitset(a, &states_arg)) {
-    return *states_arg;
+    return (uint32_t)*states_arg;
   } else {
     return (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; /* All algs. enabled */
   }
diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h
index 23c7b7b897..653d04f427 100644
--- a/src/core/lib/channel/channel_args.h
+++ b/src/core/lib/channel/channel_args.h
@@ -81,7 +81,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state(
  *
  * The i-th bit of the returned bitset corresponds to the i-th entry in the
  * grpc_compression_algorithm enum. */
-int grpc_channel_args_compression_algorithm_get_states(
+uint32_t grpc_channel_args_compression_algorithm_get_states(
     const grpc_channel_args *a);
 
 int grpc_channel_args_compare(const grpc_channel_args *a,
diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c
index ad182d1f69..42075b127b 100644
--- a/src/core/lib/channel/channel_stack.c
+++ b/src/core/lib/channel/channel_stack.c
@@ -106,6 +106,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
                              const grpc_channel_filter **filters,
                              size_t filter_count,
                              const grpc_channel_args *channel_args,
+                             grpc_transport *optional_transport,
                              const char *name, grpc_channel_stack *stack) {
   size_t call_size =
       ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call_stack)) +
@@ -127,6 +128,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
   for (i = 0; i < filter_count; i++) {
     args.channel_stack = stack;
     args.channel_args = channel_args;
+    args.optional_transport = optional_transport;
     args.is_first = i == 0;
     args.is_last = i == (filter_count - 1);
     elems[i].filter = filters[i];
@@ -189,9 +191,9 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
-                                 grpc_call_stack *call_stack,
-                                 grpc_pollset *pollset) {
+void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                                grpc_call_stack *call_stack,
+                                                grpc_polling_entity *pollent) {
   size_t count = call_stack->count;
   grpc_call_element *call_elems;
   char *user_data;
@@ -203,17 +205,19 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
 
   /* init per-filter data */
   for (i = 0; i < count; i++) {
-    call_elems[i].filter->set_pollset(exec_ctx, &call_elems[i], pollset);
+    call_elems[i].filter->set_pollset_or_pollset_set(exec_ctx, &call_elems[i],
+                                                     pollent);
     user_data +=
         ROUND_UP_TO_ALIGNMENT_SIZE(call_elems[i].filter->sizeof_call_data);
   }
 }
 
-void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_element *elem,
-                                        grpc_pollset *pollset) {}
+void grpc_call_stack_ignore_set_pollset_or_pollset_set(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_polling_entity *pollent) {}
 
 void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
+                             const grpc_call_stats *call_stats,
                              void *and_free_memory) {
   grpc_call_element *elems = CALL_ELEMS_FROM_STACK(stack);
   size_t count = stack->count;
@@ -221,7 +225,7 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
 
   /* destroy per-filter data */
   for (i = 0; i < count; i++) {
-    elems[i].filter->destroy_call_elem(exec_ctx, &elems[i],
+    elems[i].filter->destroy_call_elem(exec_ctx, &elems[i], call_stats,
                                        i == count - 1 ? and_free_memory : NULL);
   }
 }
@@ -259,6 +263,6 @@ void grpc_call_element_send_cancel(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *cur_elem) {
   grpc_transport_stream_op op;
   memset(&op, 0, sizeof(op));
-  op.cancel_with_status = GRPC_STATUS_CANCELLED;
+  op.cancel_error = GRPC_ERROR_CANCELLED;
   grpc_call_next_op(exec_ctx, cur_elem, &op);
 }
diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index 36c17cb467..41dd4a0d8a 100644
--- a/src/core/lib/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -45,7 +45,10 @@
 
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/time.h>
+
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/transport.h"
 
 typedef struct grpc_channel_element grpc_channel_element;
@@ -57,6 +60,8 @@ typedef struct grpc_call_stack grpc_call_stack;
 typedef struct {
   grpc_channel_stack *channel_stack;
   const grpc_channel_args *channel_args;
+  /** Transport, iff it is known */
+  grpc_transport *optional_transport;
   int is_first;
   int is_last;
 } grpc_channel_element_args;
@@ -67,6 +72,12 @@ typedef struct {
   grpc_call_context_element *context;
 } grpc_call_element_args;
 
+typedef struct {
+  grpc_transport_stream_stats transport_stream_stats;
+  gpr_timespec latency; /* From call creating to enqueing of received status */
+  grpc_status_code final_status;
+} grpc_call_stats;
+
 /* Channel filters specify:
    1. the amount of memory needed in the channel & call (via the sizeof_XXX
       members)
@@ -101,14 +112,16 @@ typedef struct {
      argument. */
   void (*init_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                          grpc_call_element_args *args);
-  void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                      grpc_pollset *pollset);
+  void (*set_pollset_or_pollset_set)(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_element *elem,
+                                     grpc_polling_entity *pollent);
   /* Destroy per call data.
      The filter does not need to do any chaining.
      The bottom filter of a stack will be passed a non-NULL pointer to
      \a and_free_memory that should be passed to gpr_free when destruction
      is complete. */
   void (*destroy_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                            const grpc_call_stats *stats,
                             void *and_free_memory);
 
   /* sizeof(per channel data) */
@@ -187,6 +200,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
                              grpc_iomgr_cb_func destroy, void *destroy_arg,
                              const grpc_channel_filter **filters,
                              size_t filter_count, const grpc_channel_args *args,
+                             grpc_transport *optional_transport,
                              const char *name, grpc_channel_stack *stack);
 /* Destroy a channel stack */
 void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx,
@@ -201,10 +215,11 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx,
                           grpc_call_context_element *context,
                           const void *transport_server_data,
                           grpc_call_stack *call_stack);
-/* Set a pollset for a call stack: must occur before the first op is started */
-void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
-                                 grpc_call_stack *call_stack,
-                                 grpc_pollset *pollset);
+/* Set a pollset or a pollset_set for a call stack: must occur before the first
+ * op is started */
+void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                                grpc_call_stack *call_stack,
+                                                grpc_polling_entity *pollent);
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 #define GRPC_CALL_STACK_REF(call_stack, reason) \
@@ -228,13 +243,14 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx,
 
 /* Destroy a call stack */
 void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
+                             const grpc_call_stats *call_stats,
                              void *and_free_memory);
 
-/* Ignore set pollset - used by filters to implement the set_pollset method
-   if they don't care about pollsets at all. Does nothing. */
-void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_element *elem,
-                                        grpc_pollset *pollset);
+/* Ignore set pollset{_set} - used by filters if they don't care about pollsets
+ * at all. Does nothing. */
+void grpc_call_stack_ignore_set_pollset_or_pollset_set(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_polling_entity *pollent);
 /* Call the next operation in a call stack */
 void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                        grpc_transport_stream_op *op);
diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c
index a8646c9565..eda4968f48 100644
--- a/src/core/lib/channel/channel_stack_builder.c
+++ b/src/core/lib/channel/channel_stack_builder.c
@@ -257,8 +257,8 @@ void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
   // and initialize it
   grpc_channel_stack_init(exec_ctx, initial_refs, destroy,
                           destroy_arg == NULL ? result : destroy_arg, filters,
-                          num_filters, builder->args, builder->name,
-                          channel_stack);
+                          num_filters, builder->args, builder->transport,
+                          builder->name, channel_stack);
 
   // run post-initialization functions
   i = 0;
diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c
index 5510c79b18..32ebe53ee6 100644
--- a/src/core/lib/channel/compress_filter.c
+++ b/src/core/lib/channel/compress_filter.c
@@ -47,7 +47,7 @@
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-int grpc_compress_filter_trace = 0;
+int grpc_compression_trace = 0;
 
 typedef struct call_data {
   gpr_slice_buffer slices; /**< Buffers up input slices to be compressed */
@@ -73,8 +73,8 @@ typedef struct call_data {
 typedef struct channel_data {
   /** The default, channel-level, compression algorithm */
   grpc_compression_algorithm default_compression_algorithm;
-  /** Compression options for the channel */
-  grpc_compression_options compression_options;
+  /** Bitset of enabled algorithms */
+  uint32_t enabled_algorithms_bitset;
   /** Supported compression algorithms */
   uint32_t supported_compression_algorithms;
 } channel_data;
@@ -96,9 +96,8 @@ static grpc_mdelem *compression_md_filter(void *user_data, grpc_mdelem *md) {
               md_c_str);
       calld->compression_algorithm = GRPC_COMPRESS_NONE;
     }
-    if (grpc_compression_options_is_algorithm_enabled(
-            &channeld->compression_options, calld->compression_algorithm) ==
-        0) {
+    if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
+                    calld->compression_algorithm)) {
       gpr_log(GPR_ERROR,
               "Invalid compression algorithm: '%s' (previously disabled). "
               "Ignoring.",
@@ -155,11 +154,11 @@ static void process_send_initial_metadata(
 static void continue_send_message(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem);
 
-static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, bool success) {
+static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
   grpc_call_element *elem = elemp;
   call_data *calld = elem->call_data;
   gpr_slice_buffer_reset_and_unref(&calld->slices);
-  calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, success);
+  calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error);
 }
 
 static void finish_send_message(grpc_exec_ctx *exec_ctx,
@@ -171,26 +170,28 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
   did_compress =
       grpc_msg_compress(calld->compression_algorithm, &calld->slices, &tmp);
   if (did_compress) {
-    if (grpc_compress_filter_trace) {
+    if (grpc_compression_trace) {
       char *algo_name;
       const size_t before_size = calld->slices.length;
       const size_t after_size = tmp.length;
       const float savings_ratio = 1.0f - (float)after_size / (float)before_size;
       GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm,
                                                  &algo_name));
-      gpr_log(GPR_DEBUG,
-              "Compressed[%s] %d bytes vs. %d bytes (%.2f%% savings)",
+      gpr_log(GPR_DEBUG, "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR
+                         " bytes (%.2f%% savings)",
               algo_name, before_size, after_size, 100 * savings_ratio);
     }
     gpr_slice_buffer_swap(&calld->slices, &tmp);
     calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS;
   } else {
-    if (grpc_compress_filter_trace) {
+    if (grpc_compression_trace) {
       char *algo_name;
       GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm,
                                                  &algo_name));
-      gpr_log(GPR_DEBUG, "Algorithm '%s' enabled but decided not to compress.",
-              algo_name);
+      gpr_log(GPR_DEBUG,
+              "Algorithm '%s' enabled but decided not to compress. Input size: "
+              "%" PRIuPTR,
+              algo_name, calld->slices.length);
     }
   }
 
@@ -205,7 +206,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
   grpc_call_next_op(exec_ctx, elem, &calld->send_op);
 }
 
-static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, bool success) {
+static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
   grpc_call_element *elem = elemp;
   call_data *calld = elem->call_data;
   gpr_slice_buffer_add(&calld->slices, calld->incoming_slice);
@@ -269,7 +270,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {
+                              const grpc_call_stats *stats, void *ignored) {
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   gpr_slice_buffer_destroy(&calld->slices);
@@ -280,32 +281,26 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               grpc_channel_element *elem,
                               grpc_channel_element_args *args) {
   channel_data *channeld = elem->channel_data;
-  grpc_compression_algorithm algo_idx;
 
-  grpc_compression_options_init(&channeld->compression_options);
-  channeld->compression_options.enabled_algorithms_bitset =
-      (uint32_t)grpc_channel_args_compression_algorithm_get_states(
-          args->channel_args);
+  channeld->enabled_algorithms_bitset =
+      grpc_channel_args_compression_algorithm_get_states(args->channel_args);
 
   channeld->default_compression_algorithm =
       grpc_channel_args_get_compression_algorithm(args->channel_args);
   /* Make sure the default isn't disabled. */
-  if (!grpc_compression_options_is_algorithm_enabled(
-          &channeld->compression_options,
-          channeld->default_compression_algorithm)) {
+  if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
+                  channeld->default_compression_algorithm)) {
     gpr_log(GPR_DEBUG,
             "compression algorithm %d not enabled: switching to none",
             channeld->default_compression_algorithm);
     channeld->default_compression_algorithm = GRPC_COMPRESS_NONE;
   }
-  channeld->compression_options.default_compression_algorithm =
-      channeld->default_compression_algorithm;
 
-  channeld->supported_compression_algorithms = 0;
-  for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
+  channeld->supported_compression_algorithms = 1; /* always support identity */
+  for (grpc_compression_algorithm algo_idx = 1;
+       algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
     /* skip disabled algorithms */
-    if (grpc_compression_options_is_algorithm_enabled(
-            &channeld->compression_options, algo_idx) == 0) {
+    if (!GPR_BITGET(channeld->enabled_algorithms_bitset, algo_idx)) {
       continue;
     }
     channeld->supported_compression_algorithms |= 1u << algo_idx;
@@ -323,7 +318,7 @@ const grpc_channel_filter grpc_compress_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/channel/compress_filter.h b/src/core/lib/channel/compress_filter.h
index cf5879d82e..e4a2a829d5 100644
--- a/src/core/lib/channel/compress_filter.h
+++ b/src/core/lib/channel/compress_filter.h
@@ -34,11 +34,11 @@
 #ifndef GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H
 #define GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H
 
-#include "src/core/lib/channel/channel_stack.h"
+#include <grpc/impl/codegen/compression_types.h>
 
-#define GRPC_COMPRESS_REQUEST_ALGORITHM_KEY "grpc-internal-encoding-request"
+#include "src/core/lib/channel/channel_stack.h"
 
-extern int grpc_compress_filter_trace;
+extern int grpc_compression_trace;
 
 /** Compression filter for outgoing data.
  *
@@ -48,7 +48,7 @@ extern int grpc_compress_filter_trace;
  *  - Channel configuration, as established at channel creation time.
  *  - The metadata accompanying the outgoing data to be compressed. This is
  *    taken as a request only. We may choose not to honor it. The metadata key
- *    is given by \a GRPC_COMPRESS_REQUEST_ALGORITHM_KEY.
+ *    is given by \a GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY.
  *
  * Compression can be disabled for concrete messages (for instance in order to
  * prevent CRIME/BEAST type attacks) by having the GRPC_WRITE_NO_COMPRESS set in
diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c
index 68a3a7d6fd..0a7d27a1dc 100644
--- a/src/core/lib/channel/connected_channel.c
+++ b/src/core/lib/channel/connected_channel.c
@@ -93,16 +93,18 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   GPR_ASSERT(r == 0);
 }
 
-static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                        grpc_pollset *pollset) {
+static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                       grpc_call_element *elem,
+                                       grpc_polling_entity *pollent) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  grpc_transport_set_pollset(exec_ctx, chand->transport,
-                             TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollset);
+  grpc_transport_set_pops(exec_ctx, chand->transport,
+                          TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollent);
 }
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats,
                               void *and_free_memory) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
@@ -137,7 +139,7 @@ static const grpc_channel_filter connected_channel_filter = {
     con_start_transport_op,
     sizeof(call_data),
     init_call_elem,
-    set_pollset,
+    set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c
index 516e708d1f..ab6c6c9ef0 100644
--- a/src/core/lib/channel/http_client_filter.c
+++ b/src/core/lib/channel/http_client_filter.c
@@ -38,6 +38,10 @@
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport_impl.h"
+
+#define EXPECTED_CONTENT_TYPE "application/grpc"
+#define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
 typedef struct call_data {
   grpc_linked_mdelem method;
@@ -74,13 +78,31 @@ static grpc_mdelem *client_recv_filter(void *user_data, grpc_mdelem *md) {
   } else if (md->key == GRPC_MDSTR_STATUS) {
     grpc_call_element_send_cancel(a->exec_ctx, a->elem);
     return NULL;
+  } else if (md == GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC) {
+    return NULL;
   } else if (md->key == GRPC_MDSTR_CONTENT_TYPE) {
+    const char *value_str = grpc_mdstr_as_c_string(md->value);
+    if (strncmp(value_str, EXPECTED_CONTENT_TYPE,
+                EXPECTED_CONTENT_TYPE_LENGTH) == 0 &&
+        (value_str[EXPECTED_CONTENT_TYPE_LENGTH] == '+' ||
+         value_str[EXPECTED_CONTENT_TYPE_LENGTH] == ';')) {
+      /* Although the C implementation doesn't (currently) generate them,
+         any custom +-suffix is explicitly valid. */
+      /* TODO(klempner): We should consider preallocating common values such
+         as +proto or +json, or at least stashing them if we see them. */
+      /* TODO(klempner): Should we be surfacing this to application code? */
+    } else {
+      /* TODO(klempner): We're currently allowing this, but we shouldn't
+         see it without a proxy so log for now. */
+      gpr_log(GPR_INFO, "Unexpected content-type '%s'", value_str);
+    }
     return NULL;
   }
   return md;
 }
 
-static void hc_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
+static void hc_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
+                       grpc_error *error) {
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
   client_recv_filter_args a;
@@ -88,7 +110,7 @@ static void hc_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
   a.exec_ctx = exec_ctx;
   grpc_metadata_batch_filter(calld->recv_initial_metadata, client_recv_filter,
                              &a);
-  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success);
+  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, error);
 }
 
 static grpc_mdelem *client_strip_filter(void *user_data, grpc_mdelem *md) {
@@ -156,7 +178,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {}
+                              const grpc_call_stats *stats, void *ignored) {}
 
 static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) {
   unsigned i;
@@ -179,7 +201,8 @@ static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) {
   return GRPC_MDELEM_SCHEME_HTTP;
 }
 
-static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args) {
+static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args,
+                                        const char *transport_name) {
   gpr_strvec v;
   size_t i;
   int is_first = 1;
@@ -201,8 +224,8 @@ static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args) {
     }
   }
 
-  gpr_asprintf(&tmp, "%sgrpc-c/%s (%s)", is_first ? "" : " ",
-               grpc_version_string(), GPR_PLATFORM_STRING);
+  gpr_asprintf(&tmp, "%sgrpc-c/%s (%s; %s)", is_first ? "" : " ",
+               grpc_version_string(), GPR_PLATFORM_STRING, transport_name);
   is_first = 0;
   gpr_strvec_add(&v, tmp);
 
@@ -233,9 +256,12 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               grpc_channel_element_args *args) {
   channel_data *chand = elem->channel_data;
   GPR_ASSERT(!args->is_last);
+  GPR_ASSERT(args->optional_transport != NULL);
   chand->static_scheme = scheme_from_args(args->channel_args);
   chand->user_agent = grpc_mdelem_from_metadata_strings(
-      GRPC_MDSTR_USER_AGENT, user_agent_from_args(args->channel_args));
+      GRPC_MDSTR_USER_AGENT,
+      user_agent_from_args(args->channel_args,
+                           args->optional_transport->vtable->name));
 }
 
 /* Destructor for channel data */
@@ -250,7 +276,7 @@ const grpc_channel_filter grpc_http_client_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/channel/http_client_filter.h b/src/core/lib/channel/http_client_filter.h
index a884b36318..47081175ea 100644
--- a/src/core/lib/channel/http_client_filter.h
+++ b/src/core/lib/channel/http_client_filter.h
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2015, Google Inc.
  * All rights reserved.
  *
@@ -39,6 +38,7 @@
 /* Processes metadata on the client side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_client_filter;
 
+/* Channel arg to override the http2 :scheme header */
 #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme"
 
 #endif /* GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H */
diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c
index ba865416de..d0beebd817 100644
--- a/src/core/lib/channel/http_server_filter.c
+++ b/src/core/lib/channel/http_server_filter.c
@@ -108,7 +108,7 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
     } else {
       /* TODO(klempner): We're currently allowing this, but we shouldn't
          see it without a proxy so log for now. */
-      gpr_log(GPR_INFO, "Unexpected content-type %s", value_str);
+      gpr_log(GPR_INFO, "Unexpected content-type '%s'", value_str);
     }
     return NULL;
   } else if (md->key == GRPC_MDSTR_TE || md->key == GRPC_MDSTR_METHOD ||
@@ -142,10 +142,11 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
   }
 }
 
-static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
+static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
+                       grpc_error *err) {
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
-  if (success) {
+  if (err == GRPC_ERROR_NONE) {
     server_filter_args a;
     a.elem = elem;
     a.exec_ctx = exec_ctx;
@@ -157,27 +158,35 @@ static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
         calld->seen_path && calld->seen_authority) {
       /* do nothing */
     } else {
+      err = GRPC_ERROR_CREATE("Bad incoming HTTP headers");
       if (!calld->seen_path) {
-        gpr_log(GPR_ERROR, "Missing :path header");
+        err = grpc_error_add_child(err,
+                                   GRPC_ERROR_CREATE("Missing :path header"));
       }
       if (!calld->seen_authority) {
-        gpr_log(GPR_ERROR, "Missing :authority header");
+        err = grpc_error_add_child(
+            err, GRPC_ERROR_CREATE("Missing :authority header"));
       }
       if (!calld->seen_method) {
-        gpr_log(GPR_ERROR, "Missing :method header");
+        err = grpc_error_add_child(err,
+                                   GRPC_ERROR_CREATE("Missing :method header"));
       }
       if (!calld->seen_scheme) {
-        gpr_log(GPR_ERROR, "Missing :scheme header");
+        err = grpc_error_add_child(err,
+                                   GRPC_ERROR_CREATE("Missing :scheme header"));
       }
       if (!calld->seen_te_trailers) {
-        gpr_log(GPR_ERROR, "Missing te trailers header");
+        err = grpc_error_add_child(
+            err, GRPC_ERROR_CREATE("Missing te: trailers header"));
       }
       /* Error this call out */
-      success = 0;
       grpc_call_element_send_cancel(exec_ctx, elem);
     }
+  } else {
+    GRPC_ERROR_REF(err);
   }
-  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success);
+  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, err);
+  GRPC_ERROR_UNREF(err);
 }
 
 static void hs_mutate_op(grpc_call_element *elem,
@@ -226,7 +235,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {}
+                              const grpc_call_stats *stats, void *ignored) {}
 
 /* Constructor for channel_data */
 static void init_channel_elem(grpc_exec_ctx *exec_ctx,
@@ -244,7 +253,7 @@ const grpc_channel_filter grpc_http_server_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/compression/compression_algorithm.c b/src/core/lib/compression/compression.c
index 7039364b7b..54efb5e855 100644
--- a/src/core/lib/compression/compression_algorithm.c
+++ b/src/core/lib/compression/compression.c
@@ -125,6 +125,28 @@ grpc_mdelem *grpc_compression_encoding_mdelem(
   return NULL;
 }
 
+void grpc_compression_options_init(grpc_compression_options *opts) {
+  memset(opts, 0, sizeof(*opts));
+  /* all enabled by default */
+  opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
+}
+
+void grpc_compression_options_enable_algorithm(
+    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
+  GPR_BITSET(&opts->enabled_algorithms_bitset, algorithm);
+}
+
+void grpc_compression_options_disable_algorithm(
+    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
+  GPR_BITCLEAR(&opts->enabled_algorithms_bitset, algorithm);
+}
+
+int grpc_compression_options_is_algorithm_enabled(
+    const grpc_compression_options *opts,
+    grpc_compression_algorithm algorithm) {
+  return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
+}
+
 /* TODO(dgq): Add the ability to specify parameters to the individual
  * compression algorithms */
 grpc_compression_algorithm grpc_compression_algorithm_for_level(
@@ -180,24 +202,3 @@ grpc_compression_algorithm grpc_compression_algorithm_for_level(
       abort();
   };
 }
-
-void grpc_compression_options_init(grpc_compression_options *opts) {
-  opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
-  opts->default_compression_algorithm = GRPC_COMPRESS_NONE;
-}
-
-void grpc_compression_options_enable_algorithm(
-    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
-  GPR_BITSET(&opts->enabled_algorithms_bitset, algorithm);
-}
-
-void grpc_compression_options_disable_algorithm(
-    grpc_compression_options *opts, grpc_compression_algorithm algorithm) {
-  GPR_BITCLEAR(&opts->enabled_algorithms_bitset, algorithm);
-}
-
-int grpc_compression_options_is_algorithm_enabled(
-    const grpc_compression_options *opts,
-    grpc_compression_algorithm algorithm) {
-  return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
-}
diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c
index 555f497b78..c56046785b 100644
--- a/src/core/lib/debug/trace.c
+++ b/src/core/lib/debug/trace.c
@@ -88,7 +88,11 @@ static void parse(const char *s) {
   split(s, &strings, &nstrings);
 
   for (i = 0; i < nstrings; i++) {
-    grpc_tracer_set_enabled(strings[i], 1);
+    if (strings[i][0] == '-') {
+      grpc_tracer_set_enabled(strings[i] + 1, 0);
+    } else {
+      grpc_tracer_set_enabled(strings[i], 1);
+    }
   }
 
   for (i = 0; i < nstrings; i++) {
@@ -117,7 +121,7 @@ int grpc_tracer_set_enabled(const char *name, int enabled) {
   tracer *t;
   if (0 == strcmp(name, "all")) {
     for (t = tracers; t; t = t->next) {
-      *t->flag = 1;
+      *t->flag = enabled;
     }
   } else {
     int found = 0;
diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c
index f22721ac8f..18135bcb58 100644
--- a/src/core/lib/http/httpcli.c
+++ b/src/core/lib/http/httpcli.c
@@ -39,12 +39,14 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
 
 #include "src/core/lib/http/format_request.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/support/string.h"
 
@@ -59,16 +61,16 @@ typedef struct {
   gpr_timespec deadline;
   int have_read_byte;
   const grpc_httpcli_handshaker *handshaker;
-  grpc_httpcli_response_cb on_response;
-  void *user_data;
+  grpc_closure *on_done;
   grpc_httpcli_context *context;
-  grpc_pollset *pollset;
+  grpc_polling_entity *pollent;
   grpc_iomgr_object iomgr_obj;
   gpr_slice_buffer incoming;
   gpr_slice_buffer outgoing;
   grpc_closure on_read;
   grpc_closure done_write;
   grpc_closure connected;
+  grpc_error *overall_error;
 } internal_request;
 
 static grpc_httpcli_get_override g_get_override = NULL;
@@ -76,6 +78,7 @@ static grpc_httpcli_post_override g_post_override = NULL;
 
 static void plaintext_handshake(grpc_exec_ctx *exec_ctx, void *arg,
                                 grpc_endpoint *endpoint, const char *host,
+                                gpr_timespec deadline,
                                 void (*on_done)(grpc_exec_ctx *exec_ctx,
                                                 void *arg,
                                                 grpc_endpoint *endpoint)) {
@@ -93,14 +96,14 @@ void grpc_httpcli_context_destroy(grpc_httpcli_context *context) {
   grpc_pollset_set_destroy(context->pollset_set);
 }
 
-static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req);
+static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req,
+                         grpc_error *due_to_error);
 
 static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
-                   int success) {
-  grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set,
-                               req->pollset);
-  req->on_response(exec_ctx, req->user_data,
-                   success ? &req->parser.http.response : NULL);
+                   grpc_error *error) {
+  grpc_polling_entity_del_from_pollset_set(exec_ctx, req->pollent,
+                                           req->context->pollset_set);
+  grpc_exec_ctx_sched(exec_ctx, req->on_done, error, NULL);
   grpc_http_parser_destroy(&req->parser);
   if (req->addresses != NULL) {
     grpc_resolved_addresses_destroy(req->addresses);
@@ -114,39 +117,49 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
   grpc_iomgr_unregister_object(&req->iomgr_obj);
   gpr_slice_buffer_destroy(&req->incoming);
   gpr_slice_buffer_destroy(&req->outgoing);
+  GRPC_ERROR_UNREF(req->overall_error);
   gpr_free(req);
 }
 
-static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success);
+static void append_error(internal_request *req, grpc_error *error) {
+  if (req->overall_error == GRPC_ERROR_NONE) {
+    req->overall_error = GRPC_ERROR_CREATE("Failed HTTP/1 client request");
+  }
+  grpc_resolved_address *addr = &req->addresses->addrs[req->next_address - 1];
+  char *addr_text = grpc_sockaddr_to_uri((struct sockaddr *)addr->addr);
+  req->overall_error = grpc_error_add_child(
+      req->overall_error,
+      grpc_error_set_str(error, GRPC_ERROR_STR_TARGET_ADDRESS, addr_text));
+  gpr_free(addr_text);
+}
 
 static void do_read(grpc_exec_ctx *exec_ctx, internal_request *req) {
   grpc_endpoint_read(exec_ctx, req->ep, &req->incoming, &req->on_read);
 }
 
-static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
+static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
+                    grpc_error *error) {
   internal_request *req = user_data;
   size_t i;
 
   for (i = 0; i < req->incoming.count; i++) {
     if (GPR_SLICE_LENGTH(req->incoming.slices[i])) {
       req->have_read_byte = 1;
-      if (!grpc_http_parser_parse(&req->parser, req->incoming.slices[i])) {
-        finish(exec_ctx, req, 0);
+      grpc_error *err =
+          grpc_http_parser_parse(&req->parser, req->incoming.slices[i]);
+      if (err != GRPC_ERROR_NONE) {
+        finish(exec_ctx, req, err);
         return;
       }
     }
   }
 
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     do_read(exec_ctx, req);
   } else if (!req->have_read_byte) {
-    next_address(exec_ctx, req);
+    next_address(exec_ctx, req, GRPC_ERROR_REF(error));
   } else {
-    int parse_success = grpc_http_parser_eof(&req->parser);
-    if (parse_success && (req->parser.type != GRPC_HTTP_RESPONSE)) {
-      parse_success = 0;
-    }
-    finish(exec_ctx, req, parse_success);
+    finish(exec_ctx, req, grpc_http_parser_eof(&req->parser));
   }
 }
 
@@ -154,12 +167,12 @@ static void on_written(grpc_exec_ctx *exec_ctx, internal_request *req) {
   do_read(exec_ctx, req);
 }
 
-static void done_write(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void done_write(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   internal_request *req = arg;
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     on_written(exec_ctx, req);
   } else {
-    next_address(exec_ctx, req);
+    next_address(exec_ctx, req, GRPC_ERROR_REF(error));
   }
 }
 
@@ -174,7 +187,8 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
   internal_request *req = arg;
 
   if (!ep) {
-    next_address(exec_ctx, req);
+    next_address(exec_ctx, req,
+                 GRPC_ERROR_CREATE("Unexplained handshake failure"));
     return;
   }
 
@@ -182,23 +196,30 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
   start_write(exec_ctx, req);
 }
 
-static void on_connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void on_connected(grpc_exec_ctx *exec_ctx, void *arg,
+                         grpc_error *error) {
   internal_request *req = arg;
 
   if (!req->ep) {
-    next_address(exec_ctx, req);
+    next_address(exec_ctx, req, GRPC_ERROR_REF(error));
     return;
   }
   req->handshaker->handshake(
       exec_ctx, req, req->ep,
       req->ssl_host_override ? req->ssl_host_override : req->host,
-      on_handshake_done);
+      req->deadline, on_handshake_done);
 }
 
-static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req) {
+static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req,
+                         grpc_error *error) {
   grpc_resolved_address *addr;
+  if (error != GRPC_ERROR_NONE) {
+    append_error(req, error);
+  }
   if (req->next_address == req->addresses->naddrs) {
-    finish(exec_ctx, req, 0);
+    finish(exec_ctx, req,
+           GRPC_ERROR_CREATE_REFERENCING("Failed HTTP requests to all targets",
+                                         &req->overall_error, 1));
     return;
   }
   addr = &req->addresses->addrs[req->next_address++];
@@ -208,34 +229,34 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req) {
       (struct sockaddr *)&addr->addr, addr->len, req->deadline);
 }
 
-static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
-                        grpc_resolved_addresses *addresses) {
+static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   internal_request *req = arg;
-  if (!addresses) {
-    finish(exec_ctx, req, 0);
+  if (error != GRPC_ERROR_NONE) {
+    finish(exec_ctx, req, error);
     return;
   }
-  req->addresses = addresses;
   req->next_address = 0;
-  next_address(exec_ctx, req);
+  next_address(exec_ctx, req, GRPC_ERROR_NONE);
 }
 
-static void internal_request_begin(
-    grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-    grpc_pollset *pollset, const grpc_httpcli_request *request,
-    gpr_timespec deadline, grpc_httpcli_response_cb on_response,
-    void *user_data, const char *name, gpr_slice request_text) {
+static void internal_request_begin(grpc_exec_ctx *exec_ctx,
+                                   grpc_httpcli_context *context,
+                                   grpc_polling_entity *pollent,
+                                   const grpc_httpcli_request *request,
+                                   gpr_timespec deadline, grpc_closure *on_done,
+                                   grpc_httpcli_response *response,
+                                   const char *name, gpr_slice request_text) {
   internal_request *req = gpr_malloc(sizeof(internal_request));
   memset(req, 0, sizeof(*req));
   req->request_text = request_text;
-  grpc_http_parser_init(&req->parser);
-  req->on_response = on_response;
-  req->user_data = user_data;
+  grpc_http_parser_init(&req->parser, GRPC_HTTP_RESPONSE, response);
+  req->on_done = on_done;
   req->deadline = deadline;
   req->handshaker =
       request->handshaker ? request->handshaker : &grpc_httpcli_plaintext;
   req->context = context;
-  req->pollset = pollset;
+  req->pollent = pollent;
+  req->overall_error = GRPC_ERROR_NONE;
   grpc_closure_init(&req->on_read, on_read, req);
   grpc_closure_init(&req->done_write, done_write, req);
   gpr_slice_buffer_init(&req->incoming);
@@ -244,45 +265,46 @@ static void internal_request_begin(
   req->host = gpr_strdup(request->host);
   req->ssl_host_override = gpr_strdup(request->ssl_host_override);
 
-  grpc_pollset_set_add_pollset(exec_ctx, req->context->pollset_set,
-                               req->pollset);
+  GPR_ASSERT(pollent);
+  grpc_polling_entity_add_to_pollset_set(exec_ctx, req->pollent,
+                                         req->context->pollset_set);
   grpc_resolve_address(exec_ctx, request->host, req->handshaker->default_port,
-                       on_resolved, req);
+                       grpc_closure_create(on_resolved, req), &req->addresses);
 }
 
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                      grpc_pollset *pollset,
+                      grpc_polling_entity *pollent,
                       const grpc_httpcli_request *request,
-                      gpr_timespec deadline,
-                      grpc_httpcli_response_cb on_response, void *user_data) {
+                      gpr_timespec deadline, grpc_closure *on_done,
+                      grpc_httpcli_response *response) {
   char *name;
   if (g_get_override &&
-      g_get_override(exec_ctx, request, deadline, on_response, user_data)) {
+      g_get_override(exec_ctx, request, deadline, on_done, response)) {
     return;
   }
   gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->http.path);
-  internal_request_begin(exec_ctx, context, pollset, request, deadline,
-                         on_response, user_data, name,
+  internal_request_begin(exec_ctx, context, pollent, request, deadline, on_done,
+                         response, name,
                          grpc_httpcli_format_get_request(request));
   gpr_free(name);
 }
 
 void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                       grpc_pollset *pollset,
+                       grpc_polling_entity *pollent,
                        const grpc_httpcli_request *request,
                        const char *body_bytes, size_t body_size,
-                       gpr_timespec deadline,
-                       grpc_httpcli_response_cb on_response, void *user_data) {
+                       gpr_timespec deadline, grpc_closure *on_done,
+                       grpc_httpcli_response *response) {
   char *name;
   if (g_post_override &&
       g_post_override(exec_ctx, request, body_bytes, body_size, deadline,
-                      on_response, user_data)) {
+                      on_done, response)) {
     return;
   }
   gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->http.path);
   internal_request_begin(
-      exec_ctx, context, pollset, request, deadline, on_response, user_data,
-      name, grpc_httpcli_format_post_request(request, body_bytes, body_size));
+      exec_ctx, context, pollent, request, deadline, on_done, response, name,
+      grpc_httpcli_format_post_request(request, body_bytes, body_size));
   gpr_free(name);
 }
 
diff --git a/src/core/lib/http/httpcli.h b/src/core/lib/http/httpcli.h
index 11a32a125c..662e176f4c 100644
--- a/src/core/lib/http/httpcli.h
+++ b/src/core/lib/http/httpcli.h
@@ -41,6 +41,7 @@
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 
 /* User agent this library reports */
@@ -56,7 +57,7 @@ typedef struct grpc_httpcli_context {
 typedef struct {
   const char *default_port;
   void (*handshake)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint,
-                    const char *host,
+                    const char *host, gpr_timespec deadline,
                     void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg,
                                     grpc_endpoint *endpoint));
 } grpc_httpcli_handshaker;
@@ -81,11 +82,6 @@ typedef struct grpc_httpcli_request {
 /* Expose the parser response type as a httpcli response too */
 typedef struct grpc_http_response grpc_httpcli_response;
 
-/* Callback for grpc_httpcli_get and grpc_httpcli_post. */
-typedef void (*grpc_httpcli_response_cb)(grpc_exec_ctx *exec_ctx,
-                                         void *user_data,
-                                         const grpc_http_response *response);
-
 void grpc_httpcli_context_init(grpc_httpcli_context *context);
 void grpc_httpcli_context_destroy(grpc_httpcli_context *context);
 
@@ -100,10 +96,10 @@ void grpc_httpcli_context_destroy(grpc_httpcli_context *context);
    'on_response' is a callback to report results to (and 'user_data' is a user
      supplied pointer to pass to said call) */
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                      grpc_pollset *pollset,
+                      grpc_polling_entity *pollent,
                       const grpc_httpcli_request *request,
-                      gpr_timespec deadline,
-                      grpc_httpcli_response_cb on_response, void *user_data);
+                      gpr_timespec deadline, grpc_closure *on_complete,
+                      grpc_httpcli_response *response);
 
 /* Asynchronously perform a HTTP POST.
    'context' specifies the http context under which to do the post
@@ -121,22 +117,22 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
      supplied pointer to pass to said call)
    Does not support ?var1=val1&var2=val2 in the path. */
 void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
-                       grpc_pollset *pollset,
+                       grpc_polling_entity *pollent,
                        const grpc_httpcli_request *request,
                        const char *body_bytes, size_t body_size,
-                       gpr_timespec deadline,
-                       grpc_httpcli_response_cb on_response, void *user_data);
+                       gpr_timespec deadline, grpc_closure *on_complete,
+                       grpc_httpcli_response *response);
 
 /* override functions return 1 if they handled the request, 0 otherwise */
 typedef int (*grpc_httpcli_get_override)(grpc_exec_ctx *exec_ctx,
                                          const grpc_httpcli_request *request,
                                          gpr_timespec deadline,
-                                         grpc_httpcli_response_cb on_response,
-                                         void *user_data);
+                                         grpc_closure *on_complete,
+                                         grpc_httpcli_response *response);
 typedef int (*grpc_httpcli_post_override)(
     grpc_exec_ctx *exec_ctx, const grpc_httpcli_request *request,
     const char *body_bytes, size_t body_size, gpr_timespec deadline,
-    grpc_httpcli_response_cb on_response, void *user_data);
+    grpc_closure *on_complete, grpc_httpcli_response *response);
 
 void grpc_httpcli_set_override(grpc_httpcli_get_override get,
                                grpc_httpcli_post_override post);
diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c
index ea4bff30d4..a57d93bb7b 100644
--- a/src/core/lib/http/httpcli_security_connector.c
+++ b/src/core/lib/http/httpcli_security_connector.c
@@ -38,7 +38,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
 
@@ -61,6 +61,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) {
 static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_security_connector *sc,
                                      grpc_endpoint *nonsecure_endpoint,
+                                     gpr_timespec deadline,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
   grpc_httpcli_ssl_channel_security_connector *c =
@@ -79,7 +80,7 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
     grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, cb, user_data);
+                               nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
@@ -163,6 +164,7 @@ static void on_secure_transport_setup_done(grpc_exec_ctx *exec_ctx, void *rp,
 
 static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
                           grpc_endpoint *tcp, const char *host,
+                          gpr_timespec deadline,
                           void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg,
                                           grpc_endpoint *endpoint)) {
   grpc_channel_security_connector *sc = NULL;
@@ -181,7 +183,7 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
                  pem_root_certs, pem_root_certs_size, host, &sc) ==
              GRPC_SECURITY_OK);
   grpc_channel_security_connector_do_handshake(
-      exec_ctx, sc, tcp, on_secure_transport_setup_done, c);
+      exec_ctx, sc, tcp, deadline, on_secure_transport_setup_done, c);
   GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
 }
 
diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c
index a7efb5e73e..92ed08ae51 100644
--- a/src/core/lib/http/parser.c
+++ b/src/core/lib/http/parser.c
@@ -48,37 +48,38 @@ static char *buf2str(void *buffer, size_t length) {
   return out;
 }
 
-static int handle_response_line(grpc_http_parser *parser) {
+static grpc_error *handle_response_line(grpc_http_parser *parser) {
   uint8_t *beg = parser->cur_line;
   uint8_t *cur = beg;
   uint8_t *end = beg + parser->cur_line_length;
 
-  if (cur == end || *cur++ != 'H') goto error;
-  if (cur == end || *cur++ != 'T') goto error;
-  if (cur == end || *cur++ != 'T') goto error;
-  if (cur == end || *cur++ != 'P') goto error;
-  if (cur == end || *cur++ != '/') goto error;
-  if (cur == end || *cur++ != '1') goto error;
-  if (cur == end || *cur++ != '.') goto error;
-  if (cur == end || *cur < '0' || *cur++ > '1') goto error;
-  if (cur == end || *cur++ != ' ') goto error;
-  if (cur == end || *cur < '1' || *cur++ > '9') goto error;
-  if (cur == end || *cur < '0' || *cur++ > '9') goto error;
-  if (cur == end || *cur < '0' || *cur++ > '9') goto error;
-  parser->http.response.status =
+  if (cur == end || *cur++ != 'H') return GRPC_ERROR_CREATE("Expected 'H'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'P') return GRPC_ERROR_CREATE("Expected 'P'");
+  if (cur == end || *cur++ != '/') return GRPC_ERROR_CREATE("Expected '/'");
+  if (cur == end || *cur++ != '1') return GRPC_ERROR_CREATE("Expected '1'");
+  if (cur == end || *cur++ != '.') return GRPC_ERROR_CREATE("Expected '.'");
+  if (cur == end || *cur < '0' || *cur++ > '1') {
+    return GRPC_ERROR_CREATE("Expected HTTP/1.0 or HTTP/1.1");
+  }
+  if (cur == end || *cur++ != ' ') return GRPC_ERROR_CREATE("Expected ' '");
+  if (cur == end || *cur < '1' || *cur++ > '9')
+    return GRPC_ERROR_CREATE("Expected status code");
+  if (cur == end || *cur < '0' || *cur++ > '9')
+    return GRPC_ERROR_CREATE("Expected status code");
+  if (cur == end || *cur < '0' || *cur++ > '9')
+    return GRPC_ERROR_CREATE("Expected status code");
+  parser->http.response->status =
       (cur[-3] - '0') * 100 + (cur[-2] - '0') * 10 + (cur[-1] - '0');
-  if (cur == end || *cur++ != ' ') goto error;
+  if (cur == end || *cur++ != ' ') return GRPC_ERROR_CREATE("Expected ' '");
 
   /* we don't really care about the status code message */
 
-  return 1;
-
-error:
-  if (grpc_http1_trace) gpr_log(GPR_ERROR, "Failed parsing response line");
-  return 0;
+  return GRPC_ERROR_NONE;
 }
 
-static int handle_request_line(grpc_http_parser *parser) {
+static grpc_error *handle_request_line(grpc_http_parser *parser) {
   uint8_t *beg = parser->cur_line;
   uint8_t *cur = beg;
   uint8_t *end = beg + parser->cur_line_length;
@@ -87,83 +88,81 @@ static int handle_request_line(grpc_http_parser *parser) {
 
   while (cur != end && *cur++ != ' ')
     ;
-  if (cur == end) goto error;
-  parser->http.request.method = buf2str(beg, (size_t)(cur - beg - 1));
+  if (cur == end) return GRPC_ERROR_CREATE("No method on HTTP request line");
+  parser->http.request->method = buf2str(beg, (size_t)(cur - beg - 1));
 
   beg = cur;
   while (cur != end && *cur++ != ' ')
     ;
-  if (cur == end) goto error;
-  parser->http.request.path = buf2str(beg, (size_t)(cur - beg - 1));
-
-  if (cur == end || *cur++ != 'H') goto error;
-  if (cur == end || *cur++ != 'T') goto error;
-  if (cur == end || *cur++ != 'T') goto error;
-  if (cur == end || *cur++ != 'P') goto error;
-  if (cur == end || *cur++ != '/') goto error;
+  if (cur == end) return GRPC_ERROR_CREATE("No path on HTTP request line");
+  parser->http.request->path = buf2str(beg, (size_t)(cur - beg - 1));
+
+  if (cur == end || *cur++ != 'H') return GRPC_ERROR_CREATE("Expected 'H'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'P') return GRPC_ERROR_CREATE("Expected 'P'");
+  if (cur == end || *cur++ != '/') return GRPC_ERROR_CREATE("Expected '/'");
   vers_major = (uint8_t)(*cur++ - '1' + 1);
   ++cur;
-  if (cur == end) goto error;
+  if (cur == end)
+    return GRPC_ERROR_CREATE("End of line in HTTP version string");
   vers_minor = (uint8_t)(*cur++ - '1' + 1);
 
   if (vers_major == 1) {
     if (vers_minor == 0) {
-      parser->http.request.version = GRPC_HTTP_HTTP10;
+      parser->http.request->version = GRPC_HTTP_HTTP10;
     } else if (vers_minor == 1) {
-      parser->http.request.version = GRPC_HTTP_HTTP11;
+      parser->http.request->version = GRPC_HTTP_HTTP11;
     } else {
-      goto error;
+      return GRPC_ERROR_CREATE(
+          "Expected one of HTTP/1.0, HTTP/1.1, or HTTP/2.0");
     }
   } else if (vers_major == 2) {
     if (vers_minor == 0) {
-      parser->http.request.version = GRPC_HTTP_HTTP20;
+      parser->http.request->version = GRPC_HTTP_HTTP20;
     } else {
-      goto error;
+      return GRPC_ERROR_CREATE(
+          "Expected one of HTTP/1.0, HTTP/1.1, or HTTP/2.0");
     }
   } else {
-    goto error;
+    return GRPC_ERROR_CREATE("Expected one of HTTP/1.0, HTTP/1.1, or HTTP/2.0");
   }
 
-  return 1;
-
-error:
-  if (grpc_http1_trace) gpr_log(GPR_ERROR, "Failed parsing request line");
-  return 0;
+  return GRPC_ERROR_NONE;
 }
 
-static int handle_first_line(grpc_http_parser *parser) {
-  if (parser->cur_line[0] == 'H') {
-    parser->type = GRPC_HTTP_RESPONSE;
-    return handle_response_line(parser);
-  } else {
-    parser->type = GRPC_HTTP_REQUEST;
-    return handle_request_line(parser);
+static grpc_error *handle_first_line(grpc_http_parser *parser) {
+  switch (parser->type) {
+    case GRPC_HTTP_REQUEST:
+      return handle_request_line(parser);
+    case GRPC_HTTP_RESPONSE:
+      return handle_response_line(parser);
   }
+  GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
 }
 
-static int add_header(grpc_http_parser *parser) {
+static grpc_error *add_header(grpc_http_parser *parser) {
   uint8_t *beg = parser->cur_line;
   uint8_t *cur = beg;
   uint8_t *end = beg + parser->cur_line_length;
   size_t *hdr_count = NULL;
   grpc_http_header **hdrs = NULL;
   grpc_http_header hdr = {NULL, NULL};
+  grpc_error *error = GRPC_ERROR_NONE;
 
   GPR_ASSERT(cur != end);
 
   if (*cur == ' ' || *cur == '\t') {
-    if (grpc_http1_trace)
-      gpr_log(GPR_ERROR, "Continued header lines not supported yet");
-    goto error;
+    error = GRPC_ERROR_CREATE("Continued header lines not supported yet");
+    goto done;
   }
 
   while (cur != end && *cur != ':') {
     cur++;
   }
   if (cur == end) {
-    if (grpc_http1_trace)
-      gpr_log(GPR_ERROR, "Didn't find ':' in header string");
-    goto error;
+    error = GRPC_ERROR_CREATE("Didn't find ':' in header string");
+    goto done;
   }
   GPR_ASSERT(cur >= beg);
   hdr.key = buf2str(beg, (size_t)(cur - beg));
@@ -175,14 +174,15 @@ static int add_header(grpc_http_parser *parser) {
   GPR_ASSERT((size_t)(end - cur) >= parser->cur_line_end_length);
   hdr.value = buf2str(cur, (size_t)(end - cur) - parser->cur_line_end_length);
 
-  if (parser->type == GRPC_HTTP_RESPONSE) {
-    hdr_count = &parser->http.response.hdr_count;
-    hdrs = &parser->http.response.hdrs;
-  } else if (parser->type == GRPC_HTTP_REQUEST) {
-    hdr_count = &parser->http.request.hdr_count;
-    hdrs = &parser->http.request.hdrs;
-  } else {
-    return 0;
+  switch (parser->type) {
+    case GRPC_HTTP_RESPONSE:
+      hdr_count = &parser->http.response->hdr_count;
+      hdrs = &parser->http.response->hdrs;
+      break;
+    case GRPC_HTTP_REQUEST:
+      hdr_count = &parser->http.request->hdr_count;
+      hdrs = &parser->http.request->hdrs;
+      break;
   }
 
   if (*hdr_count == parser->hdr_capacity) {
@@ -191,20 +191,21 @@ static int add_header(grpc_http_parser *parser) {
     *hdrs = gpr_realloc(*hdrs, parser->hdr_capacity * sizeof(**hdrs));
   }
   (*hdrs)[(*hdr_count)++] = hdr;
-  return 1;
 
-error:
-  gpr_free(hdr.key);
-  gpr_free(hdr.value);
-  return 0;
+done:
+  if (error != GRPC_ERROR_NONE) {
+    gpr_free(hdr.key);
+    gpr_free(hdr.value);
+  }
+  return error;
 }
 
-static int finish_line(grpc_http_parser *parser) {
+static grpc_error *finish_line(grpc_http_parser *parser) {
+  grpc_error *err;
   switch (parser->state) {
     case GRPC_HTTP_FIRST_LINE:
-      if (!handle_first_line(parser)) {
-        return 0;
-      }
+      err = handle_first_line(parser);
+      if (err != GRPC_ERROR_NONE) return err;
       parser->state = GRPC_HTTP_HEADERS;
       break;
     case GRPC_HTTP_HEADERS:
@@ -212,30 +213,31 @@ static int finish_line(grpc_http_parser *parser) {
         parser->state = GRPC_HTTP_BODY;
         break;
       }
-      if (!add_header(parser)) {
-        return 0;
+      err = add_header(parser);
+      if (err != GRPC_ERROR_NONE) {
+        return err;
       }
       break;
     case GRPC_HTTP_BODY:
-      GPR_UNREACHABLE_CODE(return 0);
+      GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
   }
 
   parser->cur_line_length = 0;
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
-static int addbyte_body(grpc_http_parser *parser, uint8_t byte) {
+static grpc_error *addbyte_body(grpc_http_parser *parser, uint8_t byte) {
   size_t *body_length = NULL;
   char **body = NULL;
 
   if (parser->type == GRPC_HTTP_RESPONSE) {
-    body_length = &parser->http.response.body_length;
-    body = &parser->http.response.body;
+    body_length = &parser->http.response->body_length;
+    body = &parser->http.response->body;
   } else if (parser->type == GRPC_HTTP_REQUEST) {
-    body_length = &parser->http.request.body_length;
-    body = &parser->http.request.body;
+    body_length = &parser->http.request->body_length;
+    body = &parser->http.request->body;
   } else {
-    return 0;
+    GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
   }
 
   if (*body_length == parser->body_capacity) {
@@ -245,34 +247,34 @@ static int addbyte_body(grpc_http_parser *parser, uint8_t byte) {
   (*body)[*body_length] = (char)byte;
   (*body_length)++;
 
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
-static int check_line(grpc_http_parser *parser) {
+static bool check_line(grpc_http_parser *parser) {
   if (parser->cur_line_length >= 2 &&
       parser->cur_line[parser->cur_line_length - 2] == '\r' &&
       parser->cur_line[parser->cur_line_length - 1] == '\n') {
-    return 1;
+    return true;
   }
 
   // HTTP request with \n\r line termiantors.
   else if (parser->cur_line_length >= 2 &&
            parser->cur_line[parser->cur_line_length - 2] == '\n' &&
            parser->cur_line[parser->cur_line_length - 1] == '\r') {
-    return 1;
+    return true;
   }
 
   // HTTP request with only \n line terminators.
   else if (parser->cur_line_length >= 1 &&
            parser->cur_line[parser->cur_line_length - 1] == '\n') {
     parser->cur_line_end_length = 1;
-    return 1;
+    return true;
   }
 
-  return 0;
+  return false;
 }
 
-static int addbyte(grpc_http_parser *parser, uint8_t byte) {
+static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte) {
   switch (parser->state) {
     case GRPC_HTTP_FIRST_LINE:
     case GRPC_HTTP_HEADERS:
@@ -287,7 +289,7 @@ static int addbyte(grpc_http_parser *parser, uint8_t byte) {
       if (check_line(parser)) {
         return finish_line(parser);
       } else {
-        return 1;
+        return GRPC_ERROR_NONE;
       }
       GPR_UNREACHABLE_CODE(return 0);
     case GRPC_HTTP_BODY:
@@ -296,46 +298,53 @@ static int addbyte(grpc_http_parser *parser, uint8_t byte) {
   GPR_UNREACHABLE_CODE(return 0);
 }
 
-void grpc_http_parser_init(grpc_http_parser *parser) {
+void grpc_http_parser_init(grpc_http_parser *parser, grpc_http_type type,
+                           void *request_or_response) {
   memset(parser, 0, sizeof(*parser));
   parser->state = GRPC_HTTP_FIRST_LINE;
-  parser->type = GRPC_HTTP_UNKNOWN;
+  parser->type = type;
+  parser->http.request_or_response = request_or_response;
   parser->cur_line_end_length = 2;
 }
 
-void grpc_http_parser_destroy(grpc_http_parser *parser) {
+void grpc_http_parser_destroy(grpc_http_parser *parser) {}
+
+void grpc_http_request_destroy(grpc_http_request *request) {
   size_t i;
-  if (parser->type == GRPC_HTTP_RESPONSE) {
-    gpr_free(parser->http.response.body);
-    for (i = 0; i < parser->http.response.hdr_count; i++) {
-      gpr_free(parser->http.response.hdrs[i].key);
-      gpr_free(parser->http.response.hdrs[i].value);
-    }
-    gpr_free(parser->http.response.hdrs);
-  } else if (parser->type == GRPC_HTTP_REQUEST) {
-    gpr_free(parser->http.request.body);
-    for (i = 0; i < parser->http.request.hdr_count; i++) {
-      gpr_free(parser->http.request.hdrs[i].key);
-      gpr_free(parser->http.request.hdrs[i].value);
-    }
-    gpr_free(parser->http.request.hdrs);
-    gpr_free(parser->http.request.method);
-    gpr_free(parser->http.request.path);
+  gpr_free(request->body);
+  for (i = 0; i < request->hdr_count; i++) {
+    gpr_free(request->hdrs[i].key);
+    gpr_free(request->hdrs[i].value);
   }
+  gpr_free(request->hdrs);
+  gpr_free(request->method);
+  gpr_free(request->path);
 }
 
-int grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice) {
+void grpc_http_response_destroy(grpc_http_response *response) {
+  size_t i;
+  gpr_free(response->body);
+  for (i = 0; i < response->hdr_count; i++) {
+    gpr_free(response->hdrs[i].key);
+    gpr_free(response->hdrs[i].value);
+  }
+  gpr_free(response->hdrs);
+}
+
+grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice) {
   size_t i;
 
   for (i = 0; i < GPR_SLICE_LENGTH(slice); i++) {
-    if (!addbyte(parser, GPR_SLICE_START_PTR(slice)[i])) {
-      return 0;
-    }
+    grpc_error *err = addbyte(parser, GPR_SLICE_START_PTR(slice)[i]);
+    if (err != GRPC_ERROR_NONE) return err;
   }
 
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
-int grpc_http_parser_eof(grpc_http_parser *parser) {
-  return parser->state == GRPC_HTTP_BODY;
+grpc_error *grpc_http_parser_eof(grpc_http_parser *parser) {
+  if (parser->state != GRPC_HTTP_BODY) {
+    return GRPC_ERROR_CREATE("Did not finish headers");
+  }
+  return GRPC_ERROR_NONE;
 }
diff --git a/src/core/lib/http/parser.c.orig b/src/core/lib/http/parser.c.orig
new file mode 100644
index 0000000000..74d90fd8bf
--- /dev/null
+++ b/src/core/lib/http/parser.c.orig
@@ -0,0 +1,357 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/http/parser.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/useful.h>
+
+int grpc_http1_trace = 0;
+
+static char *buf2str(void *buffer, size_t length) {
+  char *out = gpr_malloc(length + 1);
+  memcpy(out, buffer, length);
+  out[length] = 0;
+  return out;
+}
+
+static grpc_error *handle_response_line(grpc_http_parser *parser) {
+  uint8_t *beg = parser->cur_line;
+  uint8_t *cur = beg;
+  uint8_t *end = beg + parser->cur_line_length;
+
+  if (cur == end || *cur++ != 'H') return GRPC_ERROR_CREATE("Expected 'H'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'P') return GRPC_ERROR_CREATE("Expected 'P'");
+  if (cur == end || *cur++ != '/') return GRPC_ERROR_CREATE("Expected '/'");
+  if (cur == end || *cur++ != '1') return GRPC_ERROR_CREATE("Expected '1'");
+  if (cur == end || *cur++ != '.') return GRPC_ERROR_CREATE("Expected '.'");
+  if (cur == end || *cur < '0' || *cur++ > '1') {
+    return GRPC_ERROR_CREATE("Expected HTTP/1.0 or HTTP/1.1");
+  }
+  if (cur == end || *cur++ != ' ') return GRPC_ERROR_CREATE("Expected ' '");
+  if (cur == end || *cur < '1' || *cur++ > '9')
+    return GRPC_ERROR_CREATE("Expected status code");
+  if (cur == end || *cur < '0' || *cur++ > '9')
+    return GRPC_ERROR_CREATE("Expected status code");
+  if (cur == end || *cur < '0' || *cur++ > '9')
+    return GRPC_ERROR_CREATE("Expected status code");
+  parser->http.response->status =
+      (cur[-3] - '0') * 100 + (cur[-2] - '0') * 10 + (cur[-1] - '0');
+  if (cur == end || *cur++ != ' ') return GRPC_ERROR_CREATE("Expected ' '");
+
+  /* we don't really care about the status code message */
+
+  return GRPC_ERROR_NONE;
+}
+
+static grpc_error *handle_request_line(grpc_http_parser *parser) {
+  uint8_t *beg = parser->cur_line;
+  uint8_t *cur = beg;
+  uint8_t *end = beg + parser->cur_line_length;
+  uint8_t vers_major = 0;
+  uint8_t vers_minor = 0;
+
+  while (cur != end && *cur++ != ' ')
+    ;
+  if (cur == end) return GRPC_ERROR_CREATE("No method on HTTP request line");
+  parser->http.request->method = buf2str(beg, (size_t)(cur - beg - 1));
+
+  beg = cur;
+  while (cur != end && *cur++ != ' ')
+    ;
+  if (cur == end) return GRPC_ERROR_CREATE("No path on HTTP request line");
+  parser->http.request->path = buf2str(beg, (size_t)(cur - beg - 1));
+
+  if (cur == end || *cur++ != 'H') return GRPC_ERROR_CREATE("Expected 'H'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'T') return GRPC_ERROR_CREATE("Expected 'T'");
+  if (cur == end || *cur++ != 'P') return GRPC_ERROR_CREATE("Expected 'P'");
+  if (cur == end || *cur++ != '/') return GRPC_ERROR_CREATE("Expected '/'");
+  vers_major = (uint8_t)(*cur++ - '1' + 1);
+  ++cur;
+  if (cur == end)
+    return GRPC_ERROR_CREATE("End of line in HTTP version string");
+  vers_minor = (uint8_t)(*cur++ - '1' + 1);
+
+  if (vers_major == 1) {
+    if (vers_minor == 0) {
+      parser->http.request->version = GRPC_HTTP_HTTP10;
+    } else if (vers_minor == 1) {
+      parser->http.request->version = GRPC_HTTP_HTTP11;
+    } else {
+      return GRPC_ERROR_CREATE(
+          "Expected one of HTTP/1.0, HTTP/1.1, or HTTP/2.0");
+    }
+  } else if (vers_major == 2) {
+    if (vers_minor == 0) {
+      parser->http.request->version = GRPC_HTTP_HTTP20;
+    } else {
+      return GRPC_ERROR_CREATE(
+          "Expected one of HTTP/1.0, HTTP/1.1, or HTTP/2.0");
+    }
+  } else {
+    return GRPC_ERROR_CREATE("Expected one of HTTP/1.0, HTTP/1.1, or HTTP/2.0");
+  }
+
+  return GRPC_ERROR_NONE;
+}
+
+static grpc_error *handle_first_line(grpc_http_parser *parser) {
+  switch (parser->type) {
+    case GRPC_HTTP_REQUEST:
+      return handle_request_line(parser);
+    case GRPC_HTTP_RESPONSE:
+      return handle_response_line(parser);
+  }
+  GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
+}
+
+static grpc_error *add_header(grpc_http_parser *parser) {
+  uint8_t *beg = parser->cur_line;
+  uint8_t *cur = beg;
+  uint8_t *end = beg + parser->cur_line_length;
+  size_t *hdr_count = NULL;
+  grpc_http_header **hdrs = NULL;
+  grpc_http_header hdr = {NULL, NULL};
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  GPR_ASSERT(cur != end);
+
+  if (*cur == ' ' || *cur == '\t') {
+    error = GRPC_ERROR_CREATE("Continued header lines not supported yet");
+    goto done;
+  }
+
+  while (cur != end && *cur != ':') {
+    cur++;
+  }
+  if (cur == end) {
+<<<<<<< HEAD
+    error = GRPC_ERROR_CREATE("Didn't find ':' in header string");
+    goto done;
+=======
+    if (grpc_http1_trace) {
+      gpr_log(GPR_ERROR, "Didn't find ':' in header string");
+    }
+    goto error;
+>>>>>>> a709afe241d8b264a1c326315f757b4a8d330207
+  }
+  GPR_ASSERT(cur >= beg);
+  hdr.key = buf2str(beg, (size_t)(cur - beg));
+  cur++; /* skip : */
+
+  while (cur != end && (*cur == ' ' || *cur == '\t')) {
+    cur++;
+  }
+  GPR_ASSERT((size_t)(end - cur) >= parser->cur_line_end_length);
+  hdr.value = buf2str(cur, (size_t)(end - cur) - parser->cur_line_end_length);
+
+  switch (parser->type) {
+    case GRPC_HTTP_RESPONSE:
+      hdr_count = &parser->http.response->hdr_count;
+      hdrs = &parser->http.response->hdrs;
+      break;
+    case GRPC_HTTP_REQUEST:
+      hdr_count = &parser->http.request->hdr_count;
+      hdrs = &parser->http.request->hdrs;
+      break;
+  }
+
+  if (*hdr_count == parser->hdr_capacity) {
+    parser->hdr_capacity =
+        GPR_MAX(parser->hdr_capacity + 1, parser->hdr_capacity * 3 / 2);
+    *hdrs = gpr_realloc(*hdrs, parser->hdr_capacity * sizeof(**hdrs));
+  }
+  (*hdrs)[(*hdr_count)++] = hdr;
+
+done:
+  if (error != GRPC_ERROR_NONE) {
+    gpr_free(hdr.key);
+    gpr_free(hdr.value);
+  }
+  return error;
+}
+
+static grpc_error *finish_line(grpc_http_parser *parser) {
+  grpc_error *err;
+  switch (parser->state) {
+    case GRPC_HTTP_FIRST_LINE:
+      err = handle_first_line(parser);
+      if (err != GRPC_ERROR_NONE) return err;
+      parser->state = GRPC_HTTP_HEADERS;
+      break;
+    case GRPC_HTTP_HEADERS:
+      if (parser->cur_line_length == parser->cur_line_end_length) {
+        parser->state = GRPC_HTTP_BODY;
+        break;
+      }
+      err = add_header(parser);
+      if (err != GRPC_ERROR_NONE) {
+        return err;
+      }
+      break;
+    case GRPC_HTTP_BODY:
+      GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
+  }
+
+  parser->cur_line_length = 0;
+  return GRPC_ERROR_NONE;
+}
+
+static grpc_error *addbyte_body(grpc_http_parser *parser, uint8_t byte) {
+  size_t *body_length = NULL;
+  char **body = NULL;
+
+  if (parser->type == GRPC_HTTP_RESPONSE) {
+    body_length = &parser->http.response->body_length;
+    body = &parser->http.response->body;
+  } else if (parser->type == GRPC_HTTP_REQUEST) {
+    body_length = &parser->http.request->body_length;
+    body = &parser->http.request->body;
+  } else {
+    GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here"));
+  }
+
+  if (*body_length == parser->body_capacity) {
+    parser->body_capacity = GPR_MAX(8, parser->body_capacity * 3 / 2);
+    *body = gpr_realloc((void *)*body, parser->body_capacity);
+  }
+  (*body)[*body_length] = (char)byte;
+  (*body_length)++;
+
+  return GRPC_ERROR_NONE;
+}
+
+static bool check_line(grpc_http_parser *parser) {
+  if (parser->cur_line_length >= 2 &&
+      parser->cur_line[parser->cur_line_length - 2] == '\r' &&
+      parser->cur_line[parser->cur_line_length - 1] == '\n') {
+    return true;
+  }
+
+  // HTTP request with \n\r line termiantors.
+  else if (parser->cur_line_length >= 2 &&
+           parser->cur_line[parser->cur_line_length - 2] == '\n' &&
+           parser->cur_line[parser->cur_line_length - 1] == '\r') {
+    return true;
+  }
+
+  // HTTP request with only \n line terminators.
+  else if (parser->cur_line_length >= 1 &&
+           parser->cur_line[parser->cur_line_length - 1] == '\n') {
+    parser->cur_line_end_length = 1;
+    return true;
+  }
+
+  return false;
+}
+
+static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte) {
+  switch (parser->state) {
+    case GRPC_HTTP_FIRST_LINE:
+    case GRPC_HTTP_HEADERS:
+      if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) {
+        if (grpc_http1_trace)
+          gpr_log(GPR_ERROR, "HTTP client max line length (%d) exceeded",
+                  GRPC_HTTP_PARSER_MAX_HEADER_LENGTH);
+        return 0;
+      }
+      parser->cur_line[parser->cur_line_length] = byte;
+      parser->cur_line_length++;
+      if (check_line(parser)) {
+        return finish_line(parser);
+      } else {
+        return GRPC_ERROR_NONE;
+      }
+      GPR_UNREACHABLE_CODE(return 0);
+    case GRPC_HTTP_BODY:
+      return addbyte_body(parser, byte);
+  }
+  GPR_UNREACHABLE_CODE(return 0);
+}
+
+void grpc_http_parser_init(grpc_http_parser *parser, grpc_http_type type,
+                           void *request_or_response) {
+  memset(parser, 0, sizeof(*parser));
+  parser->state = GRPC_HTTP_FIRST_LINE;
+  parser->type = type;
+  parser->http.request_or_response = request_or_response;
+  parser->cur_line_end_length = 2;
+}
+
+void grpc_http_parser_destroy(grpc_http_parser *parser) {}
+
+void grpc_http_request_destroy(grpc_http_request *request) {
+  size_t i;
+  gpr_free(request->body);
+  for (i = 0; i < request->hdr_count; i++) {
+    gpr_free(request->hdrs[i].key);
+    gpr_free(request->hdrs[i].value);
+  }
+  gpr_free(request->hdrs);
+  gpr_free(request->method);
+  gpr_free(request->path);
+}
+
+void grpc_http_response_destroy(grpc_http_response *response) {
+  size_t i;
+  gpr_free(response->body);
+  for (i = 0; i < response->hdr_count; i++) {
+    gpr_free(response->hdrs[i].key);
+    gpr_free(response->hdrs[i].value);
+  }
+  gpr_free(response->hdrs);
+}
+
+grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice) {
+  size_t i;
+
+  for (i = 0; i < GPR_SLICE_LENGTH(slice); i++) {
+    grpc_error *err = addbyte(parser, GPR_SLICE_START_PTR(slice)[i]);
+    if (err != GRPC_ERROR_NONE) return err;
+  }
+
+  return GRPC_ERROR_NONE;
+}
+
+grpc_error *grpc_http_parser_eof(grpc_http_parser *parser) {
+  if (parser->state != GRPC_HTTP_BODY) {
+    return GRPC_ERROR_CREATE("Did not finish headers");
+  }
+  return GRPC_ERROR_NONE;
+}
diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h
index 536637e9a2..6df3cc8b13 100644
--- a/src/core/lib/http/parser.h
+++ b/src/core/lib/http/parser.h
@@ -36,6 +36,7 @@
 
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
+#include "src/core/lib/iomgr/error.h"
 
 /* Maximum length of a header string of the form 'Key: Value\r\n' */
 #define GRPC_HTTP_PARSER_MAX_HEADER_LENGTH 4096
@@ -61,7 +62,6 @@ typedef enum {
 typedef enum {
   GRPC_HTTP_RESPONSE,
   GRPC_HTTP_REQUEST,
-  GRPC_HTTP_UNKNOWN
 } grpc_http_type;
 
 /* A request */
@@ -97,8 +97,9 @@ typedef struct {
   grpc_http_type type;
 
   union {
-    grpc_http_response response;
-    grpc_http_request request;
+    grpc_http_response *response;
+    grpc_http_request *request;
+    void *request_or_response;
   } http;
   size_t body_capacity;
   size_t hdr_capacity;
@@ -108,11 +109,15 @@ typedef struct {
   size_t cur_line_end_length;
 } grpc_http_parser;
 
-void grpc_http_parser_init(grpc_http_parser *parser);
+void grpc_http_parser_init(grpc_http_parser *parser, grpc_http_type type,
+                           void *request_or_response);
 void grpc_http_parser_destroy(grpc_http_parser *parser);
 
-int grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice);
-int grpc_http_parser_eof(grpc_http_parser *parser);
+grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice);
+grpc_error *grpc_http_parser_eof(grpc_http_parser *parser);
+
+void grpc_http_request_destroy(grpc_http_request *request);
+void grpc_http_response_destroy(grpc_http_response *response);
 
 extern int grpc_http1_trace;
 
diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c
index 27793c32e4..0b6c3b2539 100644
--- a/src/core/lib/iomgr/closure.c
+++ b/src/core/lib/iomgr/closure.c
@@ -39,25 +39,32 @@ void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
                        void *cb_arg) {
   closure->cb = cb;
   closure->cb_arg = cb_arg;
-  closure->final_data = 0;
 }
 
-void grpc_closure_list_add(grpc_closure_list *closure_list,
-                           grpc_closure *closure, bool success) {
-  if (closure == NULL) return;
-  closure->final_data = (success != 0);
+void grpc_closure_list_append(grpc_closure_list *closure_list,
+                              grpc_closure *closure, grpc_error *error) {
+  if (closure == NULL) {
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
+  closure->error = error;
+  closure->next_data.next = NULL;
   if (closure_list->head == NULL) {
     closure_list->head = closure;
   } else {
-    closure_list->tail->final_data |= (uintptr_t)closure;
+    closure_list->tail->next_data.next = closure;
   }
   closure_list->tail = closure;
 }
 
-void grpc_closure_list_fail_all(grpc_closure_list *list) {
-  for (grpc_closure *c = list->head; c != NULL; c = grpc_closure_next(c)) {
-    c->final_data &= ~(uintptr_t)1;
+void grpc_closure_list_fail_all(grpc_closure_list *list,
+                                grpc_error *forced_failure) {
+  for (grpc_closure *c = list->head; c != NULL; c = c->next_data.next) {
+    if (c->error == GRPC_ERROR_NONE) {
+      c->error = GRPC_ERROR_REF(forced_failure);
+    }
   }
+  GRPC_ERROR_UNREF(forced_failure);
 }
 
 bool grpc_closure_list_empty(grpc_closure_list closure_list) {
@@ -71,7 +78,7 @@ void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst) {
   if (dst->head == NULL) {
     *dst = *src;
   } else {
-    dst->tail->final_data |= (uintptr_t)src->head;
+    dst->tail->next_data.next = src->head;
     dst->tail = src->tail;
   }
   src->head = src->tail = NULL;
@@ -83,12 +90,13 @@ typedef struct {
   grpc_closure wrapper;
 } wrapped_closure;
 
-static void closure_wrapper(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void closure_wrapper(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
   wrapped_closure *wc = arg;
   grpc_iomgr_cb_func cb = wc->cb;
   void *cb_arg = wc->cb_arg;
   gpr_free(wc);
-  cb(exec_ctx, cb_arg, success);
+  cb(exec_ctx, cb_arg, error);
 }
 
 grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg) {
@@ -98,7 +106,3 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg) {
   grpc_closure_init(&wc->wrapper, closure_wrapper, wc);
   return &wc->wrapper;
 }
-
-grpc_closure *grpc_closure_next(grpc_closure *closure) {
-  return (grpc_closure *)(closure->final_data & ~(uintptr_t)1);
-}
diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h
index fdc2daed9d..08e59a168e 100644
--- a/src/core/lib/iomgr/closure.h
+++ b/src/core/lib/iomgr/closure.h
@@ -36,6 +36,7 @@
 
 #include <grpc/support/port_platform.h>
 #include <stdbool.h>
+#include "src/core/lib/iomgr/error.h"
 
 struct grpc_closure;
 typedef struct grpc_closure grpc_closure;
@@ -52,10 +53,10 @@ typedef struct grpc_closure_list {
 /** gRPC Callback definition.
  *
  * \param arg Arbitrary input.
- * \param success An indication on the state of the iomgr. On false, cleanup
- * actions should be taken (eg, shutdown). */
+ * \param error GRPC_ERROR_NONE if no error occurred, otherwise some grpc_error
+ *              describing what went wrong */
 typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg,
-                                   bool success);
+                                   grpc_error *error);
 
 /** A closure over a grpc_iomgr_cb_func. */
 struct grpc_closure {
@@ -65,10 +66,15 @@ struct grpc_closure {
   /** Arguments to be passed to "cb". */
   void *cb_arg;
 
-  /** Once enqueued, contains in the lower bit the success of the closure,
-      and in the upper bits the pointer to the next closure in the list.
-      Before enqueing for execution, this is usable for scratch data. */
-  uintptr_t final_data;
+  /** Once queued, the result of the closure. Before then: scratch space */
+  grpc_error *error;
+
+  /** Once queued, next indicates the next queued closure; before then, scratch
+   *  space */
+  union {
+    grpc_closure *next;
+    uintptr_t scratch;
+  } next_data;
 };
 
 /** Initializes \a closure with \a cb and \a cb_arg. */
@@ -81,13 +87,14 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg);
 #define GRPC_CLOSURE_LIST_INIT \
   { NULL, NULL }
 
-/** add \a closure to the end of \a list and set \a closure's success to \a
- * success */
-void grpc_closure_list_add(grpc_closure_list *list, grpc_closure *closure,
-                           bool success);
+/** add \a closure to the end of \a list
+    and set \a closure's result to \a error */
+void grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure,
+                              grpc_error *error);
 
 /** force all success bits in \a list to false */
-void grpc_closure_list_fail_all(grpc_closure_list *list);
+void grpc_closure_list_fail_all(grpc_closure_list *list,
+                                grpc_error *forced_failure);
 
 /** append all closures from \a src to \a dst and empty \a src. */
 void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst);
@@ -95,7 +102,4 @@ void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst);
 /** return whether \a list is empty. */
 bool grpc_closure_list_empty(grpc_closure_list list);
 
-/** return the next pointer for a queued closure list */
-grpc_closure *grpc_closure_next(grpc_closure *closure);
-
 #endif /* GRPC_CORE_LIB_IOMGR_CLOSURE_H */
diff --git a/src/core/lib/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h
index 3877ceb1e2..f9808bbda1 100644
--- a/src/core/lib/iomgr/endpoint.h
+++ b/src/core/lib/iomgr/endpoint.h
@@ -82,7 +82,7 @@ char *grpc_endpoint_get_peer(grpc_endpoint *ep);
 void grpc_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                          gpr_slice_buffer *slices, grpc_closure *cb);
 
-/* Causes any pending read/write callbacks to run immediately with
+/* Causes any pending and future read/write callbacks to run immediately with
    success==0 */
 void grpc_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
 void grpc_endpoint_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
diff --git a/src/core/lib/iomgr/endpoint_pair_posix.c b/src/core/lib/iomgr/endpoint_pair_posix.c
index e0ce47c773..e295fb4867 100644
--- a/src/core/lib/iomgr/endpoint_pair_posix.c
+++ b/src/core/lib/iomgr/endpoint_pair_posix.c
@@ -58,8 +58,8 @@ static void create_sockets(int sv[2]) {
   GPR_ASSERT(fcntl(sv[0], F_SETFL, flags | O_NONBLOCK) == 0);
   flags = fcntl(sv[1], F_GETFL, 0);
   GPR_ASSERT(fcntl(sv[1], F_SETFL, flags | O_NONBLOCK) == 0);
-  GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[0]));
-  GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[1]));
+  GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[0]) == GRPC_ERROR_NONE);
+  GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[1]) == GRPC_ERROR_NONE);
 }
 
 grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c
new file mode 100644
index 0000000000..e20a0169c5
--- /dev/null
+++ b/src/core/lib/iomgr/error.c
@@ -0,0 +1,578 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/error.h"
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <string.h>
+
+#include <grpc/status.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/avl.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#ifdef GPR_WINDOWS
+#include <grpc/support/log_windows.h>
+#endif
+
+#include "src/core/lib/profiling/timers.h"
+
+static void destroy_integer(void *key) {}
+
+static void *copy_integer(void *key) { return key; }
+
+static long compare_integers(void *key1, void *key2) {
+  return GPR_ICMP((uintptr_t)key1, (uintptr_t)key2);
+}
+
+static void destroy_string(void *str) { gpr_free(str); }
+
+static void *copy_string(void *str) { return gpr_strdup(str); }
+
+static void destroy_err(void *err) { GRPC_ERROR_UNREF(err); }
+
+static void *copy_err(void *err) { return GRPC_ERROR_REF(err); }
+
+static void destroy_time(void *tm) { gpr_free(tm); }
+
+static gpr_timespec *box_time(gpr_timespec tm) {
+  gpr_timespec *out = gpr_malloc(sizeof(*out));
+  *out = tm;
+  return out;
+}
+
+static void *copy_time(void *tm) { return box_time(*(gpr_timespec *)tm); }
+
+static const gpr_avl_vtable avl_vtable_ints = {destroy_integer, copy_integer,
+                                               compare_integers,
+                                               destroy_integer, copy_integer};
+
+static const gpr_avl_vtable avl_vtable_strs = {destroy_integer, copy_integer,
+                                               compare_integers, destroy_string,
+                                               copy_string};
+
+static const gpr_avl_vtable avl_vtable_times = {
+    destroy_integer, copy_integer, compare_integers, destroy_time, copy_time};
+
+static const gpr_avl_vtable avl_vtable_errs = {
+    destroy_integer, copy_integer, compare_integers, destroy_err, copy_err};
+
+static const char *error_int_name(grpc_error_ints key) {
+  switch (key) {
+    case GRPC_ERROR_INT_ERRNO:
+      return "errno";
+    case GRPC_ERROR_INT_FILE_LINE:
+      return "file_line";
+    case GRPC_ERROR_INT_STREAM_ID:
+      return "stream_id";
+    case GRPC_ERROR_INT_GRPC_STATUS:
+      return "grpc_status";
+    case GRPC_ERROR_INT_OFFSET:
+      return "offset";
+    case GRPC_ERROR_INT_INDEX:
+      return "index";
+    case GRPC_ERROR_INT_SIZE:
+      return "size";
+    case GRPC_ERROR_INT_HTTP2_ERROR:
+      return "http2_error";
+    case GRPC_ERROR_INT_TSI_CODE:
+      return "tsi_code";
+    case GRPC_ERROR_INT_SECURITY_STATUS:
+      return "security_status";
+    case GRPC_ERROR_INT_FD:
+      return "fd";
+    case GRPC_ERROR_INT_WSA_ERROR:
+      return "wsa_error";
+    case GRPC_ERROR_INT_HTTP_STATUS:
+      return "http_status";
+    case GRPC_ERROR_INT_LIMIT:
+      return "limit";
+  }
+  GPR_UNREACHABLE_CODE(return "unknown");
+}
+
+static const char *error_str_name(grpc_error_strs key) {
+  switch (key) {
+    case GRPC_ERROR_STR_DESCRIPTION:
+      return "description";
+    case GRPC_ERROR_STR_OS_ERROR:
+      return "os_error";
+    case GRPC_ERROR_STR_TARGET_ADDRESS:
+      return "target_address";
+    case GRPC_ERROR_STR_SYSCALL:
+      return "syscall";
+    case GRPC_ERROR_STR_FILE:
+      return "file";
+    case GRPC_ERROR_STR_GRPC_MESSAGE:
+      return "grpc_message";
+    case GRPC_ERROR_STR_RAW_BYTES:
+      return "raw_bytes";
+    case GRPC_ERROR_STR_TSI_ERROR:
+      return "tsi_error";
+    case GRPC_ERROR_STR_FILENAME:
+      return "filename";
+  }
+  GPR_UNREACHABLE_CODE(return "unknown");
+}
+
+static const char *error_time_name(grpc_error_times key) {
+  switch (key) {
+    case GRPC_ERROR_TIME_CREATED:
+      return "created";
+  }
+  GPR_UNREACHABLE_CODE(return "unknown");
+}
+
+struct grpc_error {
+  gpr_refcount refs;
+  gpr_avl ints;
+  gpr_avl strs;
+  gpr_avl times;
+  gpr_avl errs;
+  uintptr_t next_err;
+};
+
+static bool is_special(grpc_error *err) {
+  return err == GRPC_ERROR_NONE || err == GRPC_ERROR_OOM ||
+         err == GRPC_ERROR_CANCELLED;
+}
+
+#ifdef GRPC_ERROR_REFCOUNT_DEBUG
+grpc_error *grpc_error_ref(grpc_error *err, const char *file, int line,
+                           const char *func) {
+  if (is_special(err)) return err;
+  gpr_log(GPR_DEBUG, "%p: %d -> %d [%s:%d %s]", err, err->refs.count,
+          err->refs.count + 1, file, line, func);
+  gpr_ref(&err->refs);
+  return err;
+}
+#else
+grpc_error *grpc_error_ref(grpc_error *err) {
+  if (is_special(err)) return err;
+  gpr_ref(&err->refs);
+  return err;
+}
+#endif
+
+static void error_destroy(grpc_error *err) {
+  GPR_ASSERT(!is_special(err));
+  gpr_avl_unref(err->ints);
+  gpr_avl_unref(err->strs);
+  gpr_avl_unref(err->errs);
+  gpr_avl_unref(err->times);
+  gpr_free(err);
+}
+
+#ifdef GRPC_ERROR_REFCOUNT_DEBUG
+void grpc_error_unref(grpc_error *err, const char *file, int line,
+                      const char *func) {
+  if (is_special(err)) return;
+  gpr_log(GPR_DEBUG, "%p: %d -> %d [%s:%d %s]", err, err->refs.count,
+          err->refs.count - 1, file, line, func);
+  if (gpr_unref(&err->refs)) {
+    error_destroy(err);
+  }
+}
+#else
+void grpc_error_unref(grpc_error *err) {
+  if (is_special(err)) return;
+  if (gpr_unref(&err->refs)) {
+    error_destroy(err);
+  }
+}
+#endif
+
+grpc_error *grpc_error_create(const char *file, int line, const char *desc,
+                              grpc_error **referencing,
+                              size_t num_referencing) {
+  GPR_TIMER_BEGIN("grpc_error_create", 0);
+  grpc_error *err = gpr_malloc(sizeof(*err));
+  if (err == NULL) {  // TODO(ctiller): make gpr_malloc return NULL
+    return GRPC_ERROR_OOM;
+  }
+#ifdef GRPC_ERROR_REFCOUNT_DEBUG
+  gpr_log(GPR_DEBUG, "%p create [%s:%d]", err, file, line);
+#endif
+  err->ints = gpr_avl_add(gpr_avl_create(&avl_vtable_ints),
+                          (void *)(uintptr_t)GRPC_ERROR_INT_FILE_LINE,
+                          (void *)(uintptr_t)line);
+  err->strs = gpr_avl_add(
+      gpr_avl_add(gpr_avl_create(&avl_vtable_strs),
+                  (void *)(uintptr_t)GRPC_ERROR_STR_FILE, gpr_strdup(file)),
+      (void *)(uintptr_t)GRPC_ERROR_STR_DESCRIPTION, gpr_strdup(desc));
+  err->errs = gpr_avl_create(&avl_vtable_errs);
+  err->next_err = 0;
+  for (size_t i = 0; i < num_referencing; i++) {
+    if (referencing[i] == GRPC_ERROR_NONE) continue;
+    err->errs = gpr_avl_add(err->errs, (void *)(err->next_err++),
+                            GRPC_ERROR_REF(referencing[i]));
+  }
+  err->times = gpr_avl_add(gpr_avl_create(&avl_vtable_times),
+                           (void *)(uintptr_t)GRPC_ERROR_TIME_CREATED,
+                           box_time(gpr_now(GPR_CLOCK_REALTIME)));
+  gpr_ref_init(&err->refs, 1);
+  GPR_TIMER_END("grpc_error_create", 0);
+  return err;
+}
+
+static grpc_error *copy_error_and_unref(grpc_error *in) {
+  GPR_TIMER_BEGIN("copy_error_and_unref", 0);
+  grpc_error *out;
+  if (is_special(in)) {
+    if (in == GRPC_ERROR_NONE)
+      out = GRPC_ERROR_CREATE("no error");
+    else if (in == GRPC_ERROR_OOM)
+      out = GRPC_ERROR_CREATE("oom");
+    else if (in == GRPC_ERROR_CANCELLED)
+      out =
+          grpc_error_set_int(GRPC_ERROR_CREATE("cancelled"),
+                             GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED);
+    else
+      out = GRPC_ERROR_CREATE("unknown");
+  } else {
+    out = gpr_malloc(sizeof(*out));
+#ifdef GRPC_ERROR_REFCOUNT_DEBUG
+    gpr_log(GPR_DEBUG, "%p create copying", out);
+#endif
+    out->ints = gpr_avl_ref(in->ints);
+    out->strs = gpr_avl_ref(in->strs);
+    out->errs = gpr_avl_ref(in->errs);
+    out->times = gpr_avl_ref(in->times);
+    out->next_err = in->next_err;
+    gpr_ref_init(&out->refs, 1);
+    GRPC_ERROR_UNREF(in);
+  }
+  GPR_TIMER_END("copy_error_and_unref", 0);
+  return out;
+}
+
+grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which,
+                               intptr_t value) {
+  GPR_TIMER_BEGIN("grpc_error_set_int", 0);
+  grpc_error *new = copy_error_and_unref(src);
+  new->ints = gpr_avl_add(new->ints, (void *)(uintptr_t)which, (void *)value);
+  GPR_TIMER_END("grpc_error_set_int", 0);
+  return new;
+}
+
+bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) {
+  GPR_TIMER_BEGIN("grpc_error_get_int", 0);
+  void *pp;
+  if (is_special(err)) {
+    if (err == GRPC_ERROR_CANCELLED && which == GRPC_ERROR_INT_GRPC_STATUS) {
+      *p = GRPC_STATUS_CANCELLED;
+      GPR_TIMER_END("grpc_error_get_int", 0);
+      return true;
+    }
+    GPR_TIMER_END("grpc_error_get_int", 0);
+    return false;
+  }
+  if (gpr_avl_maybe_get(err->ints, (void *)(uintptr_t)which, &pp)) {
+    if (p != NULL) *p = (intptr_t)pp;
+    GPR_TIMER_END("grpc_error_get_int", 0);
+    return true;
+  }
+  GPR_TIMER_END("grpc_error_get_int", 0);
+  return false;
+}
+
+grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which,
+                               const char *value) {
+  GPR_TIMER_BEGIN("grpc_error_set_str", 0);
+  grpc_error *new = copy_error_and_unref(src);
+  new->strs =
+      gpr_avl_add(new->strs, (void *)(uintptr_t)which, gpr_strdup(value));
+  GPR_TIMER_END("grpc_error_set_str", 0);
+  return new;
+}
+
+const char *grpc_error_get_str(grpc_error *err, grpc_error_strs which) {
+  if (is_special(err)) return NULL;
+  return gpr_avl_get(err->strs, (void *)(uintptr_t)which);
+}
+
+grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) {
+  GPR_TIMER_BEGIN("grpc_error_add_child", 0);
+  grpc_error *new = copy_error_and_unref(src);
+  new->errs = gpr_avl_add(new->errs, (void *)(new->next_err++), child);
+  GPR_TIMER_END("grpc_error_add_child", 0);
+  return new;
+}
+
+static const char *no_error_string = "null";
+static const char *oom_error_string = "\"Out of memory\"";
+static const char *cancelled_error_string = "\"Cancelled\"";
+
+typedef struct {
+  char *key;
+  char *value;
+} kv_pair;
+
+typedef struct {
+  kv_pair *kvs;
+  size_t num_kvs;
+  size_t cap_kvs;
+} kv_pairs;
+
+static void append_kv(kv_pairs *kvs, char *key, char *value) {
+  if (kvs->num_kvs == kvs->cap_kvs) {
+    kvs->cap_kvs = GPR_MAX(3 * kvs->cap_kvs / 2, 4);
+    kvs->kvs = gpr_realloc(kvs->kvs, sizeof(*kvs->kvs) * kvs->cap_kvs);
+  }
+  kvs->kvs[kvs->num_kvs].key = key;
+  kvs->kvs[kvs->num_kvs].value = value;
+  kvs->num_kvs++;
+}
+
+static void collect_kvs(gpr_avl_node *node, char *key(void *k),
+                        char *fmt(void *v), kv_pairs *kvs) {
+  if (node == NULL) return;
+  append_kv(kvs, key(node->key), fmt(node->value));
+  collect_kvs(node->left, key, fmt, kvs);
+  collect_kvs(node->right, key, fmt, kvs);
+}
+
+static char *key_int(void *p) {
+  return gpr_strdup(error_int_name((grpc_error_ints)(uintptr_t)p));
+}
+
+static char *key_str(void *p) {
+  return gpr_strdup(error_str_name((grpc_error_strs)(uintptr_t)p));
+}
+
+static char *key_time(void *p) {
+  return gpr_strdup(error_time_name((grpc_error_times)(uintptr_t)p));
+}
+
+static char *fmt_int(void *p) {
+  char *s;
+  gpr_asprintf(&s, "%" PRIdPTR, (intptr_t)p);
+  return s;
+}
+
+static void append_chr(char c, char **s, size_t *sz, size_t *cap) {
+  if (*sz == *cap) {
+    *cap = GPR_MAX(8, 3 * *cap / 2);
+    *s = gpr_realloc(*s, *cap);
+  }
+  (*s)[(*sz)++] = c;
+}
+
+static void append_str(const char *str, char **s, size_t *sz, size_t *cap) {
+  for (const char *c = str; *c; c++) {
+    append_chr(*c, s, sz, cap);
+  }
+}
+
+static void append_esc_str(const char *str, char **s, size_t *sz, size_t *cap) {
+  static const char *hex = "0123456789abcdef";
+  append_chr('"', s, sz, cap);
+  for (const uint8_t *c = (const uint8_t *)str; *c; c++) {
+    if (*c < 32 || *c >= 127) {
+      append_chr('\\', s, sz, cap);
+      switch (*c) {
+        case '\b':
+          append_chr('b', s, sz, cap);
+          break;
+        case '\f':
+          append_chr('f', s, sz, cap);
+          break;
+        case '\n':
+          append_chr('n', s, sz, cap);
+          break;
+        case '\r':
+          append_chr('r', s, sz, cap);
+          break;
+        case '\t':
+          append_chr('t', s, sz, cap);
+          break;
+        default:
+          append_chr('u', s, sz, cap);
+          append_chr('0', s, sz, cap);
+          append_chr('0', s, sz, cap);
+          append_chr(hex[*c >> 4], s, sz, cap);
+          append_chr(hex[*c & 0x0f], s, sz, cap);
+          break;
+      }
+    } else {
+      append_chr((char)*c, s, sz, cap);
+    }
+  }
+  append_chr('"', s, sz, cap);
+}
+
+static char *fmt_str(void *p) {
+  char *s = NULL;
+  size_t sz = 0;
+  size_t cap = 0;
+  append_esc_str(p, &s, &sz, &cap);
+  append_chr(0, &s, &sz, &cap);
+  return s;
+}
+
+static char *fmt_time(void *p) {
+  gpr_timespec tm = *(gpr_timespec *)p;
+  char *out;
+  char *pfx = "!!";
+  switch (tm.clock_type) {
+    case GPR_CLOCK_MONOTONIC:
+      pfx = "@monotonic:";
+      break;
+    case GPR_CLOCK_REALTIME:
+      pfx = "@";
+      break;
+    case GPR_CLOCK_PRECISE:
+      pfx = "@precise:";
+      break;
+    case GPR_TIMESPAN:
+      pfx = "";
+      break;
+  }
+  gpr_asprintf(&out, "\"%s%" PRId64 ".%09d\"", pfx, tm.tv_sec, tm.tv_nsec);
+  return out;
+}
+
+static void add_errs(gpr_avl_node *n, char **s, size_t *sz, size_t *cap) {
+  if (n == NULL) return;
+  add_errs(n->left, s, sz, cap);
+  const char *e = grpc_error_string(n->value);
+  append_str(e, s, sz, cap);
+  grpc_error_free_string(e);
+  add_errs(n->right, s, sz, cap);
+}
+
+static char *errs_string(grpc_error *err) {
+  char *s = NULL;
+  size_t sz = 0;
+  size_t cap = 0;
+  append_chr('[', &s, &sz, &cap);
+  add_errs(err->errs.root, &s, &sz, &cap);
+  append_chr(']', &s, &sz, &cap);
+  append_chr(0, &s, &sz, &cap);
+  return s;
+}
+
+static int cmp_kvs(const void *a, const void *b) {
+  const kv_pair *ka = a;
+  const kv_pair *kb = b;
+  return strcmp(ka->key, kb->key);
+}
+
+static const char *finish_kvs(kv_pairs *kvs) {
+  char *s = NULL;
+  size_t sz = 0;
+  size_t cap = 0;
+
+  append_chr('{', &s, &sz, &cap);
+  for (size_t i = 0; i < kvs->num_kvs; i++) {
+    if (i != 0) append_chr(',', &s, &sz, &cap);
+    append_esc_str(kvs->kvs[i].key, &s, &sz, &cap);
+    gpr_free(kvs->kvs[i].key);
+    append_chr(':', &s, &sz, &cap);
+    append_str(kvs->kvs[i].value, &s, &sz, &cap);
+    gpr_free(kvs->kvs[i].value);
+  }
+  append_chr('}', &s, &sz, &cap);
+  append_chr(0, &s, &sz, &cap);
+
+  gpr_free(kvs->kvs);
+  return s;
+}
+
+void grpc_error_free_string(const char *str) {
+  if (str == no_error_string) return;
+  if (str == oom_error_string) return;
+  if (str == cancelled_error_string) return;
+  gpr_free((char *)str);
+}
+
+const char *grpc_error_string(grpc_error *err) {
+  GPR_TIMER_BEGIN("grpc_error_string", 0);
+  if (err == GRPC_ERROR_NONE) return no_error_string;
+  if (err == GRPC_ERROR_OOM) return oom_error_string;
+  if (err == GRPC_ERROR_CANCELLED) return cancelled_error_string;
+
+  kv_pairs kvs;
+  memset(&kvs, 0, sizeof(kvs));
+
+  collect_kvs(err->ints.root, key_int, fmt_int, &kvs);
+  collect_kvs(err->strs.root, key_str, fmt_str, &kvs);
+  collect_kvs(err->times.root, key_time, fmt_time, &kvs);
+  if (!gpr_avl_is_empty(err->errs)) {
+    append_kv(&kvs, gpr_strdup("referenced_errors"), errs_string(err));
+  }
+
+  qsort(kvs.kvs, kvs.num_kvs, sizeof(kv_pair), cmp_kvs);
+
+  const char *out = finish_kvs(&kvs);
+  GPR_TIMER_END("grpc_error_string", 0);
+  return out;
+}
+
+grpc_error *grpc_os_error(const char *file, int line, int err,
+                          const char *call_name) {
+  return grpc_error_set_str(
+      grpc_error_set_str(
+          grpc_error_set_int(grpc_error_create(file, line, "OS Error", NULL, 0),
+                             GRPC_ERROR_INT_ERRNO, err),
+          GRPC_ERROR_STR_OS_ERROR, strerror(err)),
+      GRPC_ERROR_STR_SYSCALL, call_name);
+}
+
+#ifdef GPR_WINDOWS
+grpc_error *grpc_wsa_error(const char *file, int line, int err,
+                           const char *call_name) {
+  char *utf8_message = gpr_format_message(err);
+  grpc_error *error = grpc_error_set_str(
+      grpc_error_set_str(
+          grpc_error_set_int(grpc_error_create(file, line, "OS Error", NULL, 0),
+                             GRPC_ERROR_INT_WSA_ERROR, err),
+          GRPC_ERROR_STR_OS_ERROR, utf8_message),
+      GRPC_ERROR_STR_SYSCALL, call_name);
+  gpr_free(utf8_message);
+  return error;
+}
+#endif
+
+bool grpc_log_if_error(const char *what, grpc_error *error, const char *file,
+                       int line) {
+  if (error == GRPC_ERROR_NONE) return true;
+  const char *msg = grpc_error_string(error);
+  gpr_log(file, line, GPR_LOG_SEVERITY_ERROR, "%s: %s", what, msg);
+  grpc_error_free_string(msg);
+  GRPC_ERROR_UNREF(error);
+  return false;
+}
diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h
new file mode 100644
index 0000000000..13f898e31a
--- /dev/null
+++ b/src/core/lib/iomgr/error.h
@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_ERROR_H
+#define GRPC_CORE_LIB_IOMGR_ERROR_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include <grpc/support/time.h>
+
+/// Opaque representation of an error.
+/// Errors are refcounted objects that represent the result of an operation.
+/// Ownership laws:
+///  if a grpc_error is returned by a function, the caller owns a ref to that
+///    instance
+///  if a grpc_error is passed to a grpc_closure callback function (functions
+///    with the signature:
+///      void (*f)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error))
+///    then those functions do not automatically own a ref to error
+///  if a grpc_error is passed to *ANY OTHER FUNCTION* then that function takes
+///    ownership of the error
+/// Errors have:
+///  a set of ints, strings, and timestamps that describe the error
+///  always present are:
+///    GRPC_ERROR_STR_FILE, GRPC_ERROR_INT_FILE_LINE - source location the error
+///      was generated
+///    GRPC_ERROR_STR_DESCRIPTION - a human readable description of the error
+///    GRPC_ERROR_TIME_CREATED - a timestamp indicating when the error happened
+///  an error can also have children; these are other errors that are believed
+///    to have contributed to this one. By accumulating children, we can begin
+///    to root cause high level failures from low level failures, without having
+///    to derive execution paths from log lines
+typedef struct grpc_error grpc_error;
+
+typedef enum {
+  /// 'errno' from the operating system
+  GRPC_ERROR_INT_ERRNO,
+  /// __LINE__ from the call site creating the error
+  GRPC_ERROR_INT_FILE_LINE,
+  /// stream identifier: for errors that are associated with an individual
+  /// wire stream
+  GRPC_ERROR_INT_STREAM_ID,
+  /// grpc status code representing this error
+  GRPC_ERROR_INT_GRPC_STATUS,
+  /// offset into some binary blob (usually represented by
+  /// GRPC_ERROR_STR_RAW_BYTES) where the error occurred
+  GRPC_ERROR_INT_OFFSET,
+  /// context sensitive index associated with the error
+  GRPC_ERROR_INT_INDEX,
+  /// context sensitive size associated with the error
+  GRPC_ERROR_INT_SIZE,
+  /// http2 error code associated with the error (see the HTTP2 RFC)
+  GRPC_ERROR_INT_HTTP2_ERROR,
+  /// TSI status code associated with the error
+  GRPC_ERROR_INT_TSI_CODE,
+  /// grpc_security_status associated with the error
+  GRPC_ERROR_INT_SECURITY_STATUS,
+  /// WSAGetLastError() reported when this error occurred
+  GRPC_ERROR_INT_WSA_ERROR,
+  /// File descriptor associated with this error
+  GRPC_ERROR_INT_FD,
+  /// HTTP status (i.e. 404)
+  GRPC_ERROR_INT_HTTP_STATUS,
+  /// context sensitive limit associated with the error
+  GRPC_ERROR_INT_LIMIT,
+} grpc_error_ints;
+
+typedef enum {
+  /// top-level textual description of this error
+  GRPC_ERROR_STR_DESCRIPTION,
+  /// source file in which this error occurred
+  GRPC_ERROR_STR_FILE,
+  /// operating system description of this error
+  GRPC_ERROR_STR_OS_ERROR,
+  /// syscall that generated this error
+  GRPC_ERROR_STR_SYSCALL,
+  /// peer that we were trying to communicate when this error occurred
+  GRPC_ERROR_STR_TARGET_ADDRESS,
+  /// grpc status message associated with this error
+  GRPC_ERROR_STR_GRPC_MESSAGE,
+  /// hex dump (or similar) with the data that generated this error
+  GRPC_ERROR_STR_RAW_BYTES,
+  /// tsi error string associated with this error
+  GRPC_ERROR_STR_TSI_ERROR,
+  /// filename that we were trying to read/write when this error occurred
+  GRPC_ERROR_STR_FILENAME,
+} grpc_error_strs;
+
+typedef enum {
+  /// timestamp of error creation
+  GRPC_ERROR_TIME_CREATED,
+} grpc_error_times;
+
+#define GRPC_ERROR_NONE ((grpc_error *)NULL)
+#define GRPC_ERROR_OOM ((grpc_error *)1)
+#define GRPC_ERROR_CANCELLED ((grpc_error *)2)
+
+const char *grpc_error_string(grpc_error *error);
+void grpc_error_free_string(const char *str);
+
+/// Create an error - but use GRPC_ERROR_CREATE instead
+grpc_error *grpc_error_create(const char *file, int line, const char *desc,
+                              grpc_error **referencing, size_t num_referencing);
+/// Create an error (this is the preferred way of generating an error that is
+///   not due to a system call - for system calls, use GRPC_OS_ERROR or
+///   GRPC_WSA_ERROR as appropriate)
+/// \a referencing is an array of num_referencing elements indicating one or
+/// more errors that are believed to have contributed to this one
+/// err = grpc_error_create(x, y, z, r, nr) is equivalent to:
+///   err = grpc_error_create(x, y, z, NULL, 0);
+///   for (i=0; i<nr; i++) err = grpc_error_add_child(err, r[i]);
+#define GRPC_ERROR_CREATE(desc) \
+  grpc_error_create(__FILE__, __LINE__, desc, NULL, 0)
+
+// Create an error that references some other errors. This function adds a
+// reference to each error in errs - it does not consume an existing reference
+#define GRPC_ERROR_CREATE_REFERENCING(desc, errs, count) \
+  grpc_error_create(__FILE__, __LINE__, desc, errs, count)
+
+//#define GRPC_ERROR_REFCOUNT_DEBUG
+#ifdef GRPC_ERROR_REFCOUNT_DEBUG
+grpc_error *grpc_error_ref(grpc_error *err, const char *file, int line,
+                           const char *func);
+void grpc_error_unref(grpc_error *err, const char *file, int line,
+                      const char *func);
+#define GRPC_ERROR_REF(err) grpc_error_ref(err, __FILE__, __LINE__, __func__)
+#define GRPC_ERROR_UNREF(err) \
+  grpc_error_unref(err, __FILE__, __LINE__, __func__)
+#else
+grpc_error *grpc_error_ref(grpc_error *err);
+void grpc_error_unref(grpc_error *err);
+#define GRPC_ERROR_REF(err) grpc_error_ref(err)
+#define GRPC_ERROR_UNREF(err) grpc_error_unref(err)
+#endif
+
+grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which,
+                               intptr_t value) GRPC_MUST_USE_RESULT;
+bool grpc_error_get_int(grpc_error *error, grpc_error_ints which, intptr_t *p);
+grpc_error *grpc_error_set_time(grpc_error *src, grpc_error_times which,
+                                gpr_timespec value) GRPC_MUST_USE_RESULT;
+grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which,
+                               const char *value) GRPC_MUST_USE_RESULT;
+const char *grpc_error_get_str(grpc_error *error, grpc_error_strs which);
+/// Add a child error: an error that is believed to have contributed to this
+/// error occurring. Allows root causing high level errors from lower level
+/// errors that contributed to them.
+grpc_error *grpc_error_add_child(grpc_error *src,
+                                 grpc_error *child) GRPC_MUST_USE_RESULT;
+grpc_error *grpc_os_error(const char *file, int line, int err,
+                          const char *call_name) GRPC_MUST_USE_RESULT;
+/// create an error associated with errno!=0 (an 'operating system' error)
+#define GRPC_OS_ERROR(err, call_name) \
+  grpc_os_error(__FILE__, __LINE__, err, call_name)
+grpc_error *grpc_wsa_error(const char *file, int line, int err,
+                           const char *call_name) GRPC_MUST_USE_RESULT;
+/// windows only: create an error associated with WSAGetLastError()!=0
+#define GRPC_WSA_ERROR(err, call_name) \
+  grpc_wsa_error(__FILE__, __LINE__, err, call_name)
+
+bool grpc_log_if_error(const char *what, grpc_error *error, const char *file,
+                       int line);
+#define GRPC_LOG_IF_ERROR(what, error) \
+  grpc_log_if_error((what), (error), __FILE__, __LINE__)
+
+#endif /* GRPC_CORE_LIB_IOMGR_ERROR_H */
diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c
new file mode 100644
index 0000000000..cf0fe736a0
--- /dev/null
+++ b/src/core/lib/iomgr/ev_epoll_linux.c
@@ -0,0 +1,1872 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc_posix.h>
+#include <grpc/support/port_platform.h>
+
+/* This polling engine is only relevant on linux kernels supporting epoll() */
+#ifdef GPR_LINUX_EPOLL
+
+#include "src/core/lib/iomgr/ev_epoll_linux.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <poll.h>
+#include <signal.h>
+#include <string.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/tls.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/block_annotate.h"
+
+/* TODO: sreek - Move this to init.c and initialize this like other tracers. */
+static int grpc_polling_trace = 0; /* Disabled by default */
+#define GRPC_POLLING_TRACE(fmt, ...)       \
+  if (grpc_polling_trace) {                \
+    gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \
+  }
+
+static int grpc_wakeup_signal = -1;
+static bool is_grpc_wakeup_signal_initialized = false;
+
+/* Implements the function defined in grpc_posix.h. This function might be
+ * called before even calling grpc_init() to set either a different signal to
+ * use. If signum == -1, then the use of signals is disabled */
+void grpc_use_signal(int signum) {
+  grpc_wakeup_signal = signum;
+  is_grpc_wakeup_signal_initialized = true;
+
+  if (grpc_wakeup_signal < 0) {
+    gpr_log(GPR_INFO,
+            "Use of signals is disabled. Epoll engine will not be used");
+  } else {
+    gpr_log(GPR_INFO, "epoll engine will be using signal: %d",
+            grpc_wakeup_signal);
+  }
+}
+
+struct polling_island;
+
+/*******************************************************************************
+ * Fd Declarations
+ */
+struct grpc_fd {
+  int fd;
+  /* refst format:
+       bit 0    : 1=Active / 0=Orphaned
+       bits 1-n : refcount
+     Ref/Unref by two to avoid altering the orphaned bit */
+  gpr_atm refst;
+
+  gpr_mu mu;
+
+  /* Indicates that the fd is shutdown and that any pending read/write closures
+     should fail */
+  bool shutdown;
+
+  /* The fd is either closed or we relinquished control of it. In either cases,
+     this indicates that the 'fd' on this structure is no longer valid */
+  bool orphaned;
+
+  /* TODO: sreek - Move this to a lockfree implementation */
+  grpc_closure *read_closure;
+  grpc_closure *write_closure;
+
+  /* The polling island to which this fd belongs to and the mutex protecting the
+     the field */
+  gpr_mu pi_mu;
+  struct polling_island *polling_island;
+
+  struct grpc_fd *freelist_next;
+  grpc_closure *on_done_closure;
+
+  /* The pollset that last noticed that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
+
+  grpc_iomgr_object iomgr_object;
+};
+
+/* Reference counting for fds */
+// #define GRPC_FD_REF_COUNT_DEBUG
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line);
+static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
+                     int line);
+#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__)
+#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__)
+#else
+static void fd_ref(grpc_fd *fd);
+static void fd_unref(grpc_fd *fd);
+#define GRPC_FD_REF(fd, reason) fd_ref(fd)
+#define GRPC_FD_UNREF(fd, reason) fd_unref(fd)
+#endif
+
+static void fd_global_init(void);
+static void fd_global_shutdown(void);
+
+#define CLOSURE_NOT_READY ((grpc_closure *)0)
+#define CLOSURE_READY ((grpc_closure *)1)
+
+/*******************************************************************************
+ * Polling island Declarations
+ */
+
+// #define GRPC_PI_REF_COUNT_DEBUG
+#ifdef GRPC_PI_REF_COUNT_DEBUG
+
+#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__)
+#define PI_UNREF(p, r) pi_unref_dbg((p), (r), __FILE__, __LINE__)
+
+#else /* defined(GRPC_PI_REF_COUNT_DEBUG) */
+
+#define PI_ADD_REF(p, r) pi_add_ref((p))
+#define PI_UNREF(p, r) pi_unref((p))
+
+#endif /* !defined(GPRC_PI_REF_COUNT_DEBUG) */
+
+typedef struct polling_island {
+  gpr_mu mu;
+  /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement
+     the refcount.
+     Once the ref count becomes zero, this structure is destroyed which means
+     we should ensure that there is never a scenario where a PI_ADD_REF() is
+     racing with a PI_UNREF() that just made the ref_count zero. */
+  gpr_refcount ref_count;
+
+  /* Pointer to the polling_island this merged into.
+   * merged_to value is only set once in polling_island's lifetime (and that too
+   * only if the island is merged with another island). Because of this, we can
+   * use gpr_atm type here so that we can do atomic access on this and reduce
+   * lock contention on 'mu' mutex.
+   *
+   * Note that if this field is not NULL (i.e not 0), all the remaining fields
+   * (except mu and ref_count) are invalid and must be ignored. */
+  gpr_atm merged_to;
+
+  /* The fd of the underlying epoll set */
+  int epoll_fd;
+
+  /* The file descriptors in the epoll set */
+  size_t fd_cnt;
+  size_t fd_capacity;
+  grpc_fd **fds;
+
+  /* Polling islands that are no longer needed are kept in a freelist so that
+     they can be reused. This field points to the next polling island in the
+     free list */
+  struct polling_island *next_free;
+} polling_island;
+
+/*******************************************************************************
+ * Pollset Declarations
+ */
+struct grpc_pollset_worker {
+  /* Thread id of this worker */
+  pthread_t pt_id;
+
+  /* Used to prevent a worker from getting kicked multiple times */
+  gpr_atm is_kicked;
+  struct grpc_pollset_worker *next;
+  struct grpc_pollset_worker *prev;
+};
+
+struct grpc_pollset {
+  gpr_mu mu;
+  grpc_pollset_worker root_worker;
+  bool kicked_without_pollers;
+
+  bool shutting_down;          /* Is the pollset shutting down ? */
+  bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */
+  grpc_closure *shutdown_done; /* Called after after shutdown is complete */
+
+  /* The polling island to which this pollset belongs to */
+  struct polling_island *polling_island;
+};
+
+/*******************************************************************************
+ * Pollset-set Declarations
+ */
+/* TODO: sreek - Change the pollset_set implementation such that a pollset_set
+ * directly points to a polling_island (and adding an fd/pollset/pollset_set to
+ * the current pollset_set would result in polling island merges. This would
+ * remove the need to maintain fd_count here. This will also significantly
+ * simplify the grpc_fd structure since we would no longer need to explicitly
+ * maintain the orphaned state */
+struct grpc_pollset_set {
+  gpr_mu mu;
+
+  size_t pollset_count;
+  size_t pollset_capacity;
+  grpc_pollset **pollsets;
+
+  size_t pollset_set_count;
+  size_t pollset_set_capacity;
+  struct grpc_pollset_set **pollset_sets;
+
+  size_t fd_count;
+  size_t fd_capacity;
+  grpc_fd **fds;
+};
+
+/*******************************************************************************
+ * Common helpers
+ */
+
+static void append_error(grpc_error **composite, grpc_error *error,
+                         const char *desc) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE(desc);
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+/*******************************************************************************
+ * Polling island Definitions
+ */
+
+/* The wakeup fd that is used to wake up all threads in a Polling island. This
+   is useful in the polling island merge operation where we need to wakeup all
+   the threads currently polling the smaller polling island (so that they can
+   start polling the new/merged polling island)
+
+   NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the
+   threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */
+static grpc_wakeup_fd polling_island_wakeup_fd;
+
+/* Polling island freelist */
+static gpr_mu g_pi_freelist_mu;
+static polling_island *g_pi_freelist = NULL;
+
+static void polling_island_delete(); /* Forward declaration */
+
+#ifdef GRPC_TSAN
+/* Currently TSAN may incorrectly flag data races between epoll_ctl and
+   epoll_wait for any grpc_fd structs that are added to the epoll set via
+   epoll_ctl and are returned (within a very short window) via epoll_wait().
+
+   To work-around this race, we establish a happens-before relation between
+   the code just-before epoll_ctl() and the code after epoll_wait() by using
+   this atomic */
+gpr_atm g_epoll_sync;
+#endif /* defined(GRPC_TSAN) */
+
+#ifdef GRPC_PI_REF_COUNT_DEBUG
+void pi_add_ref(polling_island *pi);
+void pi_unref(polling_island *pi);
+
+void pi_add_ref_dbg(polling_island *pi, char *reason, char *file, int line) {
+  long old_cnt = gpr_atm_acq_load(&(pi->ref_count.count));
+  pi_add_ref(pi);
+  gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)",
+          (void *)pi, old_cnt, old_cnt + 1, reason, file, line);
+}
+
+void pi_unref_dbg(polling_island *pi, char *reason, char *file, int line) {
+  long old_cnt = gpr_atm_acq_load(&(pi->ref_count.count));
+  pi_unref(pi);
+  gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)",
+          (void *)pi, old_cnt, (old_cnt - 1), reason, file, line);
+}
+#endif
+
+void pi_add_ref(polling_island *pi) { gpr_ref(&pi->ref_count); }
+
+void pi_unref(polling_island *pi) {
+  /* If ref count went to zero, delete the polling island.
+     Note that this deletion not be done under a lock. Once the ref count goes
+     to zero, we are guaranteed that no one else holds a reference to the
+     polling island (and that there is no racing pi_add_ref() call either).
+
+     Also, if we are deleting the polling island and the merged_to field is
+     non-empty, we should remove a ref to the merged_to polling island
+   */
+  if (gpr_unref(&pi->ref_count)) {
+    polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+    polling_island_delete(pi);
+    if (next != NULL) {
+      PI_UNREF(next, "pi_delete"); /* Recursive call */
+    }
+  }
+}
+
+/* The caller is expected to hold pi->mu lock before calling this function */
+static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds,
+                                          size_t fd_count, bool add_fd_refs,
+                                          grpc_error **error) {
+  int err;
+  size_t i;
+  struct epoll_event ev;
+  char *err_msg;
+  const char *err_desc = "polling_island_add_fds";
+
+#ifdef GRPC_TSAN
+  /* See the definition of g_epoll_sync for more context */
+  gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0);
+#endif /* defined(GRPC_TSAN) */
+
+  for (i = 0; i < fd_count; i++) {
+    ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET);
+    ev.data.ptr = fds[i];
+    err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev);
+
+    if (err < 0) {
+      if (errno != EEXIST) {
+        gpr_asprintf(
+            &err_msg,
+            "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)",
+            pi->epoll_fd, fds[i]->fd, errno, strerror(errno));
+        append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+        gpr_free(err_msg);
+      }
+
+      continue;
+    }
+
+    if (pi->fd_cnt == pi->fd_capacity) {
+      pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2);
+      pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity);
+    }
+
+    pi->fds[pi->fd_cnt++] = fds[i];
+    if (add_fd_refs) {
+      GRPC_FD_REF(fds[i], "polling_island");
+    }
+  }
+}
+
+/* The caller is expected to hold pi->mu before calling this */
+static void polling_island_add_wakeup_fd_locked(polling_island *pi,
+                                                grpc_wakeup_fd *wakeup_fd,
+                                                grpc_error **error) {
+  struct epoll_event ev;
+  int err;
+  char *err_msg;
+  const char *err_desc = "polling_island_add_wakeup_fd";
+
+  ev.events = (uint32_t)(EPOLLIN | EPOLLET);
+  ev.data.ptr = wakeup_fd;
+  err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD,
+                  GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev);
+  if (err < 0 && errno != EEXIST) {
+    gpr_asprintf(&err_msg,
+                 "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with "
+                 "error: %d (%s)",
+                 pi->epoll_fd,
+                 GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), errno,
+                 strerror(errno));
+    append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+    gpr_free(err_msg);
+  }
+}
+
+/* The caller is expected to hold pi->mu lock before calling this function */
+static void polling_island_remove_all_fds_locked(polling_island *pi,
+                                                 bool remove_fd_refs,
+                                                 grpc_error **error) {
+  int err;
+  size_t i;
+  char *err_msg;
+  const char *err_desc = "polling_island_remove_fds";
+
+  for (i = 0; i < pi->fd_cnt; i++) {
+    err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, NULL);
+    if (err < 0 && errno != ENOENT) {
+      gpr_asprintf(&err_msg,
+                   "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with "
+                   "error: %d (%s)",
+                   pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno));
+      append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+      gpr_free(err_msg);
+    }
+
+    if (remove_fd_refs) {
+      GRPC_FD_UNREF(pi->fds[i], "polling_island");
+    }
+  }
+
+  pi->fd_cnt = 0;
+}
+
+/* The caller is expected to hold pi->mu lock before calling this function */
+static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd,
+                                            bool is_fd_closed,
+                                            grpc_error **error) {
+  int err;
+  size_t i;
+  char *err_msg;
+  const char *err_desc = "polling_island_remove_fd";
+
+  /* If fd is already closed, then it would have been automatically been removed
+     from the epoll set */
+  if (!is_fd_closed) {
+    err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL);
+    if (err < 0 && errno != ENOENT) {
+      gpr_asprintf(
+          &err_msg,
+          "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)",
+          pi->epoll_fd, fd->fd, errno, strerror(errno));
+      append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+      gpr_free(err_msg);
+    }
+  }
+
+  for (i = 0; i < pi->fd_cnt; i++) {
+    if (pi->fds[i] == fd) {
+      pi->fds[i] = pi->fds[--pi->fd_cnt];
+      GRPC_FD_UNREF(fd, "polling_island");
+      break;
+    }
+  }
+}
+
+/* Might return NULL in case of an error */
+static polling_island *polling_island_create(grpc_fd *initial_fd,
+                                             grpc_error **error) {
+  polling_island *pi = NULL;
+  char *err_msg;
+  const char *err_desc = "polling_island_create";
+
+  /* Try to get one from the polling island freelist */
+  gpr_mu_lock(&g_pi_freelist_mu);
+  if (g_pi_freelist != NULL) {
+    pi = g_pi_freelist;
+    g_pi_freelist = g_pi_freelist->next_free;
+    pi->next_free = NULL;
+  }
+  gpr_mu_unlock(&g_pi_freelist_mu);
+
+  /* Create new polling island if we could not get one from the free list */
+  if (pi == NULL) {
+    pi = gpr_malloc(sizeof(*pi));
+    gpr_mu_init(&pi->mu);
+    pi->fd_cnt = 0;
+    pi->fd_capacity = 0;
+    pi->fds = NULL;
+  }
+
+  gpr_ref_init(&pi->ref_count, 0);
+  gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL);
+
+  pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+
+  if (pi->epoll_fd < 0) {
+    gpr_asprintf(&err_msg, "epoll_create1 failed with error %d (%s)", errno,
+                 strerror(errno));
+    append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+    gpr_free(err_msg);
+  } else {
+    polling_island_add_wakeup_fd_locked(pi, &grpc_global_wakeup_fd, error);
+    pi->next_free = NULL;
+
+    if (initial_fd != NULL) {
+      /* Lock the polling island here just in case we got this structure from
+         the freelist and the polling island lock was not released yet (by the
+         code that adds the polling island to the freelist) */
+      gpr_mu_lock(&pi->mu);
+      polling_island_add_fds_locked(pi, &initial_fd, 1, true, error);
+      gpr_mu_unlock(&pi->mu);
+    }
+  }
+
+  return pi;
+}
+
+static void polling_island_delete(polling_island *pi) {
+  GPR_ASSERT(pi->fd_cnt == 0);
+
+  gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL);
+
+  close(pi->epoll_fd);
+  pi->epoll_fd = -1;
+
+  gpr_mu_lock(&g_pi_freelist_mu);
+  pi->next_free = g_pi_freelist;
+  g_pi_freelist = pi;
+  gpr_mu_unlock(&g_pi_freelist_mu);
+}
+
+/* Attempts to gets the last polling island in the linked list (liked by the
+ * 'merged_to' field). Since this does not lock the polling island, there are no
+ * guarantees that the island returned is the last island */
+static polling_island *polling_island_maybe_get_latest(polling_island *pi) {
+  polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+  while (next != NULL) {
+    pi = next;
+    next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+  }
+
+  return pi;
+}
+
+/* Gets the lock on the *latest* polling island i.e the last polling island in
+   the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the
+   returned polling island's mu.
+   Usage: To lock/unlock polling island "pi", do the following:
+      polling_island *pi_latest = polling_island_lock(pi);
+      ...
+      ... critical section ..
+      ...
+      gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */
+static polling_island *polling_island_lock(polling_island *pi) {
+  polling_island *next = NULL;
+
+  while (true) {
+    next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+    if (next == NULL) {
+      /* Looks like 'pi' is the last node in the linked list but unless we check
+         this by holding the pi->mu lock, we cannot be sure (i.e without the
+         pi->mu lock, we don't prevent island merges).
+         To be absolutely sure, check once more by holding the pi->mu lock */
+      gpr_mu_lock(&pi->mu);
+      next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+      if (next == NULL) {
+        /* pi is infact the last node and we have the pi->mu lock. we're done */
+        break;
+      }
+
+      /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu
+       * isn't the lock we are interested in. Continue traversing the list */
+      gpr_mu_unlock(&pi->mu);
+    }
+
+    pi = next;
+  }
+
+  return pi;
+}
+
+/* Gets the lock on the *latest* polling islands in the linked lists pointed by
+   *p and *q (and also updates *p and *q to point to the latest polling islands)
+
+   This function is needed because calling the following block of code to obtain
+   locks on polling islands (*p and *q) is prone to deadlocks.
+     {
+       polling_island_lock(*p, true);
+       polling_island_lock(*q, true);
+     }
+
+   Usage/example:
+     polling_island *p1;
+     polling_island *p2;
+     ..
+     polling_island_lock_pair(&p1, &p2);
+     ..
+     .. Critical section with both p1 and p2 locked
+     ..
+     // Release locks: Always call polling_island_unlock_pair() to release locks
+     polling_island_unlock_pair(p1, p2);
+*/
+static void polling_island_lock_pair(polling_island **p, polling_island **q) {
+  polling_island *pi_1 = *p;
+  polling_island *pi_2 = *q;
+  polling_island *next_1 = NULL;
+  polling_island *next_2 = NULL;
+
+  /* The algorithm is simple:
+      - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and
+        keep updating pi_1 and pi_2)
+      - Then obtain locks on the islands by following a lock order rule of
+        locking polling_island with lower address first
+           Special case: Before obtaining the locks, check if pi_1 and pi_2 are
+           pointing to the same island. If that is the case, we can just call
+           polling_island_lock()
+      - After obtaining both the locks, double check that the polling islands
+        are still the last polling islands in their respective linked lists
+        (this is because there might have been polling island merges before
+        we got the lock)
+      - If the polling islands are the last islands, we are done. If not,
+        release the locks and continue the process from the first step */
+  while (true) {
+    next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to);
+    while (next_1 != NULL) {
+      pi_1 = next_1;
+      next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to);
+    }
+
+    next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to);
+    while (next_2 != NULL) {
+      pi_2 = next_2;
+      next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to);
+    }
+
+    if (pi_1 == pi_2) {
+      pi_1 = pi_2 = polling_island_lock(pi_1);
+      break;
+    }
+
+    if (pi_1 < pi_2) {
+      gpr_mu_lock(&pi_1->mu);
+      gpr_mu_lock(&pi_2->mu);
+    } else {
+      gpr_mu_lock(&pi_2->mu);
+      gpr_mu_lock(&pi_1->mu);
+    }
+
+    next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to);
+    next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to);
+    if (next_1 == NULL && next_2 == NULL) {
+      break;
+    }
+
+    gpr_mu_unlock(&pi_1->mu);
+    gpr_mu_unlock(&pi_2->mu);
+  }
+
+  *p = pi_1;
+  *q = pi_2;
+}
+
+static void polling_island_unlock_pair(polling_island *p, polling_island *q) {
+  if (p == q) {
+    gpr_mu_unlock(&p->mu);
+  } else {
+    gpr_mu_unlock(&p->mu);
+    gpr_mu_unlock(&q->mu);
+  }
+}
+
+static polling_island *polling_island_merge(polling_island *p,
+                                            polling_island *q,
+                                            grpc_error **error) {
+  /* Get locks on both the polling islands */
+  polling_island_lock_pair(&p, &q);
+
+  if (p != q) {
+    /* Make sure that p points to the polling island with fewer fds than q */
+    if (p->fd_cnt > q->fd_cnt) {
+      GPR_SWAP(polling_island *, p, q);
+    }
+
+    /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q
+       Note that the refcounts on the fds being moved will not change here.
+       This is why the last param in the following two functions is 'false') */
+    polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error);
+    polling_island_remove_all_fds_locked(p, false, error);
+
+    /* Wakeup all the pollers (if any) on p so that they pickup this change */
+    polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error);
+
+    /* Add the 'merged_to' link from p --> q */
+    gpr_atm_rel_store(&p->merged_to, (gpr_atm)q);
+    PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */
+  }
+  /* else if p == q, nothing needs to be done */
+
+  polling_island_unlock_pair(p, q);
+
+  /* Return the merged polling island (Note that no merge would have happened
+     if p == q which is ok) */
+  return q;
+}
+
+static grpc_error *polling_island_global_init() {
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  gpr_mu_init(&g_pi_freelist_mu);
+  g_pi_freelist = NULL;
+
+  error = grpc_wakeup_fd_init(&polling_island_wakeup_fd);
+  if (error == GRPC_ERROR_NONE) {
+    error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd);
+  }
+
+  return error;
+}
+
+static void polling_island_global_shutdown() {
+  polling_island *next;
+  gpr_mu_lock(&g_pi_freelist_mu);
+  gpr_mu_unlock(&g_pi_freelist_mu);
+  while (g_pi_freelist != NULL) {
+    next = g_pi_freelist->next_free;
+    gpr_mu_destroy(&g_pi_freelist->mu);
+    gpr_free(g_pi_freelist->fds);
+    gpr_free(g_pi_freelist);
+    g_pi_freelist = next;
+  }
+  gpr_mu_destroy(&g_pi_freelist_mu);
+
+  grpc_wakeup_fd_destroy(&polling_island_wakeup_fd);
+}
+
+/*******************************************************************************
+ * Fd Definitions
+ */
+
+/* We need to keep a freelist not because of any concerns of malloc performance
+ * but instead so that implementations with multiple threads in (for example)
+ * epoll_wait deal with the race between pollset removal and incoming poll
+ * notifications.
+ *
+ * The problem is that the poller ultimately holds a reference to this
+ * object, so it is very difficult to know when is safe to free it, at least
+ * without some expensive synchronization.
+ *
+ * If we keep the object freelisted, in the worst case losing this race just
+ * becomes a spurious read notification on a reused fd.
+ */
+
+/* The alarm system needs to be able to wakeup 'some poller' sometimes
+ * (specifically when a new alarm needs to be triggered earlier than the next
+ * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a
+ * case occurs. */
+
+/* TODO: sreek: Right now, this wakes up all pollers. In future we should make
+ * sure to wake up one polling thread (which can wake up other threads if
+ * needed) */
+grpc_wakeup_fd grpc_global_wakeup_fd;
+
+static grpc_fd *fd_freelist = NULL;
+static gpr_mu fd_freelist_mu;
+
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__)
+#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__)
+static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file,
+                   int line) {
+  gpr_log(GPR_DEBUG, "FD %d %p   ref %d %ld -> %ld [%s; %s:%d]", fd->fd,
+          (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst),
+          gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line);
+#else
+#define REF_BY(fd, n, reason) ref_by(fd, n)
+#define UNREF_BY(fd, n, reason) unref_by(fd, n)
+static void ref_by(grpc_fd *fd, int n) {
+#endif
+  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
+}
+
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file,
+                     int line) {
+  gpr_atm old;
+  gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd,
+          (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst),
+          gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line);
+#else
+static void unref_by(grpc_fd *fd, int n) {
+  gpr_atm old;
+#endif
+  old = gpr_atm_full_fetch_add(&fd->refst, -n);
+  if (old == n) {
+    /* Add the fd to the freelist */
+    gpr_mu_lock(&fd_freelist_mu);
+    fd->freelist_next = fd_freelist;
+    fd_freelist = fd;
+    grpc_iomgr_unregister_object(&fd->iomgr_object);
+
+    gpr_mu_unlock(&fd_freelist_mu);
+  } else {
+    GPR_ASSERT(old > n);
+  }
+}
+
+/* Increment refcount by two to avoid changing the orphan bit */
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void fd_ref(grpc_fd *fd, const char *reason, const char *file,
+                   int line) {
+  ref_by(fd, 2, reason, file, line);
+}
+
+static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
+                     int line) {
+  unref_by(fd, 2, reason, file, line);
+}
+#else
+static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
+static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
+#endif
+
+static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); }
+
+static void fd_global_shutdown(void) {
+  gpr_mu_lock(&fd_freelist_mu);
+  gpr_mu_unlock(&fd_freelist_mu);
+  while (fd_freelist != NULL) {
+    grpc_fd *fd = fd_freelist;
+    fd_freelist = fd_freelist->freelist_next;
+    gpr_mu_destroy(&fd->mu);
+    gpr_free(fd);
+  }
+  gpr_mu_destroy(&fd_freelist_mu);
+}
+
+static grpc_fd *fd_create(int fd, const char *name) {
+  grpc_fd *new_fd = NULL;
+
+  gpr_mu_lock(&fd_freelist_mu);
+  if (fd_freelist != NULL) {
+    new_fd = fd_freelist;
+    fd_freelist = fd_freelist->freelist_next;
+  }
+  gpr_mu_unlock(&fd_freelist_mu);
+
+  if (new_fd == NULL) {
+    new_fd = gpr_malloc(sizeof(grpc_fd));
+    gpr_mu_init(&new_fd->mu);
+    gpr_mu_init(&new_fd->pi_mu);
+  }
+
+  /* Note: It is not really needed to get the new_fd->mu lock here. If this is a
+     newly created fd (or an fd we got from the freelist), no one else would be
+     holding a lock to it anyway. */
+  gpr_mu_lock(&new_fd->mu);
+
+  gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
+  new_fd->fd = fd;
+  new_fd->shutdown = false;
+  new_fd->orphaned = false;
+  new_fd->read_closure = CLOSURE_NOT_READY;
+  new_fd->write_closure = CLOSURE_NOT_READY;
+  new_fd->polling_island = NULL;
+  new_fd->freelist_next = NULL;
+  new_fd->on_done_closure = NULL;
+  new_fd->read_notifier_pollset = NULL;
+
+  gpr_mu_unlock(&new_fd->mu);
+
+  char *fd_name;
+  gpr_asprintf(&fd_name, "%s fd=%d", name, fd);
+  grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name);
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+  gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name);
+#endif
+  gpr_free(fd_name);
+  return new_fd;
+}
+
+static bool fd_is_orphaned(grpc_fd *fd) {
+  return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
+}
+
+static int fd_wrapped_fd(grpc_fd *fd) {
+  int ret_fd = -1;
+  gpr_mu_lock(&fd->mu);
+  if (!fd->orphaned) {
+    ret_fd = fd->fd;
+  }
+  gpr_mu_unlock(&fd->mu);
+
+  return ret_fd;
+}
+
+static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                      grpc_closure *on_done, int *release_fd,
+                      const char *reason) {
+  bool is_fd_closed = false;
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  gpr_mu_lock(&fd->mu);
+  fd->on_done_closure = on_done;
+
+  /* If release_fd is not NULL, we should be relinquishing control of the file
+     descriptor fd->fd (but we still own the grpc_fd structure). */
+  if (release_fd != NULL) {
+    *release_fd = fd->fd;
+  } else {
+    close(fd->fd);
+    is_fd_closed = true;
+  }
+
+  fd->orphaned = true;
+
+  /* Remove the active status but keep referenced. We want this grpc_fd struct
+     to be alive (and not added to freelist) until the end of this function */
+  REF_BY(fd, 1, reason);
+
+  /* Remove the fd from the polling island:
+     - Get a lock on the latest polling island (i.e the last island in the
+       linked list pointed by fd->polling_island). This is the island that
+       would actually contain the fd
+     - Remove the fd from the latest polling island
+     - Unlock the latest polling island
+     - Set fd->polling_island to NULL (but remove the ref on the polling island
+       before doing this.) */
+  gpr_mu_lock(&fd->pi_mu);
+  if (fd->polling_island != NULL) {
+    polling_island *pi_latest = polling_island_lock(fd->polling_island);
+    polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error);
+    gpr_mu_unlock(&pi_latest->mu);
+
+    PI_UNREF(fd->polling_island, "fd_orphan");
+    fd->polling_island = NULL;
+  }
+  gpr_mu_unlock(&fd->pi_mu);
+
+  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, error, NULL);
+
+  gpr_mu_unlock(&fd->mu);
+  UNREF_BY(fd, 2, reason); /* Drop the reference */
+  GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error));
+}
+
+static grpc_error *fd_shutdown_error(bool shutdown) {
+  if (!shutdown) {
+    return GRPC_ERROR_NONE;
+  } else {
+    return GRPC_ERROR_CREATE("FD shutdown");
+  }
+}
+
+static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                             grpc_closure **st, grpc_closure *closure) {
+  if (fd->shutdown) {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"),
+                        NULL);
+  } else if (*st == CLOSURE_NOT_READY) {
+    /* not ready ==> switch to a waiting state by setting the closure */
+    *st = closure;
+  } else if (*st == CLOSURE_READY) {
+    /* already ready ==> queue the closure to run immediately */
+    *st = CLOSURE_NOT_READY;
+    grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
+                        NULL);
+  } else {
+    /* upcallptr was set to a different closure.  This is an error! */
+    gpr_log(GPR_ERROR,
+            "User called a notify_on function with a previous callback still "
+            "pending");
+    abort();
+  }
+}
+
+/* returns 1 if state becomes not ready */
+static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                            grpc_closure **st) {
+  if (*st == CLOSURE_READY) {
+    /* duplicate ready ==> ignore */
+    return 0;
+  } else if (*st == CLOSURE_NOT_READY) {
+    /* not ready, and not waiting ==> flag ready */
+    *st = CLOSURE_READY;
+    return 0;
+  } else {
+    /* waiting ==> queue closure */
+    grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
+    *st = CLOSURE_NOT_READY;
+    return 1;
+  }
+}
+
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
+static bool fd_is_shutdown(grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  const bool r = fd->shutdown;
+  gpr_mu_unlock(&fd->mu);
+  return r;
+}
+
+/* Might be called multiple times */
+static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  /* Do the actual shutdown only once */
+  if (!fd->shutdown) {
+    fd->shutdown = true;
+
+    shutdown(fd->fd, SHUT_RDWR);
+    /* Flush any pending read and write closures. Since fd->shutdown is 'true'
+       at this point, the closures would be called with 'success = false' */
+    set_ready_locked(exec_ctx, fd, &fd->read_closure);
+    set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  }
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                              grpc_closure *closure) {
+  gpr_mu_lock(&fd->mu);
+  notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_closure *closure) {
+  gpr_mu_lock(&fd->mu);
+  notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+/*******************************************************************************
+ * Pollset Definitions
+ */
+GPR_TLS_DECL(g_current_thread_pollset);
+GPR_TLS_DECL(g_current_thread_worker);
+static __thread bool g_initialized_sigmask;
+static __thread sigset_t g_orig_sigmask;
+
+static void sig_handler(int sig_num) {
+#ifdef GRPC_EPOLL_DEBUG
+  gpr_log(GPR_INFO, "Received signal %d", sig_num);
+#endif
+}
+
+static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); }
+
+/* Global state management */
+static grpc_error *pollset_global_init(void) {
+  gpr_tls_init(&g_current_thread_pollset);
+  gpr_tls_init(&g_current_thread_worker);
+  poller_kick_init();
+  return grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
+}
+
+static void pollset_global_shutdown(void) {
+  grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd);
+  gpr_tls_destroy(&g_current_thread_pollset);
+  gpr_tls_destroy(&g_current_thread_worker);
+}
+
+static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) {
+  grpc_error *err = GRPC_ERROR_NONE;
+
+  /* Kick the worker only if it was not already kicked */
+  if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) {
+    GRPC_POLLING_TRACE(
+        "pollset_worker_kick: Kicking worker: %p (thread id: %ld)",
+        (void *)worker, worker->pt_id);
+    int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal);
+    if (err_num != 0) {
+      err = GRPC_OS_ERROR(err_num, "pthread_kill");
+    }
+  }
+  return err;
+}
+
+/* Return 1 if the pollset has active threads in pollset_work (pollset must
+ * be locked) */
+static int pollset_has_workers(grpc_pollset *p) {
+  return p->root_worker.next != &p->root_worker;
+}
+
+static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev->next = worker->next;
+  worker->next->prev = worker->prev;
+}
+
+static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
+  if (pollset_has_workers(p)) {
+    grpc_pollset_worker *w = p->root_worker.next;
+    remove_worker(p, w);
+    return w;
+  } else {
+    return NULL;
+  }
+}
+
+static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->next = &p->root_worker;
+  worker->prev = worker->next->prev;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev = &p->root_worker;
+  worker->next = worker->prev->next;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+/* p->mu must be held before calling this function */
+static grpc_error *pollset_kick(grpc_pollset *p,
+                                grpc_pollset_worker *specific_worker) {
+  GPR_TIMER_BEGIN("pollset_kick", 0);
+  grpc_error *error = GRPC_ERROR_NONE;
+  const char *err_desc = "Kick Failure";
+  grpc_pollset_worker *worker = specific_worker;
+  if (worker != NULL) {
+    if (worker == GRPC_POLLSET_KICK_BROADCAST) {
+      if (pollset_has_workers(p)) {
+        GPR_TIMER_BEGIN("pollset_kick.broadcast", 0);
+        for (worker = p->root_worker.next; worker != &p->root_worker;
+             worker = worker->next) {
+          if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) {
+            append_error(&error, pollset_worker_kick(worker), err_desc);
+          }
+        }
+        GPR_TIMER_END("pollset_kick.broadcast", 0);
+      } else {
+        p->kicked_without_pollers = true;
+      }
+    } else {
+      GPR_TIMER_MARK("kicked_specifically", 0);
+      if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) {
+        append_error(&error, pollset_worker_kick(worker), err_desc);
+      }
+    }
+  } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) {
+    /* Since worker == NULL, it means that we can kick "any" worker on this
+       pollset 'p'. If 'p' happens to be the same pollset this thread is
+       currently polling (i.e in pollset_work() function), then there is no need
+       to kick any other worker since the current thread can just absorb the
+       kick. This is the reason why we enter this case only when
+       g_current_thread_pollset is != p */
+
+    GPR_TIMER_MARK("kick_anonymous", 0);
+    worker = pop_front_worker(p);
+    if (worker != NULL) {
+      GPR_TIMER_MARK("finally_kick", 0);
+      push_back_worker(p, worker);
+      append_error(&error, pollset_worker_kick(worker), err_desc);
+    } else {
+      GPR_TIMER_MARK("kicked_no_pollers", 0);
+      p->kicked_without_pollers = true;
+    }
+  }
+
+  GPR_TIMER_END("pollset_kick", 0);
+  GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error));
+  return error;
+}
+
+static grpc_error *kick_poller(void) {
+  return grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd);
+}
+
+static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
+  gpr_mu_init(&pollset->mu);
+  *mu = &pollset->mu;
+
+  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
+  pollset->kicked_without_pollers = false;
+
+  pollset->shutting_down = false;
+  pollset->finish_shutdown_called = false;
+  pollset->shutdown_done = NULL;
+
+  pollset->polling_island = NULL;
+}
+
+/* Convert a timespec to milliseconds:
+   - Very small or negative poll times are clamped to zero to do a non-blocking
+     poll (which becomes spin polling)
+   - Other small values are rounded up to one millisecond
+   - Longer than a millisecond polls are rounded up to the next nearest
+     millisecond to avoid spinning
+   - Infinite timeouts are converted to -1 */
+static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
+                                           gpr_timespec now) {
+  gpr_timespec timeout;
+  static const int64_t max_spin_polling_us = 10;
+  if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) {
+    return -1;
+  }
+
+  if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros(
+                                                   max_spin_polling_us,
+                                                   GPR_TIMESPAN))) <= 0) {
+    return 0;
+  }
+  timeout = gpr_time_sub(deadline, now);
+  return gpr_time_to_millis(gpr_time_add(
+      timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN)));
+}
+
+static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_pollset *notifier) {
+  /* Need the fd->mu since we might be racing with fd_notify_on_read */
+  gpr_mu_lock(&fd->mu);
+  set_ready_locked(exec_ctx, fd, &fd->read_closure);
+  fd->read_notifier_pollset = notifier;
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  /* Need the fd->mu since we might be racing with fd_notify_on_write */
+  gpr_mu_lock(&fd->mu);
+  set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void pollset_release_polling_island(grpc_pollset *ps, char *reason) {
+  if (ps->polling_island != NULL) {
+    PI_UNREF(ps->polling_island, reason);
+  }
+  ps->polling_island = NULL;
+}
+
+static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx,
+                                   grpc_pollset *pollset) {
+  /* The pollset cannot have any workers if we are at this stage */
+  GPR_ASSERT(!pollset_has_workers(pollset));
+
+  pollset->finish_shutdown_called = true;
+
+  /* Release the ref and set pollset->polling_island to NULL */
+  pollset_release_polling_island(pollset, "ps_shutdown");
+  grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
+}
+
+/* pollset->mu lock must be held by the caller before calling this */
+static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                             grpc_closure *closure) {
+  GPR_TIMER_BEGIN("pollset_shutdown", 0);
+  GPR_ASSERT(!pollset->shutting_down);
+  pollset->shutting_down = true;
+  pollset->shutdown_done = closure;
+  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
+
+  /* If the pollset has any workers, we cannot call finish_shutdown_locked()
+     because it would release the underlying polling island. In such a case, we
+     let the last worker call finish_shutdown_locked() from pollset_work() */
+  if (!pollset_has_workers(pollset)) {
+    GPR_ASSERT(!pollset->finish_shutdown_called);
+    GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0);
+    finish_shutdown_locked(exec_ctx, pollset);
+  }
+  GPR_TIMER_END("pollset_shutdown", 0);
+}
+
+/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other
+ * than destroying the mutexes, there is nothing special that needs to be done
+ * here */
+static void pollset_destroy(grpc_pollset *pollset) {
+  GPR_ASSERT(!pollset_has_workers(pollset));
+  gpr_mu_destroy(&pollset->mu);
+}
+
+static void pollset_reset(grpc_pollset *pollset) {
+  GPR_ASSERT(pollset->shutting_down);
+  GPR_ASSERT(!pollset_has_workers(pollset));
+  pollset->shutting_down = false;
+  pollset->finish_shutdown_called = false;
+  pollset->kicked_without_pollers = false;
+  pollset->shutdown_done = NULL;
+  pollset_release_polling_island(pollset, "ps_reset");
+}
+
+#define GRPC_EPOLL_MAX_EVENTS 1000
+/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */
+static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset *pollset,
+                                    grpc_pollset_worker *worker, int timeout_ms,
+                                    sigset_t *sig_mask, grpc_error **error) {
+  struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
+  int epoll_fd = -1;
+  int ep_rv;
+  polling_island *pi = NULL;
+  char *err_msg;
+  const char *err_desc = "pollset_work_and_unlock";
+  GPR_TIMER_BEGIN("pollset_work_and_unlock", 0);
+
+  /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the
+     latest polling island pointed by pollset->polling_island.
+
+     Since epoll_fd is immutable, we can read it without obtaining the polling
+     island lock. There is however a possibility that the polling island (from
+     which we got the epoll_fd) got merged with another island while we are
+     in this function. This is still okay because in such a case, we will wakeup
+     right-away from epoll_wait() and pick up the latest polling_island the next
+     this function (i.e pollset_work_and_unlock()) is called */
+
+  if (pollset->polling_island == NULL) {
+    pollset->polling_island = polling_island_create(NULL, error);
+    if (pollset->polling_island == NULL) {
+      GPR_TIMER_END("pollset_work_and_unlock", 0);
+      return; /* Fatal error. We cannot continue */
+    }
+
+    PI_ADD_REF(pollset->polling_island, "ps");
+    GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p",
+                       (void *)pollset, (void *)pollset->polling_island);
+  }
+
+  pi = polling_island_maybe_get_latest(pollset->polling_island);
+  epoll_fd = pi->epoll_fd;
+
+  /* Update the pollset->polling_island since the island being pointed by
+     pollset->polling_island maybe older than the one pointed by pi) */
+  if (pollset->polling_island != pi) {
+    /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the
+       polling island to be deleted */
+    PI_ADD_REF(pi, "ps");
+    PI_UNREF(pollset->polling_island, "ps");
+    pollset->polling_island = pi;
+  }
+
+  /* Add an extra ref so that the island does not get destroyed (which means
+     the epoll_fd won't be closed) while we are are doing an epoll_wait() on the
+     epoll_fd */
+  PI_ADD_REF(pi, "ps_work");
+  gpr_mu_unlock(&pollset->mu);
+
+  do {
+    ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms,
+                        sig_mask);
+    if (ep_rv < 0) {
+      if (errno != EINTR) {
+        gpr_asprintf(&err_msg,
+                     "epoll_wait() epoll fd: %d failed with error: %d (%s)",
+                     epoll_fd, errno, strerror(errno));
+        append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+      } else {
+        /* We were interrupted. Save an interation by doing a zero timeout
+           epoll_wait to see if there are any other events of interest */
+        GRPC_POLLING_TRACE(
+            "pollset_work: pollset: %p, worker: %p received kick",
+            (void *)pollset, (void *)worker);
+        ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
+      }
+    }
+
+#ifdef GRPC_TSAN
+    /* See the definition of g_poll_sync for more details */
+    gpr_atm_acq_load(&g_epoll_sync);
+#endif /* defined(GRPC_TSAN) */
+
+    for (int i = 0; i < ep_rv; ++i) {
+      void *data_ptr = ep_ev[i].data.ptr;
+      if (data_ptr == &grpc_global_wakeup_fd) {
+        append_error(error,
+                     grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd),
+                     err_desc);
+      } else if (data_ptr == &polling_island_wakeup_fd) {
+        GRPC_POLLING_TRACE(
+            "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: "
+            "%d) got merged",
+            (void *)pollset, (void *)worker, epoll_fd);
+        /* This means that our polling island is merged with a different
+           island. We do not have to do anything here since the subsequent call
+           to the function pollset_work_and_unlock() will pick up the correct
+           epoll_fd */
+      } else {
+        grpc_fd *fd = data_ptr;
+        int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
+        int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
+        int write_ev = ep_ev[i].events & EPOLLOUT;
+        if (read_ev || cancel) {
+          fd_become_readable(exec_ctx, fd, pollset);
+        }
+        if (write_ev || cancel) {
+          fd_become_writable(exec_ctx, fd);
+        }
+      }
+    }
+  } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
+
+  GPR_ASSERT(pi != NULL);
+
+  /* Before leaving, release the extra ref we added to the polling island. It
+     is important to use "pi" here (i.e our old copy of pollset->polling_island
+     that we got before releasing the polling island lock). This is because
+     pollset->polling_island pointer might get udpated in other parts of the
+     code when there is an island merge while we are doing epoll_wait() above */
+  PI_UNREF(pi, "ps_work");
+
+  GPR_TIMER_END("pollset_work_and_unlock", 0);
+}
+
+/* pollset->mu lock must be held by the caller before calling this.
+   The function pollset_work() may temporarily release the lock (pollset->mu)
+   during the course of its execution but it will always re-acquire the lock and
+   ensure that it is held by the time the function returns */
+static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                                grpc_pollset_worker **worker_hdl,
+                                gpr_timespec now, gpr_timespec deadline) {
+  GPR_TIMER_BEGIN("pollset_work", 0);
+  grpc_error *error = GRPC_ERROR_NONE;
+  int timeout_ms = poll_deadline_to_millis_timeout(deadline, now);
+
+  sigset_t new_mask;
+
+  grpc_pollset_worker worker;
+  worker.next = worker.prev = NULL;
+  worker.pt_id = pthread_self();
+  gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0);
+
+  *worker_hdl = &worker;
+
+  gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
+  gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
+
+  if (pollset->kicked_without_pollers) {
+    /* If the pollset was kicked without pollers, pretend that the current
+       worker got the kick and skip polling. A kick indicates that there is some
+       work that needs attention like an event on the completion queue or an
+       alarm */
+    GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0);
+    pollset->kicked_without_pollers = 0;
+  } else if (!pollset->shutting_down) {
+    /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up
+       (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the
+       worker that there is some pending work that needs immediate attention
+       (like an event on the completion queue, or a polling island merge that
+       results in a new epoll-fd to wait on) and that the worker should not
+       spend time waiting in epoll_pwait().
+
+       A worker can be kicked anytime from the point it is added to the pollset
+       via push_front_worker() (or push_back_worker()) to the point it is
+       removed via remove_worker().
+       If the worker is kicked before/during it calls epoll_pwait(), it should
+       immediately exit from epoll_wait(). If the worker is kicked after it
+       returns from epoll_wait(), then nothing really needs to be done.
+
+       To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all
+       times *except* when it is in epoll_pwait(). This way, the worker never
+       misses acting on a kick */
+
+    if (!g_initialized_sigmask) {
+      sigemptyset(&new_mask);
+      sigaddset(&new_mask, grpc_wakeup_signal);
+      pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask);
+      sigdelset(&g_orig_sigmask, grpc_wakeup_signal);
+      g_initialized_sigmask = true;
+      /* new_mask:       The new thread mask which blocks 'grpc_wakeup_signal'.
+                         This is the mask used at all times *except during
+                         epoll_wait()*"
+         g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and
+                         this is the mask to use *during epoll_wait()*
+
+         The new_mask is set on the worker before it is added to the pollset
+         (i.e before it can be kicked) */
+    }
+
+    push_front_worker(pollset, &worker); /* Add worker to pollset */
+
+    pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms,
+                            &g_orig_sigmask, &error);
+    grpc_exec_ctx_flush(exec_ctx);
+
+    gpr_mu_lock(&pollset->mu);
+
+    /* Note: There is no need to reset worker.is_kicked to 0 since we are no
+       longer going to use this worker */
+    remove_worker(pollset, &worker);
+  }
+
+  /* If we are the last worker on the pollset (i.e pollset_has_workers() is
+     false at this point) and the pollset is shutting down, we may have to
+     finish the shutdown process by calling finish_shutdown_locked().
+     See pollset_shutdown() for more details.
+
+     Note: Continuing to access pollset here is safe; it is the caller's
+     responsibility to not destroy a pollset when it has outstanding calls to
+     pollset_work() */
+  if (pollset->shutting_down && !pollset_has_workers(pollset) &&
+      !pollset->finish_shutdown_called) {
+    GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0);
+    finish_shutdown_locked(exec_ctx, pollset);
+
+    gpr_mu_unlock(&pollset->mu);
+    grpc_exec_ctx_flush(exec_ctx);
+    gpr_mu_lock(&pollset->mu);
+  }
+
+  *worker_hdl = NULL;
+
+  gpr_tls_set(&g_current_thread_pollset, (intptr_t)0);
+  gpr_tls_set(&g_current_thread_worker, (intptr_t)0);
+
+  GPR_TIMER_END("pollset_work", 0);
+
+  GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error));
+  return error;
+}
+
+static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           grpc_fd *fd) {
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  gpr_mu_lock(&pollset->mu);
+  gpr_mu_lock(&fd->pi_mu);
+
+  polling_island *pi_new = NULL;
+
+  /* 1) If fd->polling_island and pollset->polling_island are both non-NULL and
+   *    equal, do nothing.
+   * 2) If fd->polling_island and pollset->polling_island are both NULL, create
+   *    a new polling island (with a refcount of 2) and make the polling_island
+   *    fields in both fd and pollset to point to the new island
+   * 3) If one of fd->polling_island or pollset->polling_island is NULL, update
+   *    the NULL polling_island field to point to the non-NULL polling_island
+   *    field (ensure that the refcount on the polling island is incremented by
+   *    1 to account for the newly added reference)
+   * 4) Finally, if fd->polling_island and pollset->polling_island are non-NULL
+   *    and different, merge both the polling islands and update the
+   *    polling_island fields in both fd and pollset to point to the merged
+   *    polling island.
+   */
+  if (fd->polling_island == pollset->polling_island) {
+    pi_new = fd->polling_island;
+    if (pi_new == NULL) {
+      pi_new = polling_island_create(fd, &error);
+
+      GRPC_POLLING_TRACE(
+          "pollset_add_fd: Created new polling island. pi_new: %p (fd: %d, "
+          "pollset: %p)",
+          (void *)pi_new, fd->fd, (void *)pollset);
+    }
+  } else if (fd->polling_island == NULL) {
+    pi_new = polling_island_lock(pollset->polling_island);
+    polling_island_add_fds_locked(pi_new, &fd, 1, true, &error);
+    gpr_mu_unlock(&pi_new->mu);
+
+    GRPC_POLLING_TRACE(
+        "pollset_add_fd: fd->pi was NULL. pi_new: %p (fd: %d, pollset: %p, "
+        "pollset->pi: %p)",
+        (void *)pi_new, fd->fd, (void *)pollset,
+        (void *)pollset->polling_island);
+  } else if (pollset->polling_island == NULL) {
+    pi_new = polling_island_lock(fd->polling_island);
+    gpr_mu_unlock(&pi_new->mu);
+
+    GRPC_POLLING_TRACE(
+        "pollset_add_fd: pollset->pi was NULL. pi_new: %p (fd: %d, pollset: "
+        "%p, fd->pi: %p",
+        (void *)pi_new, fd->fd, (void *)pollset, (void *)fd->polling_island);
+  } else {
+    pi_new = polling_island_merge(fd->polling_island, pollset->polling_island,
+                                  &error);
+    GRPC_POLLING_TRACE(
+        "pollset_add_fd: polling islands merged. pi_new: %p (fd: %d, pollset: "
+        "%p, fd->pi: %p, pollset->pi: %p)",
+        (void *)pi_new, fd->fd, (void *)pollset, (void *)fd->polling_island,
+        (void *)pollset->polling_island);
+  }
+
+  /* At this point, pi_new is the polling island that both fd->polling_island
+     and pollset->polling_island must be pointing to */
+
+  if (fd->polling_island != pi_new) {
+    PI_ADD_REF(pi_new, "fd");
+    if (fd->polling_island != NULL) {
+      PI_UNREF(fd->polling_island, "fd");
+    }
+    fd->polling_island = pi_new;
+  }
+
+  if (pollset->polling_island != pi_new) {
+    PI_ADD_REF(pi_new, "ps");
+    if (pollset->polling_island != NULL) {
+      PI_UNREF(pollset->polling_island, "ps");
+    }
+    pollset->polling_island = pi_new;
+  }
+
+  gpr_mu_unlock(&fd->pi_mu);
+  gpr_mu_unlock(&pollset->mu);
+}
+
+/*******************************************************************************
+ * Pollset-set Definitions
+ */
+
+static grpc_pollset_set *pollset_set_create(void) {
+  grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set));
+  memset(pollset_set, 0, sizeof(*pollset_set));
+  gpr_mu_init(&pollset_set->mu);
+  return pollset_set;
+}
+
+static void pollset_set_destroy(grpc_pollset_set *pollset_set) {
+  size_t i;
+  gpr_mu_destroy(&pollset_set->mu);
+  for (i = 0; i < pollset_set->fd_count; i++) {
+    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+  }
+  gpr_free(pollset_set->pollsets);
+  gpr_free(pollset_set->pollset_sets);
+  gpr_free(pollset_set->fds);
+  gpr_free(pollset_set);
+}
+
+static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  if (pollset_set->fd_count == pollset_set->fd_capacity) {
+    pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity);
+    pollset_set->fds = gpr_realloc(
+        pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds));
+  }
+  GRPC_FD_REF(fd, "pollset_set");
+  pollset_set->fds[pollset_set->fd_count++] = fd;
+  for (i = 0; i < pollset_set->pollset_count; i++) {
+    pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
+  }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  for (i = 0; i < pollset_set->fd_count; i++) {
+    if (pollset_set->fds[i] == fd) {
+      pollset_set->fd_count--;
+      GPR_SWAP(grpc_fd *, pollset_set->fds[i],
+               pollset_set->fds[pollset_set->fd_count]);
+      GRPC_FD_UNREF(fd, "pollset_set");
+      break;
+    }
+  }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset_set *pollset_set,
+                                    grpc_pollset *pollset) {
+  size_t i, j;
+  gpr_mu_lock(&pollset_set->mu);
+  if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
+    pollset_set->pollset_capacity =
+        GPR_MAX(8, 2 * pollset_set->pollset_capacity);
+    pollset_set->pollsets =
+        gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity *
+                                               sizeof(*pollset_set->pollsets));
+  }
+  pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
+  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
+    if (fd_is_orphaned(pollset_set->fds[i])) {
+      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+    } else {
+      pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
+      pollset_set->fds[j++] = pollset_set->fds[i];
+    }
+  }
+  pollset_set->fd_count = j;
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset_set *pollset_set,
+                                    grpc_pollset *pollset) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  for (i = 0; i < pollset_set->pollset_count; i++) {
+    if (pollset_set->pollsets[i] == pollset) {
+      pollset_set->pollset_count--;
+      GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i],
+               pollset_set->pollsets[pollset_set->pollset_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset_set *bag,
+                                        grpc_pollset_set *item) {
+  size_t i, j;
+  gpr_mu_lock(&bag->mu);
+  if (bag->pollset_set_count == bag->pollset_set_capacity) {
+    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
+    bag->pollset_sets =
+        gpr_realloc(bag->pollset_sets,
+                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
+  }
+  bag->pollset_sets[bag->pollset_set_count++] = item;
+  for (i = 0, j = 0; i < bag->fd_count; i++) {
+    if (fd_is_orphaned(bag->fds[i])) {
+      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
+    } else {
+      pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
+      bag->fds[j++] = bag->fds[i];
+    }
+  }
+  bag->fd_count = j;
+  gpr_mu_unlock(&bag->mu);
+}
+
+static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset_set *bag,
+                                        grpc_pollset_set *item) {
+  size_t i;
+  gpr_mu_lock(&bag->mu);
+  for (i = 0; i < bag->pollset_set_count; i++) {
+    if (bag->pollset_sets[i] == item) {
+      bag->pollset_set_count--;
+      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
+               bag->pollset_sets[bag->pollset_set_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&bag->mu);
+}
+
+/* Test helper functions
+ * */
+void *grpc_fd_get_polling_island(grpc_fd *fd) {
+  polling_island *pi;
+
+  gpr_mu_lock(&fd->pi_mu);
+  pi = fd->polling_island;
+  gpr_mu_unlock(&fd->pi_mu);
+
+  return pi;
+}
+
+void *grpc_pollset_get_polling_island(grpc_pollset *ps) {
+  polling_island *pi;
+
+  gpr_mu_lock(&ps->mu);
+  pi = ps->polling_island;
+  gpr_mu_unlock(&ps->mu);
+
+  return pi;
+}
+
+bool grpc_are_polling_islands_equal(void *p, void *q) {
+  polling_island *p1 = p;
+  polling_island *p2 = q;
+
+  /* Note: polling_island_lock_pair() may change p1 and p2 to point to the
+     latest polling islands in their respective linked lists */
+  polling_island_lock_pair(&p1, &p2);
+  polling_island_unlock_pair(p1, p2);
+
+  return p1 == p2;
+}
+
+/*******************************************************************************
+ * Event engine binding
+ */
+
+static void shutdown_engine(void) {
+  fd_global_shutdown();
+  pollset_global_shutdown();
+  polling_island_global_shutdown();
+}
+
+static const grpc_event_engine_vtable vtable = {
+    .pollset_size = sizeof(grpc_pollset),
+
+    .fd_create = fd_create,
+    .fd_wrapped_fd = fd_wrapped_fd,
+    .fd_orphan = fd_orphan,
+    .fd_shutdown = fd_shutdown,
+    .fd_is_shutdown = fd_is_shutdown,
+    .fd_notify_on_read = fd_notify_on_read,
+    .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
+
+    .pollset_init = pollset_init,
+    .pollset_shutdown = pollset_shutdown,
+    .pollset_reset = pollset_reset,
+    .pollset_destroy = pollset_destroy,
+    .pollset_work = pollset_work,
+    .pollset_kick = pollset_kick,
+    .pollset_add_fd = pollset_add_fd,
+
+    .pollset_set_create = pollset_set_create,
+    .pollset_set_destroy = pollset_set_destroy,
+    .pollset_set_add_pollset = pollset_set_add_pollset,
+    .pollset_set_del_pollset = pollset_set_del_pollset,
+    .pollset_set_add_pollset_set = pollset_set_add_pollset_set,
+    .pollset_set_del_pollset_set = pollset_set_del_pollset_set,
+    .pollset_set_add_fd = pollset_set_add_fd,
+    .pollset_set_del_fd = pollset_set_del_fd,
+
+    .kick_poller = kick_poller,
+
+    .shutdown_engine = shutdown_engine,
+};
+
+/* It is possible that GLIBC has epoll but the underlying kernel doesn't.
+ * Create a dummy epoll_fd to make sure epoll support is available */
+static bool is_epoll_available() {
+  int fd = epoll_create1(EPOLL_CLOEXEC);
+  if (fd < 0) {
+    gpr_log(
+        GPR_ERROR,
+        "epoll_create1 failed with error: %d. Not using epoll polling engine",
+        fd);
+    return false;
+  }
+  close(fd);
+  return true;
+}
+
+const grpc_event_engine_vtable *grpc_init_epoll_linux(void) {
+  /* If use of signals is disabled, we cannot use epoll engine*/
+  if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) {
+    return NULL;
+  }
+
+  if (!is_epoll_available()) {
+    return NULL;
+  }
+
+  if (!is_grpc_wakeup_signal_initialized) {
+    grpc_use_signal(SIGRTMIN + 2);
+  }
+
+  fd_global_init();
+
+  if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) {
+    return NULL;
+  }
+
+  if (!GRPC_LOG_IF_ERROR("polling_island_global_init",
+                         polling_island_global_init())) {
+    return NULL;
+  }
+
+  return &vtable;
+}
+
+#else /* defined(GPR_LINUX_EPOLL) */
+#if defined(GPR_POSIX_SOCKET)
+#include "src/core/lib/iomgr/ev_posix.h"
+/* If GPR_LINUX_EPOLL is not defined, it means epoll is not available. Return
+ * NULL */
+const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; }
+#endif /* defined(GPR_POSIX_SOCKET) */
+
+void grpc_use_signal(int signum) {}
+#endif /* !defined(GPR_LINUX_EPOLL) */
diff --git a/src/core/lib/surface/surface_trace.h b/src/core/lib/iomgr/ev_epoll_linux.h
index a69a0fff57..7a494aba19 100644
--- a/src/core/lib/surface/surface_trace.h
+++ b/src/core/lib/iomgr/ev_epoll_linux.h
@@ -31,18 +31,17 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SURFACE_SURFACE_TRACE_H
-#define GRPC_CORE_LIB_SURFACE_SURFACE_TRACE_H
+#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H
+#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H
 
-#include <grpc/support/log.h>
-#include "src/core/lib/debug/trace.h"
-#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/iomgr/ev_posix.h"
 
-#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)    \
-  if (grpc_api_trace) {                                 \
-    char *_ev = grpc_event_string(event);               \
-    gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \
-    gpr_free(_ev);                                      \
-  }
+const grpc_event_engine_vtable *grpc_init_epoll_linux(void);
 
-#endif /* GRPC_CORE_LIB_SURFACE_SURFACE_TRACE_H */
+#ifdef GPR_LINUX_EPOLL
+void *grpc_fd_get_polling_island(grpc_fd *fd);
+void *grpc_pollset_get_polling_island(grpc_pollset *ps);
+bool grpc_are_polling_islands_equal(void *p, void *q);
+#endif /* defined(GPR_LINUX_EPOLL) */
+
+#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H */
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
index 3c8127e1a8..9e306af5fa 100644
--- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
@@ -126,6 +126,9 @@ struct grpc_fd {
   grpc_closure *on_done_closure;
 
   grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
 };
 
 /* Begin polling on an fd.
@@ -147,7 +150,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
    if got_read or got_write are 1, also does the become_{readable,writable} as
    appropriate. */
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
 
 /* Return 1 if this fd is orphaned, 0 otherwise */
 static bool fd_is_orphaned(grpc_fd *fd);
@@ -217,9 +221,10 @@ struct grpc_pollset {
 struct grpc_pollset_vtable {
   void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                  struct grpc_fd *fd, int and_unlock_pollset);
-  void (*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                grpc_pollset_worker *worker,
-                                gpr_timespec deadline, gpr_timespec now);
+  grpc_error *(*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx,
+                                       grpc_pollset *pollset,
+                                       grpc_pollset_worker *worker,
+                                       gpr_timespec deadline, gpr_timespec now);
   void (*finish_shutdown)(grpc_pollset *pollset);
   void (*destroy)(grpc_pollset *pollset);
 };
@@ -247,9 +252,9 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 #define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2
 /* As per pollset_kick, with an extended set of flags (defined above)
    -- mostly for fd_posix's use. */
-static void pollset_kick_ext(grpc_pollset *p,
-                             grpc_pollset_worker *specific_worker,
-                             uint32_t flags);
+static grpc_error *pollset_kick_ext(grpc_pollset *p,
+                                    grpc_pollset_worker *specific_worker,
+                                    uint32_t flags) GRPC_MUST_USE_RESULT;
 
 /* turn a pollset into a multipoller: platform specific */
 typedef void (*platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx,
@@ -342,6 +347,7 @@ static grpc_fd *alloc_fd(int fd) {
   r->on_done_closure = NULL;
   r->closed = 0;
   r->released = 0;
+  r->read_notifier_pollset = NULL;
   gpr_mu_unlock(&r->mu);
   return r;
 }
@@ -415,12 +421,13 @@ static bool fd_is_orphaned(grpc_fd *fd) {
   return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
 }
 
-static void pollset_kick_locked(grpc_fd_watcher *watcher) {
+static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) {
   gpr_mu_lock(&watcher->pollset->mu);
   GPR_ASSERT(watcher->worker);
-  pollset_kick_ext(watcher->pollset, watcher->worker,
-                   GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
+  grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker,
+                                     GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
   gpr_mu_unlock(&watcher->pollset->mu);
+  return err;
 }
 
 static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
@@ -459,7 +466,7 @@ static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   } else {
     remove_fd_from_all_epoll_sets(fd->fd);
   }
-  grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, true, NULL);
+  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL);
 }
 
 static int fd_wrapped_fd(grpc_fd *fd) {
@@ -508,15 +515,27 @@ static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
 static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
 #endif
 
+static grpc_error *fd_shutdown_error(bool shutdown) {
+  if (!shutdown) {
+    return GRPC_ERROR_NONE;
+  } else {
+    return GRPC_ERROR_CREATE("FD shutdown");
+  }
+}
+
 static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure **st, grpc_closure *closure) {
-  if (*st == CLOSURE_NOT_READY) {
+  if (fd->shutdown) {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"),
+                        NULL);
+  } else if (*st == CLOSURE_NOT_READY) {
     /* not ready ==> switch to a waiting state by setting the closure */
     *st = closure;
   } else if (*st == CLOSURE_READY) {
     /* already ready ==> queue the closure to run immediately */
     *st = CLOSURE_NOT_READY;
-    grpc_exec_ctx_enqueue(exec_ctx, closure, !fd->shutdown, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
+                        NULL);
     maybe_wake_one_watcher_locked(fd);
   } else {
     /* upcallptr was set to a different closure.  This is an error! */
@@ -539,19 +558,35 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
     return 0;
   } else {
     /* waiting ==> queue closure */
-    grpc_exec_ctx_enqueue(exec_ctx, *st, !fd->shutdown, NULL);
+    grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
     *st = CLOSURE_NOT_READY;
     return 1;
   }
 }
 
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_lock(&fd->mu);
-  GPR_ASSERT(!fd->shutdown);
-  fd->shutdown = 1;
-  set_ready_locked(exec_ctx, fd, &fd->read_closure);
-  set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  /* only shutdown once */
+  if (!fd->shutdown) {
+    fd->shutdown = 1;
+    /* signal read/write closed to OS so that future operations fail */
+    shutdown(fd->fd, SHUT_RDWR);
+    set_ready_locked(exec_ctx, fd, &fd->read_closure);
+    set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  }
+  gpr_mu_unlock(&fd->mu);
+}
+
+static bool fd_is_shutdown(grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  bool r = fd->shutdown;
   gpr_mu_unlock(&fd->mu);
+  return r;
 }
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
@@ -568,6 +603,18 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   gpr_mu_unlock(&fd->mu);
 }
 
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
 static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
                               grpc_pollset_worker *worker, uint32_t read_mask,
                               uint32_t write_mask, grpc_fd_watcher *watcher) {
@@ -620,7 +667,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
 }
 
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
   int was_polling = 0;
   int kick = 0;
   grpc_fd *fd = watcher->fd;
@@ -656,6 +704,10 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
     if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
       kick = 1;
     }
+
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
   }
   if (got_write) {
     if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
@@ -717,10 +769,19 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
   worker->prev->next = worker->next->prev = worker;
 }
 
-static void pollset_kick_ext(grpc_pollset *p,
-                             grpc_pollset_worker *specific_worker,
-                             uint32_t flags) {
+static void kick_append_error(grpc_error **composite, grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE("Kick Failure");
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+static grpc_error *pollset_kick_ext(grpc_pollset *p,
+                                    grpc_pollset_worker *specific_worker,
+                                    uint32_t flags) {
   GPR_TIMER_BEGIN("pollset_kick_ext", 0);
+  grpc_error *error = GRPC_ERROR_NONE;
 
   /* pollset->mu already held */
   if (specific_worker != NULL) {
@@ -730,25 +791,28 @@ static void pollset_kick_ext(grpc_pollset *p,
       for (specific_worker = p->root_worker.next;
            specific_worker != &p->root_worker;
            specific_worker = specific_worker->next) {
-        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
+        kick_append_error(
+            &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
       }
-      p->kicked_without_pollers = 1;
+      p->kicked_without_pollers = true;
       GPR_TIMER_END("pollset_kick_ext.broadcast", 0);
     } else if (gpr_tls_get(&g_current_thread_worker) !=
                (intptr_t)specific_worker) {
       GPR_TIMER_MARK("different_thread_worker", 0);
       if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
-        specific_worker->reevaluate_polling_on_wakeup = 1;
+        specific_worker->reevaluate_polling_on_wakeup = true;
       }
-      specific_worker->kicked_specifically = 1;
-      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
+      specific_worker->kicked_specifically = true;
+      kick_append_error(&error,
+                        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
     } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) {
       GPR_TIMER_MARK("kick_yoself", 0);
       if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
-        specific_worker->reevaluate_polling_on_wakeup = 1;
+        specific_worker->reevaluate_polling_on_wakeup = true;
       }
-      specific_worker->kicked_specifically = 1;
-      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
+      specific_worker->kicked_specifically = true;
+      kick_append_error(&error,
+                        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
     }
   } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) {
     GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
@@ -769,39 +833,41 @@ static void pollset_kick_ext(grpc_pollset *p,
       if (specific_worker != NULL) {
         GPR_TIMER_MARK("finally_kick", 0);
         push_back_worker(p, specific_worker);
-        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
+        kick_append_error(
+            &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
       }
     } else {
       GPR_TIMER_MARK("kicked_no_pollers", 0);
-      p->kicked_without_pollers = 1;
+      p->kicked_without_pollers = true;
     }
   }
 
   GPR_TIMER_END("pollset_kick_ext", 0);
+  return error;
 }
 
-static void pollset_kick(grpc_pollset *p,
-                         grpc_pollset_worker *specific_worker) {
-  pollset_kick_ext(p, specific_worker, 0);
+static grpc_error *pollset_kick(grpc_pollset *p,
+                                grpc_pollset_worker *specific_worker) {
+  return pollset_kick_ext(p, specific_worker, 0);
 }
 
 /* global state management */
 
-static void pollset_global_init(void) {
+static grpc_error *pollset_global_init(void) {
   gpr_tls_init(&g_current_thread_poller);
   gpr_tls_init(&g_current_thread_worker);
-  grpc_wakeup_fd_global_init();
-  grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
+  return grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
 }
 
 static void pollset_global_shutdown(void) {
   grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd);
   gpr_tls_destroy(&g_current_thread_poller);
   gpr_tls_destroy(&g_current_thread_worker);
-  grpc_wakeup_fd_global_destroy();
 }
 
-static void kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); }
+static grpc_error *kick_poller(void) {
+  return grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd);
+}
 
 /* main interface */
 
@@ -864,14 +930,23 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
   GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs));
   pollset->vtable->finish_shutdown(pollset);
-  grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL);
+  grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
+}
+
+static void work_combine_error(grpc_error **composite, grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE("pollset_work");
+  }
+  *composite = grpc_error_add_child(*composite, error);
 }
 
-static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                         grpc_pollset_worker **worker_hdl, gpr_timespec now,
-                         gpr_timespec deadline) {
+static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                                grpc_pollset_worker **worker_hdl,
+                                gpr_timespec now, gpr_timespec deadline) {
   grpc_pollset_worker worker;
   *worker_hdl = &worker;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   /* pollset->mu already held */
   int added_worker = 0;
@@ -887,7 +962,10 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     pollset->local_wakeup_cache = worker.wakeup_fd->next;
   } else {
     worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
-    grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
+    error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
+    if (error != GRPC_ERROR_NONE) {
+      return error;
+    }
   }
   worker.kicked_specifically = 0;
   /* If there's work waiting for the pollset to be idle, and the
@@ -924,8 +1002,9 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
       gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
       GPR_TIMER_BEGIN("maybe_work_and_unlock", 0);
-      pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, &worker,
-                                             deadline, now);
+      work_combine_error(&error,
+                         pollset->vtable->maybe_work_and_unlock(
+                             exec_ctx, pollset, &worker, deadline, now));
       GPR_TIMER_END("maybe_work_and_unlock", 0);
       locked = 0;
       gpr_tls_set(&g_current_thread_poller, 0);
@@ -987,6 +1066,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   }
   *worker_hdl = NULL;
   GPR_TIMER_END("pollset_work", 0);
+  return error;
 }
 
 static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -1035,7 +1115,7 @@ typedef struct grpc_unary_promote_args {
 } grpc_unary_promote_args;
 
 static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args,
-                             bool success) {
+                             grpc_error *error) {
   grpc_unary_promote_args *up_args = args;
   const grpc_pollset_vtable *original_vtable = up_args->original_vtable;
   grpc_pollset *pollset = up_args->pollset;
@@ -1137,7 +1217,8 @@ static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   up_args->promotion_closure.cb = basic_do_promote;
   up_args->promotion_closure.cb_arg = up_args;
 
-  grpc_closure_list_add(&pollset->idle_jobs, &up_args->promotion_closure, 1);
+  grpc_closure_list_append(&pollset->idle_jobs, &up_args->promotion_closure,
+                           GRPC_ERROR_NONE);
   pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
 
 exit:
@@ -1146,11 +1227,9 @@ exit:
   }
 }
 
-static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
-                                                grpc_pollset *pollset,
-                                                grpc_pollset_worker *worker,
-                                                gpr_timespec deadline,
-                                                gpr_timespec now) {
+static grpc_error *basic_pollset_maybe_work_and_unlock(
+    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
+    gpr_timespec deadline, gpr_timespec now) {
 #define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
 #define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
 
@@ -1160,6 +1239,7 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
   int timeout;
   int r;
   nfds_t nfds;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   fd = pollset->data.ptr;
   if (fd && fd_is_orphaned(fd)) {
@@ -1200,33 +1280,37 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
 
   if (r < 0) {
     if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
+      work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
     }
     if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
     }
   } else if (r == 0) {
     if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
     }
   } else {
     if (pfd[0].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd));
     }
     if (pfd[1].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd));
     }
     if (nfds > 2) {
       fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK,
-                  pfd[2].revents & POLLOUT_CHECK);
+                  pfd[2].revents & POLLOUT_CHECK, pollset);
     } else if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
     }
   }
 
   if (fd) {
     GRPC_FD_UNREF(fd, "basicpoll_begin");
   }
+
+  return error;
 }
 
 static void basic_pollset_destroy(grpc_pollset *pollset) {
@@ -1287,7 +1371,7 @@ exit:
   }
 }
 
-static void multipoll_with_poll_pollset_maybe_work_and_unlock(
+static grpc_error *multipoll_with_poll_pollset_maybe_work_and_unlock(
     grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
     gpr_timespec deadline, gpr_timespec now) {
 #define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
@@ -1301,6 +1385,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock(
   /* TODO(ctiller): inline some elements to avoid an allocation */
   grpc_fd_watcher *watchers;
   struct pollfd *pfds;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   h = pollset->data.ptr;
   timeout = poll_deadline_to_millis_timeout(deadline, now);
@@ -1353,34 +1438,38 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock(
 
   if (r < 0) {
     if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
+      work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
     }
     for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+      fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
     }
   } else if (r == 0) {
     for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+      fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
     }
   } else {
     if (pfds[0].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd));
     }
     if (pfds[1].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd));
     }
     for (i = 2; i < pfd_count; i++) {
       if (watchers[i].fd == NULL) {
-        fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+        fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         continue;
       }
       fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                  pfds[i].revents & POLLOUT_CHECK);
+                  pfds[i].revents & POLLOUT_CHECK, pollset);
     }
   }
 
   gpr_free(pfds);
   gpr_free(watchers);
+
+  return error;
 }
 
 static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) {
@@ -1451,20 +1540,31 @@ static void poll_become_multipoller(grpc_exec_ctx *exec_ctx,
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
 
-static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st) {
+static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st,
+                      grpc_pollset *read_notifier_pollset) {
   /* only one set_ready can be active at once (but there may be a racing
      notify_on) */
   gpr_mu_lock(&fd->mu);
   set_ready_locked(exec_ctx, fd, st);
+
+  /* A non-NULL read_notifier_pollset means that the fd is readable. */
+  if (read_notifier_pollset != NULL) {
+    /* Note: Since the fd might be a part of multiple pollsets, this might be
+     * called multiple times (for each time the fd becomes readable) and it is
+     * okay to set the fd's read-notifier pollset to anyone of these pollsets */
+    set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+  }
+
   gpr_mu_unlock(&fd->mu);
 }
 
-static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->read_closure);
+static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_pollset *notifier_pollset) {
+  set_ready(exec_ctx, fd, &fd->read_closure, notifier_pollset);
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->write_closure);
+  set_ready(exec_ctx, fd, &fd->write_closure, NULL);
 }
 
 struct epoll_fd_list {
@@ -1556,11 +1656,11 @@ static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
     }
   }
-  fd_end_poll(exec_ctx, &watcher, 0, 0);
+  fd_end_poll(exec_ctx, &watcher, 0, 0, NULL);
 }
 
 static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg,
-                                bool iomgr_status) {
+                                grpc_error *error) {
   delayed_add *da = arg;
 
   if (!fd_is_orphaned(da->fd)) {
@@ -1573,7 +1673,8 @@ static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg,
     /* We don't care about this pollset anymore. */
     if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) {
       da->pollset->called_shutdown = 1;
-      grpc_exec_ctx_enqueue(exec_ctx, da->pollset->shutdown_done, true, NULL);
+      grpc_exec_ctx_sched(exec_ctx, da->pollset->shutdown_done, GRPC_ERROR_NONE,
+                          NULL);
     }
   }
   gpr_mu_unlock(&da->pollset->mu);
@@ -1597,14 +1698,14 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
     GRPC_FD_REF(fd, "delayed_add");
     grpc_closure_init(&da->closure, perform_delayed_add, da);
     pollset->in_flight_cbs++;
-    grpc_exec_ctx_enqueue(exec_ctx, &da->closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &da->closure, GRPC_ERROR_NONE, NULL);
   }
 }
 
 /* TODO(klempner): We probably want to turn this down a bit */
 #define GRPC_EPOLL_MAX_EVENTS 1000
 
-static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
+static grpc_error *multipoll_with_epoll_pollset_maybe_work_and_unlock(
     grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
     gpr_timespec deadline, gpr_timespec now) {
   struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
@@ -1613,6 +1714,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
   epoll_hdr *h = pollset->data.ptr;
   int timeout_ms;
   struct pollfd pfds[2];
+  grpc_error *error = GRPC_ERROR_NONE;
 
   /* If you want to ignore epoll's ability to sanely handle parallel pollers,
    * for a more apples-to-apples performance comparison with poll, add a
@@ -1641,13 +1743,14 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
 
   if (poll_rv < 0) {
     if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
+      work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
     }
   } else if (poll_rv == 0) {
     /* do nothing */
   } else {
     if (pfds[0].revents) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
+      work_combine_error(&error,
+                         grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd));
     }
     if (pfds[1].revents) {
       do {
@@ -1655,7 +1758,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
         ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
         if (ep_rv < 0) {
           if (errno != EINTR) {
-            gpr_log(GPR_ERROR, "epoll_wait() failed: %s", strerror(errno));
+            work_combine_error(&error, GRPC_OS_ERROR(errno, "epoll_wait"));
           }
         } else {
           int i;
@@ -1667,10 +1770,11 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
             int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
             int write_ev = ep_ev[i].events & EPOLLOUT;
             if (fd == NULL) {
-              grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
+              work_combine_error(&error, grpc_wakeup_fd_consume_wakeup(
+                                             &grpc_global_wakeup_fd));
             } else {
               if (read_ev || cancel) {
-                fd_become_readable(exec_ctx, fd);
+                fd_become_readable(exec_ctx, fd, pollset);
               }
               if (write_ev || cancel) {
                 fd_become_writable(exec_ctx, fd);
@@ -1681,6 +1785,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
       } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
     }
   }
+  return error;
 }
 
 static void multipoll_with_epoll_pollset_finish_shutdown(
@@ -1897,8 +2002,10 @@ static const grpc_event_engine_vtable vtable = {
     .fd_wrapped_fd = fd_wrapped_fd,
     .fd_orphan = fd_orphan,
     .fd_shutdown = fd_shutdown,
+    .fd_is_shutdown = fd_is_shutdown,
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
new file mode 100644
index 0000000000..45c0a5e954
--- /dev/null
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -0,0 +1,1267 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_POSIX_SOCKET
+
+#include "src/core/lib/iomgr/ev_poll_posix.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <poll.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/tls.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/block_annotate.h"
+
+/*******************************************************************************
+ * FD declarations
+ */
+
+typedef struct grpc_fd_watcher {
+  struct grpc_fd_watcher *next;
+  struct grpc_fd_watcher *prev;
+  grpc_pollset *pollset;
+  grpc_pollset_worker *worker;
+  grpc_fd *fd;
+} grpc_fd_watcher;
+
+struct grpc_fd {
+  int fd;
+  /* refst format:
+     bit0:   1=active/0=orphaned
+     bit1-n: refcount
+     meaning that mostly we ref by two to avoid altering the orphaned bit,
+     and just unref by 1 when we're ready to flag the object as orphaned */
+  gpr_atm refst;
+
+  gpr_mu mu;
+  int shutdown;
+  int closed;
+  int released;
+
+  /* The watcher list.
+
+     The following watcher related fields are protected by watcher_mu.
+
+     An fd_watcher is an ephemeral object created when an fd wants to
+     begin polling, and destroyed after the poll.
+
+     It denotes the fd's interest in whether to read poll or write poll
+     or both or neither on this fd.
+
+     If a watcher is asked to poll for reads or writes, the read_watcher
+     or write_watcher fields are set respectively. A watcher may be asked
+     to poll for both, in which case both fields will be set.
+
+     read_watcher and write_watcher may be NULL if no watcher has been
+     asked to poll for reads or writes.
+
+     If an fd_watcher is not asked to poll for reads or writes, it's added
+     to a linked list of inactive watchers, rooted at inactive_watcher_root.
+     If at a later time there becomes need of a poller to poll, one of
+     the inactive pollers may be kicked out of their poll loops to take
+     that responsibility. */
+  grpc_fd_watcher inactive_watcher_root;
+  grpc_fd_watcher *read_watcher;
+  grpc_fd_watcher *write_watcher;
+
+  grpc_closure *read_closure;
+  grpc_closure *write_closure;
+
+  grpc_closure *on_done_closure;
+
+  grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
+};
+
+/* Begin polling on an fd.
+   Registers that the given pollset is interested in this fd - so that if read
+   or writability interest changes, the pollset can be kicked to pick up that
+   new interest.
+   Return value is:
+     (fd_needs_read? read_mask : 0) | (fd_needs_write? write_mask : 0)
+   i.e. a combination of read_mask and write_mask determined by the fd's current
+   interest in said events.
+   Polling strategies that do not need to alter their behavior depending on the
+   fd's current interest (such as epoll) do not need to call this function.
+   MUST NOT be called with a pollset lock taken */
+static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
+                              grpc_pollset_worker *worker, uint32_t read_mask,
+                              uint32_t write_mask, grpc_fd_watcher *rec);
+/* Complete polling previously started with fd_begin_poll
+   MUST NOT be called with a pollset lock taken
+   if got_read or got_write are 1, also does the become_{readable,writable} as
+   appropriate. */
+static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
+
+/* Return 1 if this fd is orphaned, 0 otherwise */
+static bool fd_is_orphaned(grpc_fd *fd);
+
+/* Reference counting for fds */
+/*#define GRPC_FD_REF_COUNT_DEBUG*/
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line);
+static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
+                     int line);
+#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__)
+#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__)
+#else
+static void fd_ref(grpc_fd *fd);
+static void fd_unref(grpc_fd *fd);
+#define GRPC_FD_REF(fd, reason) fd_ref(fd)
+#define GRPC_FD_UNREF(fd, reason) fd_unref(fd)
+#endif
+
+#define CLOSURE_NOT_READY ((grpc_closure *)0)
+#define CLOSURE_READY ((grpc_closure *)1)
+
+/*******************************************************************************
+ * pollset declarations
+ */
+
+typedef struct grpc_cached_wakeup_fd {
+  grpc_wakeup_fd fd;
+  struct grpc_cached_wakeup_fd *next;
+} grpc_cached_wakeup_fd;
+
+struct grpc_pollset_worker {
+  grpc_cached_wakeup_fd *wakeup_fd;
+  int reevaluate_polling_on_wakeup;
+  int kicked_specifically;
+  struct grpc_pollset_worker *next;
+  struct grpc_pollset_worker *prev;
+};
+
+struct grpc_pollset {
+  gpr_mu mu;
+  grpc_pollset_worker root_worker;
+  int shutting_down;
+  int called_shutdown;
+  int kicked_without_pollers;
+  grpc_closure *shutdown_done;
+  grpc_closure_list idle_jobs;
+  /* all polled fds */
+  size_t fd_count;
+  size_t fd_capacity;
+  grpc_fd **fds;
+  /* Local cache of eventfds for workers */
+  grpc_cached_wakeup_fd *local_wakeup_cache;
+};
+
+/* Add an fd to a pollset */
+static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           struct grpc_fd *fd);
+
+static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd);
+
+/* Convert a timespec to milliseconds:
+   - very small or negative poll times are clamped to zero to do a
+     non-blocking poll (which becomes spin polling)
+   - other small values are rounded up to one millisecond
+   - longer than a millisecond polls are rounded up to the next nearest
+     millisecond to avoid spinning
+   - infinite timeouts are converted to -1 */
+static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
+                                           gpr_timespec now);
+
+/* Allow kick to wakeup the currently polling worker */
+#define GRPC_POLLSET_CAN_KICK_SELF 1
+/* Force the wakee to repoll when awoken */
+#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2
+/* As per pollset_kick, with an extended set of flags (defined above)
+   -- mostly for fd_posix's use. */
+static grpc_error *pollset_kick_ext(grpc_pollset *p,
+                                    grpc_pollset_worker *specific_worker,
+                                    uint32_t flags) GRPC_MUST_USE_RESULT;
+
+/* Return 1 if the pollset has active threads in pollset_work (pollset must
+ * be locked) */
+static int pollset_has_workers(grpc_pollset *pollset);
+
+/*******************************************************************************
+ * pollset_set definitions
+ */
+
+struct grpc_pollset_set {
+  gpr_mu mu;
+
+  size_t pollset_count;
+  size_t pollset_capacity;
+  grpc_pollset **pollsets;
+
+  size_t pollset_set_count;
+  size_t pollset_set_capacity;
+  struct grpc_pollset_set **pollset_sets;
+
+  size_t fd_count;
+  size_t fd_capacity;
+  grpc_fd **fds;
+};
+
+/*******************************************************************************
+ * fd_posix.c
+ */
+
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__)
+#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__)
+static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file,
+                   int line) {
+  gpr_log(GPR_DEBUG, "FD %d %p   ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
+          gpr_atm_no_barrier_load(&fd->refst),
+          gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line);
+#else
+#define REF_BY(fd, n, reason) ref_by(fd, n)
+#define UNREF_BY(fd, n, reason) unref_by(fd, n)
+static void ref_by(grpc_fd *fd, int n) {
+#endif
+  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
+}
+
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file,
+                     int line) {
+  gpr_atm old;
+  gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
+          gpr_atm_no_barrier_load(&fd->refst),
+          gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line);
+#else
+static void unref_by(grpc_fd *fd, int n) {
+  gpr_atm old;
+#endif
+  old = gpr_atm_full_fetch_add(&fd->refst, -n);
+  if (old == n) {
+    gpr_mu_destroy(&fd->mu);
+    grpc_iomgr_unregister_object(&fd->iomgr_object);
+    gpr_free(fd);
+  } else {
+    GPR_ASSERT(old > n);
+  }
+}
+
+static grpc_fd *fd_create(int fd, const char *name) {
+  grpc_fd *r = gpr_malloc(sizeof(*r));
+  gpr_mu_init(&r->mu);
+  gpr_atm_rel_store(&r->refst, 1);
+  r->shutdown = 0;
+  r->read_closure = CLOSURE_NOT_READY;
+  r->write_closure = CLOSURE_NOT_READY;
+  r->fd = fd;
+  r->inactive_watcher_root.next = r->inactive_watcher_root.prev =
+      &r->inactive_watcher_root;
+  r->read_watcher = r->write_watcher = NULL;
+  r->on_done_closure = NULL;
+  r->closed = 0;
+  r->released = 0;
+  r->read_notifier_pollset = NULL;
+
+  char *name2;
+  gpr_asprintf(&name2, "%s fd=%d", name, fd);
+  grpc_iomgr_register_object(&r->iomgr_object, name2);
+  gpr_free(name2);
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+  gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name);
+#endif
+  return r;
+}
+
+static bool fd_is_orphaned(grpc_fd *fd) {
+  return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
+}
+
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
+static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) {
+  gpr_mu_lock(&watcher->pollset->mu);
+  GPR_ASSERT(watcher->worker);
+  grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker,
+                                     GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
+  gpr_mu_unlock(&watcher->pollset->mu);
+  return err;
+}
+
+static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
+  if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) {
+    pollset_kick_locked(fd->inactive_watcher_root.next);
+  } else if (fd->read_watcher) {
+    pollset_kick_locked(fd->read_watcher);
+  } else if (fd->write_watcher) {
+    pollset_kick_locked(fd->write_watcher);
+  }
+}
+
+static void wake_all_watchers_locked(grpc_fd *fd) {
+  grpc_fd_watcher *watcher;
+  for (watcher = fd->inactive_watcher_root.next;
+       watcher != &fd->inactive_watcher_root; watcher = watcher->next) {
+    pollset_kick_locked(watcher);
+  }
+  if (fd->read_watcher) {
+    pollset_kick_locked(fd->read_watcher);
+  }
+  if (fd->write_watcher && fd->write_watcher != fd->read_watcher) {
+    pollset_kick_locked(fd->write_watcher);
+  }
+}
+
+static int has_watchers(grpc_fd *fd) {
+  return fd->read_watcher != NULL || fd->write_watcher != NULL ||
+         fd->inactive_watcher_root.next != &fd->inactive_watcher_root;
+}
+
+static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  fd->closed = 1;
+  if (!fd->released) {
+    close(fd->fd);
+  }
+  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL);
+}
+
+static int fd_wrapped_fd(grpc_fd *fd) {
+  if (fd->released || fd->closed) {
+    return -1;
+  } else {
+    return fd->fd;
+  }
+}
+
+static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                      grpc_closure *on_done, int *release_fd,
+                      const char *reason) {
+  fd->on_done_closure = on_done;
+  fd->released = release_fd != NULL;
+  if (!fd->released) {
+    shutdown(fd->fd, SHUT_RDWR);
+  } else {
+    *release_fd = fd->fd;
+  }
+  gpr_mu_lock(&fd->mu);
+  REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
+  if (!has_watchers(fd)) {
+    close_fd_locked(exec_ctx, fd);
+  } else {
+    wake_all_watchers_locked(fd);
+  }
+  gpr_mu_unlock(&fd->mu);
+  UNREF_BY(fd, 2, reason); /* drop the reference */
+}
+
+/* increment refcount by two to avoid changing the orphan bit */
+#ifdef GRPC_FD_REF_COUNT_DEBUG
+static void fd_ref(grpc_fd *fd, const char *reason, const char *file,
+                   int line) {
+  ref_by(fd, 2, reason, file, line);
+}
+
+static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
+                     int line) {
+  unref_by(fd, 2, reason, file, line);
+}
+#else
+static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
+
+static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
+#endif
+
+static grpc_error *fd_shutdown_error(bool shutdown) {
+  if (!shutdown) {
+    return GRPC_ERROR_NONE;
+  } else {
+    return GRPC_ERROR_CREATE("FD shutdown");
+  }
+}
+
+static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                             grpc_closure **st, grpc_closure *closure) {
+  if (fd->shutdown) {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"),
+                        NULL);
+  } else if (*st == CLOSURE_NOT_READY) {
+    /* not ready ==> switch to a waiting state by setting the closure */
+    *st = closure;
+  } else if (*st == CLOSURE_READY) {
+    /* already ready ==> queue the closure to run immediately */
+    *st = CLOSURE_NOT_READY;
+    grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
+                        NULL);
+    maybe_wake_one_watcher_locked(fd);
+  } else {
+    /* upcallptr was set to a different closure.  This is an error! */
+    gpr_log(GPR_ERROR,
+            "User called a notify_on function with a previous callback still "
+            "pending");
+    abort();
+  }
+}
+
+/* returns 1 if state becomes not ready */
+static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                            grpc_closure **st) {
+  if (*st == CLOSURE_READY) {
+    /* duplicate ready ==> ignore */
+    return 0;
+  } else if (*st == CLOSURE_NOT_READY) {
+    /* not ready, and not waiting ==> flag ready */
+    *st = CLOSURE_READY;
+    return 0;
+  } else {
+    /* waiting ==> queue closure */
+    grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
+    *st = CLOSURE_NOT_READY;
+    return 1;
+  }
+}
+
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
+static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  /* only shutdown once */
+  if (!fd->shutdown) {
+    fd->shutdown = 1;
+    /* signal read/write closed to OS so that future operations fail */
+    shutdown(fd->fd, SHUT_RDWR);
+    set_ready_locked(exec_ctx, fd, &fd->read_closure);
+    set_ready_locked(exec_ctx, fd, &fd->write_closure);
+  }
+  gpr_mu_unlock(&fd->mu);
+}
+
+static bool fd_is_shutdown(grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  bool r = fd->shutdown;
+  gpr_mu_unlock(&fd->mu);
+  return r;
+}
+
+static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                              grpc_closure *closure) {
+  gpr_mu_lock(&fd->mu);
+  notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_closure *closure) {
+  gpr_mu_lock(&fd->mu);
+  notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
+  gpr_mu_unlock(&fd->mu);
+}
+
+static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
+                              grpc_pollset_worker *worker, uint32_t read_mask,
+                              uint32_t write_mask, grpc_fd_watcher *watcher) {
+  uint32_t mask = 0;
+  grpc_closure *cur;
+  int requested;
+  /* keep track of pollers that have requested our events, in case they change
+   */
+  GRPC_FD_REF(fd, "poll");
+
+  gpr_mu_lock(&fd->mu);
+
+  /* if we are shutdown, then don't add to the watcher set */
+  if (fd->shutdown) {
+    watcher->fd = NULL;
+    watcher->pollset = NULL;
+    watcher->worker = NULL;
+    gpr_mu_unlock(&fd->mu);
+    GRPC_FD_UNREF(fd, "poll");
+    return 0;
+  }
+
+  /* if there is nobody polling for read, but we need to, then start doing so */
+  cur = fd->read_closure;
+  requested = cur != CLOSURE_READY;
+  if (read_mask && fd->read_watcher == NULL && requested) {
+    fd->read_watcher = watcher;
+    mask |= read_mask;
+  }
+  /* if there is nobody polling for write, but we need to, then start doing so
+   */
+  cur = fd->write_closure;
+  requested = cur != CLOSURE_READY;
+  if (write_mask && fd->write_watcher == NULL && requested) {
+    fd->write_watcher = watcher;
+    mask |= write_mask;
+  }
+  /* if not polling, remember this watcher in case we need someone to later */
+  if (mask == 0 && worker != NULL) {
+    watcher->next = &fd->inactive_watcher_root;
+    watcher->prev = watcher->next->prev;
+    watcher->next->prev = watcher->prev->next = watcher;
+  }
+  watcher->pollset = pollset;
+  watcher->worker = worker;
+  watcher->fd = fd;
+  gpr_mu_unlock(&fd->mu);
+
+  return mask;
+}
+
+static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
+  int was_polling = 0;
+  int kick = 0;
+  grpc_fd *fd = watcher->fd;
+
+  if (fd == NULL) {
+    return;
+  }
+
+  gpr_mu_lock(&fd->mu);
+
+  if (watcher == fd->read_watcher) {
+    /* remove read watcher, kick if we still need a read */
+    was_polling = 1;
+    if (!got_read) {
+      kick = 1;
+    }
+    fd->read_watcher = NULL;
+  }
+  if (watcher == fd->write_watcher) {
+    /* remove write watcher, kick if we still need a write */
+    was_polling = 1;
+    if (!got_write) {
+      kick = 1;
+    }
+    fd->write_watcher = NULL;
+  }
+  if (!was_polling && watcher->worker != NULL) {
+    /* remove from inactive list */
+    watcher->next->prev = watcher->prev;
+    watcher->prev->next = watcher->next;
+  }
+  if (got_read) {
+    if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
+      kick = 1;
+    }
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
+  }
+  if (got_write) {
+    if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
+      kick = 1;
+    }
+  }
+  if (kick) {
+    maybe_wake_one_watcher_locked(fd);
+  }
+  if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) {
+    close_fd_locked(exec_ctx, fd);
+  }
+  gpr_mu_unlock(&fd->mu);
+
+  GRPC_FD_UNREF(fd, "poll");
+}
+
+/*******************************************************************************
+ * pollset_posix.c
+ */
+
+GPR_TLS_DECL(g_current_thread_poller);
+GPR_TLS_DECL(g_current_thread_worker);
+
+static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev->next = worker->next;
+  worker->next->prev = worker->prev;
+}
+
+static int pollset_has_workers(grpc_pollset *p) {
+  return p->root_worker.next != &p->root_worker;
+}
+
+static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
+  if (pollset_has_workers(p)) {
+    grpc_pollset_worker *w = p->root_worker.next;
+    remove_worker(p, w);
+    return w;
+  } else {
+    return NULL;
+  }
+}
+
+static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->next = &p->root_worker;
+  worker->prev = worker->next->prev;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
+  worker->prev = &p->root_worker;
+  worker->next = worker->prev->next;
+  worker->prev->next = worker->next->prev = worker;
+}
+
+static void kick_append_error(grpc_error **composite, grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE("Kick Failure");
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+static grpc_error *pollset_kick_ext(grpc_pollset *p,
+                                    grpc_pollset_worker *specific_worker,
+                                    uint32_t flags) {
+  GPR_TIMER_BEGIN("pollset_kick_ext", 0);
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  /* pollset->mu already held */
+  if (specific_worker != NULL) {
+    if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
+      GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0);
+      GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
+      for (specific_worker = p->root_worker.next;
+           specific_worker != &p->root_worker;
+           specific_worker = specific_worker->next) {
+        kick_append_error(
+            &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+      }
+      p->kicked_without_pollers = true;
+      GPR_TIMER_END("pollset_kick_ext.broadcast", 0);
+    } else if (gpr_tls_get(&g_current_thread_worker) !=
+               (intptr_t)specific_worker) {
+      GPR_TIMER_MARK("different_thread_worker", 0);
+      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
+        specific_worker->reevaluate_polling_on_wakeup = true;
+      }
+      specific_worker->kicked_specifically = true;
+      kick_append_error(&error,
+                        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+    } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) {
+      GPR_TIMER_MARK("kick_yoself", 0);
+      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
+        specific_worker->reevaluate_polling_on_wakeup = true;
+      }
+      specific_worker->kicked_specifically = true;
+      kick_append_error(&error,
+                        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+    }
+  } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) {
+    GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
+    GPR_TIMER_MARK("kick_anonymous", 0);
+    specific_worker = pop_front_worker(p);
+    if (specific_worker != NULL) {
+      if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) {
+        GPR_TIMER_MARK("kick_anonymous_not_self", 0);
+        push_back_worker(p, specific_worker);
+        specific_worker = pop_front_worker(p);
+        if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 &&
+            gpr_tls_get(&g_current_thread_worker) ==
+                (intptr_t)specific_worker) {
+          push_back_worker(p, specific_worker);
+          specific_worker = NULL;
+        }
+      }
+      if (specific_worker != NULL) {
+        GPR_TIMER_MARK("finally_kick", 0);
+        push_back_worker(p, specific_worker);
+        kick_append_error(
+            &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd));
+      }
+    } else {
+      GPR_TIMER_MARK("kicked_no_pollers", 0);
+      p->kicked_without_pollers = true;
+    }
+  }
+
+  GPR_TIMER_END("pollset_kick_ext", 0);
+  GRPC_LOG_IF_ERROR("pollset_kick_ext", GRPC_ERROR_REF(error));
+  return error;
+}
+
+static grpc_error *pollset_kick(grpc_pollset *p,
+                                grpc_pollset_worker *specific_worker) {
+  return pollset_kick_ext(p, specific_worker, 0);
+}
+
+/* global state management */
+
+static grpc_error *pollset_global_init(void) {
+  gpr_tls_init(&g_current_thread_poller);
+  gpr_tls_init(&g_current_thread_worker);
+  return grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
+}
+
+static void pollset_global_shutdown(void) {
+  grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd);
+  gpr_tls_destroy(&g_current_thread_poller);
+  gpr_tls_destroy(&g_current_thread_worker);
+}
+
+static grpc_error *kick_poller(void) {
+  return grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd);
+}
+
+/* main interface */
+
+static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
+  gpr_mu_init(&pollset->mu);
+  *mu = &pollset->mu;
+  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
+  pollset->shutting_down = 0;
+  pollset->called_shutdown = 0;
+  pollset->kicked_without_pollers = 0;
+  pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL;
+  pollset->local_wakeup_cache = NULL;
+  pollset->kicked_without_pollers = 0;
+  pollset->fd_count = 0;
+  pollset->fd_capacity = 0;
+  pollset->fds = NULL;
+}
+
+static void pollset_destroy(grpc_pollset *pollset) {
+  GPR_ASSERT(!pollset_has_workers(pollset));
+  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
+  while (pollset->local_wakeup_cache) {
+    grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next;
+    grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd);
+    gpr_free(pollset->local_wakeup_cache);
+    pollset->local_wakeup_cache = next;
+  }
+  gpr_free(pollset->fds);
+  gpr_mu_destroy(&pollset->mu);
+}
+
+static void pollset_reset(grpc_pollset *pollset) {
+  GPR_ASSERT(pollset->shutting_down);
+  GPR_ASSERT(!pollset_has_workers(pollset));
+  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
+  GPR_ASSERT(pollset->fd_count == 0);
+  pollset->shutting_down = 0;
+  pollset->called_shutdown = 0;
+  pollset->kicked_without_pollers = 0;
+}
+
+static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           grpc_fd *fd) {
+  gpr_mu_lock(&pollset->mu);
+  size_t i;
+  /* TODO(ctiller): this is O(num_fds^2); maybe switch to a hash set here */
+  for (i = 0; i < pollset->fd_count; i++) {
+    if (pollset->fds[i] == fd) goto exit;
+  }
+  if (pollset->fd_count == pollset->fd_capacity) {
+    pollset->fd_capacity =
+        GPR_MAX(pollset->fd_capacity + 8, pollset->fd_count * 3 / 2);
+    pollset->fds =
+        gpr_realloc(pollset->fds, sizeof(grpc_fd *) * pollset->fd_capacity);
+  }
+  pollset->fds[pollset->fd_count++] = fd;
+  GRPC_FD_REF(fd, "multipoller");
+  pollset_kick(pollset, NULL);
+exit:
+  gpr_mu_unlock(&pollset->mu);
+}
+
+static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
+  GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs));
+  size_t i;
+  for (i = 0; i < pollset->fd_count; i++) {
+    GRPC_FD_UNREF(pollset->fds[i], "multipoller");
+  }
+  pollset->fd_count = 0;
+  grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
+}
+
+static void work_combine_error(grpc_error **composite, grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE("pollset_work");
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                                grpc_pollset_worker **worker_hdl,
+                                gpr_timespec now, gpr_timespec deadline) {
+  grpc_pollset_worker worker;
+  *worker_hdl = &worker;
+  grpc_error *error = GRPC_ERROR_NONE;
+
+  /* pollset->mu already held */
+  int added_worker = 0;
+  int locked = 1;
+  int queued_work = 0;
+  int keep_polling = 0;
+  GPR_TIMER_BEGIN("pollset_work", 0);
+  /* this must happen before we (potentially) drop pollset->mu */
+  worker.next = worker.prev = NULL;
+  worker.reevaluate_polling_on_wakeup = 0;
+  if (pollset->local_wakeup_cache != NULL) {
+    worker.wakeup_fd = pollset->local_wakeup_cache;
+    pollset->local_wakeup_cache = worker.wakeup_fd->next;
+  } else {
+    worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
+    error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
+    if (error != GRPC_ERROR_NONE) {
+      GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error));
+      return error;
+    }
+  }
+  worker.kicked_specifically = 0;
+  /* If there's work waiting for the pollset to be idle, and the
+     pollset is idle, then do that work */
+  if (!pollset_has_workers(pollset) &&
+      !grpc_closure_list_empty(pollset->idle_jobs)) {
+    GPR_TIMER_MARK("pollset_work.idle_jobs", 0);
+    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+    goto done;
+  }
+  /* If we're shutting down then we don't execute any extended work */
+  if (pollset->shutting_down) {
+    GPR_TIMER_MARK("pollset_work.shutting_down", 0);
+    goto done;
+  }
+  /* Start polling, and keep doing so while we're being asked to
+     re-evaluate our pollers (this allows poll() based pollers to
+     ensure they don't miss wakeups) */
+  keep_polling = 1;
+  gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
+  while (keep_polling) {
+    keep_polling = 0;
+    if (!pollset->kicked_without_pollers) {
+      if (!added_worker) {
+        push_front_worker(pollset, &worker);
+        added_worker = 1;
+        gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
+      }
+      GPR_TIMER_BEGIN("maybe_work_and_unlock", 0);
+#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
+#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
+
+      int timeout;
+      int r;
+      size_t i, fd_count;
+      nfds_t pfd_count;
+      /* TODO(ctiller): inline some elements to avoid an allocation */
+      grpc_fd_watcher *watchers;
+      struct pollfd *pfds;
+
+      timeout = poll_deadline_to_millis_timeout(deadline, now);
+      /* TODO(ctiller): perform just one malloc here if we exceed the inline
+       * case */
+      pfds = gpr_malloc(sizeof(*pfds) * (pollset->fd_count + 2));
+      watchers = gpr_malloc(sizeof(*watchers) * (pollset->fd_count + 2));
+      fd_count = 0;
+      pfd_count = 2;
+      pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
+      pfds[0].events = POLLIN;
+      pfds[0].revents = 0;
+      pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd);
+      pfds[1].events = POLLIN;
+      pfds[1].revents = 0;
+      for (i = 0; i < pollset->fd_count; i++) {
+        if (fd_is_orphaned(pollset->fds[i])) {
+          GRPC_FD_UNREF(pollset->fds[i], "multipoller");
+        } else {
+          pollset->fds[fd_count++] = pollset->fds[i];
+          watchers[pfd_count].fd = pollset->fds[i];
+          GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start");
+          pfds[pfd_count].fd = pollset->fds[i]->fd;
+          pfds[pfd_count].revents = 0;
+          pfd_count++;
+        }
+      }
+      pollset->fd_count = fd_count;
+      gpr_mu_unlock(&pollset->mu);
+
+      for (i = 2; i < pfd_count; i++) {
+        grpc_fd *fd = watchers[i].fd;
+        pfds[i].events = (short)fd_begin_poll(fd, pollset, &worker, POLLIN,
+                                              POLLOUT, &watchers[i]);
+        GRPC_FD_UNREF(fd, "multipoller_start");
+      }
+
+      /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
+         even going into the blocking annotation if possible */
+      GRPC_SCHEDULING_START_BLOCKING_REGION;
+      r = grpc_poll_function(pfds, pfd_count, timeout);
+      GRPC_SCHEDULING_END_BLOCKING_REGION;
+
+      if (r < 0) {
+        if (errno != EINTR) {
+          work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
+        }
+        for (i = 2; i < pfd_count; i++) {
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
+        }
+      } else if (r == 0) {
+        for (i = 2; i < pfd_count; i++) {
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
+        }
+      } else {
+        if (pfds[0].revents & POLLIN_CHECK) {
+          work_combine_error(
+              &error, grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd));
+        }
+        if (pfds[1].revents & POLLIN_CHECK) {
+          work_combine_error(
+              &error, grpc_wakeup_fd_consume_wakeup(&worker.wakeup_fd->fd));
+        }
+        for (i = 2; i < pfd_count; i++) {
+          if (watchers[i].fd == NULL) {
+            fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
+          } else {
+            fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
+                        pfds[i].revents & POLLOUT_CHECK, pollset);
+          }
+        }
+      }
+
+      gpr_free(pfds);
+      gpr_free(watchers);
+      GPR_TIMER_END("maybe_work_and_unlock", 0);
+      locked = 0;
+    } else {
+      GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0);
+      pollset->kicked_without_pollers = 0;
+    }
+  /* Finished execution - start cleaning up.
+     Note that we may arrive here from outside the enclosing while() loop.
+     In that case we won't loop though as we haven't added worker to the
+     worker list, which means nobody could ask us to re-evaluate polling). */
+  done:
+    if (!locked) {
+      queued_work |= grpc_exec_ctx_flush(exec_ctx);
+      gpr_mu_lock(&pollset->mu);
+      locked = 1;
+    }
+    /* If we're forced to re-evaluate polling (via pollset_kick with
+       GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force
+       a loop */
+    if (worker.reevaluate_polling_on_wakeup && error == GRPC_ERROR_NONE) {
+      worker.reevaluate_polling_on_wakeup = 0;
+      pollset->kicked_without_pollers = 0;
+      if (queued_work || worker.kicked_specifically) {
+        /* If there's queued work on the list, then set the deadline to be
+           immediate so we get back out of the polling loop quickly */
+        deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
+      }
+      keep_polling = 1;
+    }
+    if (keep_polling) {
+      now = gpr_now(now.clock_type);
+    }
+  }
+  gpr_tls_set(&g_current_thread_poller, 0);
+  if (added_worker) {
+    remove_worker(pollset, &worker);
+    gpr_tls_set(&g_current_thread_worker, 0);
+  }
+  /* release wakeup fd to the local pool */
+  worker.wakeup_fd->next = pollset->local_wakeup_cache;
+  pollset->local_wakeup_cache = worker.wakeup_fd;
+  /* check shutdown conditions */
+  if (pollset->shutting_down) {
+    if (pollset_has_workers(pollset)) {
+      pollset_kick(pollset, NULL);
+    } else if (!pollset->called_shutdown) {
+      pollset->called_shutdown = 1;
+      gpr_mu_unlock(&pollset->mu);
+      finish_shutdown(exec_ctx, pollset);
+      grpc_exec_ctx_flush(exec_ctx);
+      /* Continuing to access pollset here is safe -- it is the caller's
+       * responsibility to not destroy when it has outstanding calls to
+       * pollset_work.
+       * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */
+      gpr_mu_lock(&pollset->mu);
+    } else if (!grpc_closure_list_empty(pollset->idle_jobs)) {
+      grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+      gpr_mu_unlock(&pollset->mu);
+      grpc_exec_ctx_flush(exec_ctx);
+      gpr_mu_lock(&pollset->mu);
+    }
+  }
+  *worker_hdl = NULL;
+  GPR_TIMER_END("pollset_work", 0);
+  GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error));
+  return error;
+}
+
+static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                             grpc_closure *closure) {
+  GPR_ASSERT(!pollset->shutting_down);
+  pollset->shutting_down = 1;
+  pollset->shutdown_done = closure;
+  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
+  if (!pollset_has_workers(pollset)) {
+    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+  }
+  if (!pollset->called_shutdown && !pollset_has_workers(pollset)) {
+    pollset->called_shutdown = 1;
+    finish_shutdown(exec_ctx, pollset);
+  }
+}
+
+static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
+                                           gpr_timespec now) {
+  gpr_timespec timeout;
+  static const int64_t max_spin_polling_us = 10;
+  if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) {
+    return -1;
+  }
+  if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros(
+                                                   max_spin_polling_us,
+                                                   GPR_TIMESPAN))) <= 0) {
+    return 0;
+  }
+  timeout = gpr_time_sub(deadline, now);
+  return gpr_time_to_millis(gpr_time_add(
+      timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN)));
+}
+
+/*******************************************************************************
+ * pollset_set_posix.c
+ */
+
+static grpc_pollset_set *pollset_set_create(void) {
+  grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set));
+  memset(pollset_set, 0, sizeof(*pollset_set));
+  gpr_mu_init(&pollset_set->mu);
+  return pollset_set;
+}
+
+static void pollset_set_destroy(grpc_pollset_set *pollset_set) {
+  size_t i;
+  gpr_mu_destroy(&pollset_set->mu);
+  for (i = 0; i < pollset_set->fd_count; i++) {
+    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+  }
+  gpr_free(pollset_set->pollsets);
+  gpr_free(pollset_set->pollset_sets);
+  gpr_free(pollset_set->fds);
+  gpr_free(pollset_set);
+}
+
+static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset_set *pollset_set,
+                                    grpc_pollset *pollset) {
+  size_t i, j;
+  gpr_mu_lock(&pollset_set->mu);
+  if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
+    pollset_set->pollset_capacity =
+        GPR_MAX(8, 2 * pollset_set->pollset_capacity);
+    pollset_set->pollsets =
+        gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity *
+                                               sizeof(*pollset_set->pollsets));
+  }
+  pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
+  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
+    if (fd_is_orphaned(pollset_set->fds[i])) {
+      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+    } else {
+      pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
+      pollset_set->fds[j++] = pollset_set->fds[i];
+    }
+  }
+  pollset_set->fd_count = j;
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
+                                    grpc_pollset_set *pollset_set,
+                                    grpc_pollset *pollset) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  for (i = 0; i < pollset_set->pollset_count; i++) {
+    if (pollset_set->pollsets[i] == pollset) {
+      pollset_set->pollset_count--;
+      GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i],
+               pollset_set->pollsets[pollset_set->pollset_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset_set *bag,
+                                        grpc_pollset_set *item) {
+  size_t i, j;
+  gpr_mu_lock(&bag->mu);
+  if (bag->pollset_set_count == bag->pollset_set_capacity) {
+    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
+    bag->pollset_sets =
+        gpr_realloc(bag->pollset_sets,
+                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
+  }
+  bag->pollset_sets[bag->pollset_set_count++] = item;
+  for (i = 0, j = 0; i < bag->fd_count; i++) {
+    if (fd_is_orphaned(bag->fds[i])) {
+      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
+    } else {
+      pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
+      bag->fds[j++] = bag->fds[i];
+    }
+  }
+  bag->fd_count = j;
+  gpr_mu_unlock(&bag->mu);
+}
+
+static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset_set *bag,
+                                        grpc_pollset_set *item) {
+  size_t i;
+  gpr_mu_lock(&bag->mu);
+  for (i = 0; i < bag->pollset_set_count; i++) {
+    if (bag->pollset_sets[i] == item) {
+      bag->pollset_set_count--;
+      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
+               bag->pollset_sets[bag->pollset_set_count]);
+      break;
+    }
+  }
+  gpr_mu_unlock(&bag->mu);
+}
+
+static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  if (pollset_set->fd_count == pollset_set->fd_capacity) {
+    pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity);
+    pollset_set->fds = gpr_realloc(
+        pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds));
+  }
+  GRPC_FD_REF(fd, "pollset_set");
+  pollset_set->fds[pollset_set->fd_count++] = fd;
+  for (i = 0; i < pollset_set->pollset_count; i++) {
+    pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
+  }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
+                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
+  size_t i;
+  gpr_mu_lock(&pollset_set->mu);
+  for (i = 0; i < pollset_set->fd_count; i++) {
+    if (pollset_set->fds[i] == fd) {
+      pollset_set->fd_count--;
+      GPR_SWAP(grpc_fd *, pollset_set->fds[i],
+               pollset_set->fds[pollset_set->fd_count]);
+      GRPC_FD_UNREF(fd, "pollset_set");
+      break;
+    }
+  }
+  for (i = 0; i < pollset_set->pollset_set_count; i++) {
+    pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
+  }
+  gpr_mu_unlock(&pollset_set->mu);
+}
+
+/*******************************************************************************
+ * event engine binding
+ */
+
+static void shutdown_engine(void) { pollset_global_shutdown(); }
+
+static const grpc_event_engine_vtable vtable = {
+    .pollset_size = sizeof(grpc_pollset),
+
+    .fd_create = fd_create,
+    .fd_wrapped_fd = fd_wrapped_fd,
+    .fd_orphan = fd_orphan,
+    .fd_shutdown = fd_shutdown,
+    .fd_is_shutdown = fd_is_shutdown,
+    .fd_notify_on_read = fd_notify_on_read,
+    .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
+
+    .pollset_init = pollset_init,
+    .pollset_shutdown = pollset_shutdown,
+    .pollset_reset = pollset_reset,
+    .pollset_destroy = pollset_destroy,
+    .pollset_work = pollset_work,
+    .pollset_kick = pollset_kick,
+    .pollset_add_fd = pollset_add_fd,
+
+    .pollset_set_create = pollset_set_create,
+    .pollset_set_destroy = pollset_set_destroy,
+    .pollset_set_add_pollset = pollset_set_add_pollset,
+    .pollset_set_del_pollset = pollset_set_del_pollset,
+    .pollset_set_add_pollset_set = pollset_set_add_pollset_set,
+    .pollset_set_del_pollset_set = pollset_set_del_pollset_set,
+    .pollset_set_add_fd = pollset_set_add_fd,
+    .pollset_set_del_fd = pollset_set_del_fd,
+
+    .kick_poller = kick_poller,
+
+    .shutdown_engine = shutdown_engine,
+};
+
+const grpc_event_engine_vtable *grpc_init_poll_posix(void) {
+  if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) {
+    return NULL;
+  }
+  return &vtable;
+}
+
+#endif
diff --git a/src/ruby/ext/grpc/rb_signal.h b/src/core/lib/iomgr/ev_poll_posix.h
index 07e49c0a8b..291736a2db 100644
--- a/src/ruby/ext/grpc/rb_signal.h
+++ b/src/core/lib/iomgr/ev_poll_posix.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2016, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,9 +31,11 @@
  *
  */
 
-#ifndef GRPC_RB_SIGNAL_H_
-#define GRPC_RB_SIGNAL_H_
+#ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H
 
-void Init_grpc_signals();
+#include "src/core/lib/iomgr/ev_posix.h"
 
-#endif  /* GRPC_RB_SIGNAL_H_ */
+const grpc_event_engine_vtable *grpc_init_poll_posix(void);
+
+#endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H */
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index 7df1751352..a3c1e9db9a 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -37,23 +37,112 @@
 
 #include "src/core/lib/iomgr/ev_posix.h"
 
+#include <string.h>
+
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
 
+#include "src/core/lib/iomgr/ev_epoll_linux.h"
 #include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
+#include "src/core/lib/iomgr/ev_poll_posix.h"
+#include "src/core/lib/support/env.h"
+
+/** Default poll() function - a pointer so that it can be overridden by some
+ *  tests */
+grpc_poll_function_type grpc_poll_function = poll;
 
 static const grpc_event_engine_vtable *g_event_engine;
+static const char *g_poll_strategy_name = NULL;
+
+typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(void);
+
+typedef struct {
+  const char *name;
+  event_engine_factory_fn factory;
+} event_engine_factory;
+
+static const event_engine_factory g_factories[] = {
+    {"epoll", grpc_init_epoll_linux},
+    {"poll", grpc_init_poll_posix},
+    {"legacy", grpc_init_poll_and_epoll_posix},
+};
+
+static void add(const char *beg, const char *end, char ***ss, size_t *ns) {
+  size_t n = *ns;
+  size_t np = n + 1;
+  char *s;
+  size_t len;
+  GPR_ASSERT(end >= beg);
+  len = (size_t)(end - beg);
+  s = gpr_malloc(len + 1);
+  memcpy(s, beg, len);
+  s[len] = 0;
+  *ss = gpr_realloc(*ss, sizeof(char **) * np);
+  (*ss)[n] = s;
+  *ns = np;
+}
 
-grpc_poll_function_type grpc_poll_function = poll;
+static void split(const char *s, char ***ss, size_t *ns) {
+  const char *c = strchr(s, ',');
+  if (c == NULL) {
+    add(s, s + strlen(s), ss, ns);
+  } else {
+    add(s, c, ss, ns);
+    split(c + 1, ss, ns);
+  }
+}
+
+static bool is(const char *want, const char *have) {
+  return 0 == strcmp(want, "all") || 0 == strcmp(want, have);
+}
+
+static void try_engine(const char *engine) {
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) {
+    if (is(engine, g_factories[i].name)) {
+      if ((g_event_engine = g_factories[i].factory())) {
+        g_poll_strategy_name = g_factories[i].name;
+        gpr_log(GPR_DEBUG, "Using polling engine: %s", g_factories[i].name);
+        return;
+      }
+    }
+  }
+}
+
+/* Call this only after calling grpc_event_engine_init() */
+const char *grpc_get_poll_strategy_name() { return g_poll_strategy_name; }
 
 void grpc_event_engine_init(void) {
-  if ((g_event_engine = grpc_init_poll_and_epoll_posix())) {
-    return;
+  char *s = gpr_getenv("GRPC_POLL_STRATEGY");
+  if (s == NULL) {
+    s = gpr_strdup("all");
+  }
+
+  char **strings = NULL;
+  size_t nstrings = 0;
+  split(s, &strings, &nstrings);
+
+  for (size_t i = 0; g_event_engine == NULL && i < nstrings; i++) {
+    try_engine(strings[i]);
+  }
+
+  for (size_t i = 0; i < nstrings; i++) {
+    gpr_free(strings[i]);
+  }
+  gpr_free(strings);
+  gpr_free(s);
+
+  if (g_event_engine == NULL) {
+    gpr_log(GPR_ERROR, "No event engine could be initialized");
+    abort();
   }
-  gpr_log(GPR_ERROR, "No event engine could be initialized");
-  abort();
 }
 
-void grpc_event_engine_shutdown(void) { g_event_engine->shutdown_engine(); }
+void grpc_event_engine_shutdown(void) {
+  g_event_engine->shutdown_engine();
+  g_event_engine = NULL;
+}
 
 grpc_fd *grpc_fd_create(int fd, const char *name) {
   return g_event_engine->fd_create(fd, name);
@@ -72,6 +161,10 @@ void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   g_event_engine->fd_shutdown(exec_ctx, fd);
 }
 
+bool grpc_fd_is_shutdown(grpc_fd *fd) {
+  return g_event_engine->fd_is_shutdown(fd);
+}
+
 void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                             grpc_closure *closure) {
   g_event_engine->fd_notify_on_read(exec_ctx, fd, closure);
@@ -101,15 +194,15 @@ void grpc_pollset_destroy(grpc_pollset *pollset) {
   g_event_engine->pollset_destroy(pollset);
 }
 
-void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker **worker, gpr_timespec now,
-                       gpr_timespec deadline) {
-  g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline);
+grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                              grpc_pollset_worker **worker, gpr_timespec now,
+                              gpr_timespec deadline) {
+  return g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline);
 }
 
-void grpc_pollset_kick(grpc_pollset *pollset,
-                       grpc_pollset_worker *specific_worker) {
-  g_event_engine->pollset_kick(pollset, specific_worker);
+grpc_error *grpc_pollset_kick(grpc_pollset *pollset,
+                              grpc_pollset_worker *specific_worker) {
+  return g_event_engine->pollset_kick(pollset, specific_worker);
 }
 
 void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -159,6 +252,6 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
   g_event_engine->pollset_set_del_fd(exec_ctx, pollset_set, fd);
 }
 
-void grpc_kick_poller(void) { g_event_engine->kick_poller(); }
+grpc_error *grpc_kick_poller(void) { return g_event_engine->kick_poller(); }
 
 #endif  // GPR_POSIX_SOCKET
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 1fa9f5ef2d..579c84ef70 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -55,17 +55,20 @@ typedef struct grpc_event_engine_vtable {
                             grpc_closure *closure);
   void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
+  bool (*fd_is_shutdown)(grpc_fd *fd);
+  grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
 
   void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu);
   void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                            grpc_closure *closure);
   void (*pollset_reset)(grpc_pollset *pollset);
   void (*pollset_destroy)(grpc_pollset *pollset);
-  void (*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker **worker, gpr_timespec now,
-                       gpr_timespec deadline);
-  void (*pollset_kick)(grpc_pollset *pollset,
-                       grpc_pollset_worker *specific_worker);
+  grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                              grpc_pollset_worker **worker, gpr_timespec now,
+                              gpr_timespec deadline);
+  grpc_error *(*pollset_kick)(grpc_pollset *pollset,
+                              grpc_pollset_worker *specific_worker);
   void (*pollset_add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                          struct grpc_fd *fd);
 
@@ -88,7 +91,7 @@ typedef struct grpc_event_engine_vtable {
   void (*pollset_set_del_fd)(grpc_exec_ctx *exec_ctx,
                              grpc_pollset_set *pollset_set, grpc_fd *fd);
 
-  void (*kick_poller)(void);
+  grpc_error *(*kick_poller)(void);
 
   void (*shutdown_engine)(void);
 } grpc_event_engine_vtable;
@@ -96,6 +99,9 @@ typedef struct grpc_event_engine_vtable {
 void grpc_event_engine_init(void);
 void grpc_event_engine_shutdown(void);
 
+/* Return the name of the poll strategy */
+const char *grpc_get_poll_strategy_name();
+
 /* Create a wrapped file descriptor.
    Requires fd is a non-blocking file descriptor.
    This takes ownership of closing fd. */
@@ -114,7 +120,10 @@ int grpc_fd_wrapped_fd(grpc_fd *fd);
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
                     int *release_fd, const char *reason);
 
-/* Cause any current callbacks to error out with GRPC_CALLBACK_CANCELLED. */
+/* Has grpc_fd_shutdown been called on an fd? */
+bool grpc_fd_is_shutdown(grpc_fd *fd);
+
+/* Cause any current and future callbacks to fail. */
 void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd);
 
 /* Register read interest, causing read_cb to be called once when fd becomes
@@ -137,6 +146,10 @@ void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
 
+/* Return the read notifier pollset from the fd */
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
+
 /* pollset_posix functions */
 
 /* Add an fd to a pollset */
diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c
index 2146c7dd1f..c44aafcddf 100644
--- a/src/core/lib/iomgr/exec_ctx.c
+++ b/src/core/lib/iomgr/exec_ctx.c
@@ -39,6 +39,22 @@
 
 #include "src/core/lib/profiling/timers.h"
 
+bool grpc_exec_ctx_ready_to_finish(grpc_exec_ctx *exec_ctx) {
+  if (!exec_ctx->cached_ready_to_finish) {
+    exec_ctx->cached_ready_to_finish = exec_ctx->check_ready_to_finish(
+        exec_ctx, exec_ctx->check_ready_to_finish_arg);
+  }
+  return exec_ctx->cached_ready_to_finish;
+}
+
+bool grpc_never_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) {
+  return false;
+}
+
+bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) {
+  return true;
+}
+
 #ifndef GRPC_EXECUTION_CONTEXT_SANITIZER
 bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
   bool did_something = 0;
@@ -47,11 +63,12 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
     grpc_closure *c = exec_ctx->closure_list.head;
     exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL;
     while (c != NULL) {
-      bool success = (bool)(c->final_data & 1);
-      grpc_closure *next = (grpc_closure *)(c->final_data & ~(uintptr_t)1);
+      grpc_closure *next = c->next_data.next;
+      grpc_error *error = c->error;
       did_something = true;
       GPR_TIMER_BEGIN("grpc_exec_ctx_flush.cb", 0);
-      c->cb(exec_ctx, c->cb_arg, success);
+      c->cb(exec_ctx, c->cb_arg, error);
+      GRPC_ERROR_UNREF(error);
       GPR_TIMER_END("grpc_exec_ctx_flush.cb", 0);
       c = next;
     }
@@ -61,14 +78,15 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
 }
 
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {
+  exec_ctx->cached_ready_to_finish = true;
   grpc_exec_ctx_flush(exec_ctx);
 }
 
-void grpc_exec_ctx_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                           bool success,
-                           grpc_workqueue *offload_target_or_null) {
+void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                         grpc_error *error,
+                         grpc_workqueue *offload_target_or_null) {
   GPR_ASSERT(offload_target_or_null == NULL);
-  grpc_closure_list_add(&exec_ctx->closure_list, closure, success);
+  grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
 }
 
 void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h
index 976cc40347..38f27d9b13 100644
--- a/src/core/lib/iomgr/exec_ctx.h
+++ b/src/core/lib/iomgr/exec_ctx.h
@@ -53,6 +53,9 @@ typedef struct grpc_workqueue grpc_workqueue;
  *  - track a list of work that needs to be delayed until the top of the
  *    call stack (this provides a convenient mechanism to run callbacks
  *    without worrying about locking issues)
+ *  - provide a decision maker (via grpc_exec_ctx_ready_to_finish) that provides
+ *    signal as to whether a borrowed thread should continue to do work or
+ *    should actively try to finish up and get this thread back to its owner
  *
  *  CONVENTIONS:
  *  Instance of this must ALWAYS be constructed on the stack, never
@@ -63,18 +66,26 @@ typedef struct grpc_workqueue grpc_workqueue;
  */
 struct grpc_exec_ctx {
   grpc_closure_list closure_list;
+  bool cached_ready_to_finish;
+  void *check_ready_to_finish_arg;
+  bool (*check_ready_to_finish)(grpc_exec_ctx *exec_ctx, void *arg);
 };
 
-#define GRPC_EXEC_CTX_INIT \
-  { GRPC_CLOSURE_LIST_INIT }
+#define GRPC_EXEC_CTX_INIT_WITH_FINISH_CHECK(finish_check, finish_check_arg) \
+  { GRPC_CLOSURE_LIST_INIT, false, finish_check_arg, finish_check }
 #else
 struct grpc_exec_ctx {
-  int unused;
+  bool cached_ready_to_finish;
+  void *check_ready_to_finish_arg;
+  bool (*check_ready_to_finish)(grpc_exec_ctx *exec_ctx, void *arg);
 };
-#define GRPC_EXEC_CTX_INIT \
-  { 0 }
+#define GRPC_EXEC_CTX_INIT_WITH_FINISH_CHECK(finish_check, finish_check_arg) \
+  { false, finish_check_arg, finish_check }
 #endif
 
+#define GRPC_EXEC_CTX_INIT \
+  GRPC_EXEC_CTX_INIT_WITH_FINISH_CHECK(grpc_never_ready_to_finish, NULL)
+
 /** Flush any work that has been enqueued onto this grpc_exec_ctx.
  *  Caller must guarantee that no interfering locks are held.
  *  Returns true if work was performed, false otherwise. */
@@ -83,9 +94,17 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx);
  *  the instance is destroyed, or work may be lost. */
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx);
 /** Add a closure to be executed at the next flush/finish point */
-void grpc_exec_ctx_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                           bool success,
-                           grpc_workqueue *offload_target_or_null);
+void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                         grpc_error *error,
+                         grpc_workqueue *offload_target_or_null);
+/** Returns true if we'd like to leave this execution context as soon as
+    possible: useful for deciding whether to do something more or not depending
+    on outside context */
+bool grpc_exec_ctx_ready_to_finish(grpc_exec_ctx *exec_ctx);
+/** A finish check that is never ready to finish */
+bool grpc_never_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored);
+/** A finish check that is always ready to finish */
+bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored);
 /** Add a list of closures to be executed at the next flush/finish point.
  *  Leaves \a list empty. */
 void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c
index 36e22e4271..8d7535d6fe 100644
--- a/src/core/lib/iomgr/executor.c
+++ b/src/core/lib/iomgr/executor.c
@@ -112,10 +112,10 @@ static void maybe_spawn_locked() {
   g_executor.pending_join = 1;
 }
 
-void grpc_executor_enqueue(grpc_closure *closure, bool success) {
+void grpc_executor_push(grpc_closure *closure, grpc_error *error) {
   gpr_mu_lock(&g_executor.mu);
   if (g_executor.shutting_down == 0) {
-    grpc_closure_list_add(&g_executor.closures, closure, success);
+    grpc_closure_list_append(&g_executor.closures, closure, error);
     maybe_spawn_locked();
   }
   gpr_mu_unlock(&g_executor.mu);
diff --git a/src/core/lib/iomgr/executor.h b/src/core/lib/iomgr/executor.h
index b7e6f51aa5..da9dcd07d0 100644
--- a/src/core/lib/iomgr/executor.h
+++ b/src/core/lib/iomgr/executor.h
@@ -45,7 +45,7 @@ void grpc_executor_init();
 
 /** Enqueue \a closure for its eventual execution of \a f(arg) on a separate
  * thread */
-void grpc_executor_enqueue(grpc_closure *closure, bool success);
+void grpc_executor_push(grpc_closure *closure, grpc_error *error);
 
 /** Shutdown the executor, running all pending work as part of the call */
 void grpc_executor_shutdown();
diff --git a/src/core/lib/iomgr/iocp_windows.c b/src/core/lib/iomgr/iocp_windows.c
index d46558ab1b..2532e52e48 100644
--- a/src/core/lib/iomgr/iocp_windows.c
+++ b/src/core/lib/iomgr/iocp_windows.c
@@ -39,7 +39,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/thd.h>
 
 #include "src/core/lib/iomgr/iocp_windows.h"
@@ -104,7 +104,6 @@ grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx,
   } else if (overlapped == &socket->read_info.overlapped) {
     info = &socket->read_info;
   } else {
-    gpr_log(GPR_ERROR, "Unknown IOCP operation");
     abort();
   }
   success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes,
@@ -112,16 +111,7 @@ grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx,
   info->bytes_transfered = bytes;
   info->wsa_error = success ? 0 : WSAGetLastError();
   GPR_ASSERT(overlapped == &info->overlapped);
-  GPR_ASSERT(!info->has_pending_iocp);
-  gpr_mu_lock(&socket->state_mu);
-  if (info->closure) {
-    closure = info->closure;
-    info->closure = NULL;
-  } else {
-    info->has_pending_iocp = 1;
-  }
-  gpr_mu_unlock(&socket->state_mu);
-  grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
+  grpc_socket_become_ready(exec_ctx, socket, info);
   return GRPC_IOCP_WORK_WORK;
 }
 
@@ -176,33 +166,4 @@ void grpc_iocp_add_socket(grpc_winsocket *socket) {
   GPR_ASSERT(ret == g_iocp);
 }
 
-/* Calling notify_on_read or write means either of two things:
-   -) The IOCP already completed in the background, and we need to call
-   the callback now.
-   -) The IOCP hasn't completed yet, and we're queuing it for later. */
-static void socket_notify_on_iocp(grpc_exec_ctx *exec_ctx,
-                                  grpc_winsocket *socket, grpc_closure *closure,
-                                  grpc_winsocket_callback_info *info) {
-  GPR_ASSERT(info->closure == NULL);
-  gpr_mu_lock(&socket->state_mu);
-  if (info->has_pending_iocp) {
-    info->has_pending_iocp = 0;
-    grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
-  } else {
-    info->closure = closure;
-  }
-  gpr_mu_unlock(&socket->state_mu);
-}
-
-void grpc_socket_notify_on_write(grpc_exec_ctx *exec_ctx,
-                                 grpc_winsocket *socket,
-                                 grpc_closure *closure) {
-  socket_notify_on_iocp(exec_ctx, socket, closure, &socket->write_info);
-}
-
-void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket,
-                                grpc_closure *closure) {
-  socket_notify_on_iocp(exec_ctx, socket, closure, &socket->read_info);
-}
-
 #endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/lib/iomgr/iocp_windows.h b/src/core/lib/iomgr/iocp_windows.h
index ae210fa7d7..011fee38ff 100644
--- a/src/core/lib/iomgr/iocp_windows.h
+++ b/src/core/lib/iomgr/iocp_windows.h
@@ -52,12 +52,4 @@ void grpc_iocp_flush(void);
 void grpc_iocp_shutdown(void);
 void grpc_iocp_add_socket(grpc_winsocket *);
 
-void grpc_socket_notify_on_write(grpc_exec_ctx *exec_ctx,
-                                 grpc_winsocket *winsocket,
-                                 grpc_closure *closure);
-
-void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx,
-                                grpc_winsocket *winsocket,
-                                grpc_closure *closure);
-
 #endif /* GRPC_CORE_LIB_IOMGR_IOCP_WINDOWS_H */
diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c
index 60cef8ba77..89292a153e 100644
--- a/src/core/lib/iomgr/iomgr.c
+++ b/src/core/lib/iomgr/iomgr.c
@@ -96,7 +96,8 @@ void grpc_iomgr_shutdown(void) {
             gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), last_warning_time),
             gpr_time_from_seconds(1, GPR_TIMESPAN)) >= 0) {
       if (g_root_object.next != &g_root_object) {
-        gpr_log(GPR_DEBUG, "Waiting for %d iomgr objects to be destroyed",
+        gpr_log(GPR_DEBUG,
+                "Waiting for %" PRIuPTR " iomgr objects to be destroyed",
                 count_objects());
       }
       last_warning_time = gpr_now(GPR_CLOCK_REALTIME);
@@ -114,9 +115,9 @@ void grpc_iomgr_shutdown(void) {
       if (gpr_cv_wait(&g_rcv, &g_mu, short_deadline)) {
         if (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), shutdown_deadline) > 0) {
           if (g_root_object.next != &g_root_object) {
-            gpr_log(GPR_DEBUG,
-                    "Failed to free %d iomgr objects before shutdown deadline: "
-                    "memory leaks are likely",
+            gpr_log(GPR_DEBUG, "Failed to free %" PRIuPTR
+                               " iomgr objects before shutdown deadline: "
+                               "memory leaks are likely",
                     count_objects());
             dump_objects("LEAKED");
             if (grpc_iomgr_abort_on_leaks()) {
diff --git a/src/core/lib/iomgr/iomgr_posix.c b/src/core/lib/iomgr/iomgr_posix.c
index 016c501f75..cede97f4c6 100644
--- a/src/core/lib/iomgr/iomgr_posix.c
+++ b/src/core/lib/iomgr/iomgr_posix.c
@@ -41,12 +41,16 @@
 #include "src/core/lib/iomgr/tcp_posix.h"
 
 void grpc_iomgr_platform_init(void) {
+  grpc_wakeup_fd_global_init();
   grpc_event_engine_init();
   grpc_register_tracer("tcp", &grpc_tcp_trace);
 }
 
 void grpc_iomgr_platform_flush(void) {}
 
-void grpc_iomgr_platform_shutdown(void) { grpc_event_engine_shutdown(); }
+void grpc_iomgr_platform_shutdown(void) {
+  grpc_event_engine_shutdown();
+  grpc_wakeup_fd_global_destroy();
+}
 
 #endif /* GRPC_POSIX_SOCKET */
diff --git a/src/core/lib/iomgr/iomgr_windows.c b/src/core/lib/iomgr/iomgr_windows.c
index 398517fc75..7653f6e635 100644
--- a/src/core/lib/iomgr/iomgr_windows.c
+++ b/src/core/lib/iomgr/iomgr_windows.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 
 #include <grpc/support/log.h>
 
diff --git a/src/core/lib/support/load_file.c b/src/core/lib/iomgr/load_file.c
index f30aacdd4f..b62ecbc534 100644
--- a/src/core/lib/support/load_file.c
+++ b/src/core/lib/iomgr/load_file.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/support/load_file.h"
+#include "src/core/lib/iomgr/load_file.h"
 
 #include <errno.h>
 #include <string.h>
@@ -43,21 +43,19 @@
 #include "src/core/lib/support/block_annotate.h"
 #include "src/core/lib/support/string.h"
 
-gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
-                        int *success) {
+grpc_error *grpc_load_file(const char *filename, int add_null_terminator,
+                           gpr_slice *output) {
   unsigned char *contents = NULL;
   size_t contents_size = 0;
-  char *error_msg = NULL;
   gpr_slice result = gpr_empty_slice();
   FILE *file;
   size_t bytes_read = 0;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   GRPC_SCHEDULING_START_BLOCKING_REGION;
   file = fopen(filename, "rb");
   if (file == NULL) {
-    gpr_asprintf(&error_msg, "Could not open file %s (error = %s).", filename,
-                 strerror(errno));
-    GPR_ASSERT(error_msg != NULL);
+    error = GRPC_OS_ERROR(errno, "fopen");
     goto end;
   }
   fseek(file, 0, SEEK_END);
@@ -67,25 +65,25 @@ gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
   contents = gpr_malloc(contents_size + (add_null_terminator ? 1 : 0));
   bytes_read = fread(contents, 1, contents_size, file);
   if (bytes_read < contents_size) {
+    error = GRPC_OS_ERROR(errno, "fread");
     GPR_ASSERT(ferror(file));
-    gpr_asprintf(&error_msg, "Error %s occured while reading file %s.",
-                 strerror(errno), filename);
-    GPR_ASSERT(error_msg != NULL);
     goto end;
   }
-  if (success != NULL) *success = 1;
   if (add_null_terminator) {
     contents[contents_size++] = 0;
   }
   result = gpr_slice_new(contents, contents_size, gpr_free);
 
 end:
-  if (error_msg != NULL) {
-    gpr_log(GPR_ERROR, "%s", error_msg);
-    gpr_free(error_msg);
-    if (success != NULL) *success = 0;
-  }
+  *output = result;
   if (file != NULL) fclose(file);
+  if (error != GRPC_ERROR_NONE) {
+    grpc_error *error_out = grpc_error_set_str(
+        GRPC_ERROR_CREATE_REFERENCING("Failed to load file", &error, 1),
+        GRPC_ERROR_STR_FILENAME, filename);
+    GRPC_ERROR_UNREF(error);
+    error = error_out;
+  }
   GRPC_SCHEDULING_END_BLOCKING_REGION;
-  return result;
+  return error;
 }
diff --git a/src/core/lib/support/load_file.h b/src/core/lib/iomgr/load_file.h
index 9a4b27942e..9aac2225d1 100644
--- a/src/core/lib/support/load_file.h
+++ b/src/core/lib/iomgr/load_file.h
@@ -31,25 +31,26 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_LOAD_FILE_H
-#define GRPC_CORE_LIB_SUPPORT_LOAD_FILE_H
+#ifndef GRPC_CORE_LIB_IOMGR_LOAD_FILE_H
+#define GRPC_CORE_LIB_IOMGR_LOAD_FILE_H
 
 #include <stdio.h>
 
 #include <grpc/support/slice.h>
 
+#include "src/core/lib/iomgr/error.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* Loads the content of a file into a slice. add_null_terminator will add
-   a NULL terminator if non-zero. The success parameter, if not NULL,
-   will be set to 1 in case of success and 0 in case of failure. */
-gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
-                        int *success);
+   a NULL terminator if non-zero. */
+grpc_error *grpc_load_file(const char *filename, int add_null_terminator,
+                           gpr_slice *slice);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* GRPC_CORE_LIB_SUPPORT_LOAD_FILE_H */
+#endif /* GRPC_CORE_LIB_IOMGR_LOAD_FILE_H */
diff --git a/src/core/lib/iomgr/network_status_tracker.c b/src/core/lib/iomgr/network_status_tracker.c
new file mode 100644
index 0000000000..38a1c9b7d4
--- /dev/null
+++ b/src/core/lib/iomgr/network_status_tracker.c
@@ -0,0 +1,121 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include "src/core/lib/iomgr/endpoint.h"
+
+typedef struct endpoint_ll_node {
+  grpc_endpoint *ep;
+  struct endpoint_ll_node *next;
+} endpoint_ll_node;
+
+static endpoint_ll_node *head = NULL;
+static gpr_mu g_endpoint_mutex;
+static bool g_init_done = false;
+
+void grpc_initialize_network_status_monitor() {
+  g_init_done = true;
+  gpr_mu_init(&g_endpoint_mutex);
+  // TODO(makarandd): Install callback with OS to monitor network status.
+}
+
+void grpc_destroy_network_status_monitor() {
+  for (endpoint_ll_node *curr = head; curr != NULL;) {
+    endpoint_ll_node *next = curr->next;
+    gpr_free(curr);
+    curr = next;
+  }
+  gpr_mu_destroy(&g_endpoint_mutex);
+}
+
+void grpc_network_status_register_endpoint(grpc_endpoint *ep) {
+  if (!g_init_done) {
+    grpc_initialize_network_status_monitor();
+  }
+  gpr_mu_lock(&g_endpoint_mutex);
+  if (head == NULL) {
+    head = (endpoint_ll_node *)gpr_malloc(sizeof(endpoint_ll_node));
+    head->ep = ep;
+    head->next = NULL;
+  } else {
+    endpoint_ll_node *prev_head = head;
+    head = (endpoint_ll_node *)gpr_malloc(sizeof(endpoint_ll_node));
+    head->ep = ep;
+    head->next = prev_head;
+  }
+  gpr_mu_unlock(&g_endpoint_mutex);
+}
+
+void grpc_network_status_unregister_endpoint(grpc_endpoint *ep) {
+  gpr_mu_lock(&g_endpoint_mutex);
+  GPR_ASSERT(head);
+  bool found = false;
+  endpoint_ll_node *prev = head;
+  // if we're unregistering the head, just move head to the next
+  if (ep == head->ep) {
+    head = head->next;
+    gpr_free(prev);
+    found = true;
+  } else {
+    for (endpoint_ll_node *curr = head->next; curr != NULL; curr = curr->next) {
+      if (ep == curr->ep) {
+        prev->next = curr->next;
+        gpr_free(curr);
+        found = true;
+        break;
+      }
+      prev = curr;
+    }
+  }
+  gpr_mu_unlock(&g_endpoint_mutex);
+  GPR_ASSERT(found);
+}
+
+// Walk the linked-list from head and execute shutdown. It is possible that
+// other threads might be in the process of shutdown as well, but that has
+// no side effect since endpoint shutdown is idempotent.
+void grpc_network_status_shutdown_all_endpoints() {
+  gpr_mu_lock(&g_endpoint_mutex);
+  if (head == NULL) {
+    gpr_mu_unlock(&g_endpoint_mutex);
+    return;
+  }
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+  for (endpoint_ll_node *curr = head; curr != NULL; curr = curr->next) {
+    curr->ep->vtable->shutdown(&exec_ctx, curr->ep);
+  }
+  gpr_mu_unlock(&g_endpoint_mutex);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
diff --git a/src/core/lib/iomgr/network_status_tracker.h b/src/core/lib/iomgr/network_status_tracker.h
new file mode 100644
index 0000000000..74a1aa8135
--- /dev/null
+++ b/src/core/lib/iomgr/network_status_tracker.h
@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_NETWORK_STATUS_TRACKER_H
+#define GRPC_CORE_LIB_IOMGR_NETWORK_STATUS_TRACKER_H
+#include "src/core/lib/iomgr/endpoint.h"
+
+void grpc_network_status_register_endpoint(grpc_endpoint *ep);
+void grpc_network_status_unregister_endpoint(grpc_endpoint *ep);
+void grpc_network_status_shutdown_all_endpoints();
+#endif /* GRPC_CORE_LIB_IOMGR_NETWORK_STATUS_TRACKER_H */
diff --git a/src/core/lib/iomgr/polling_entity.c b/src/core/lib/iomgr/polling_entity.c
new file mode 100644
index 0000000000..d1686aa12f
--- /dev/null
+++ b/src/core/lib/iomgr/polling_entity.c
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/iomgr/polling_entity.h"
+
+grpc_polling_entity grpc_polling_entity_create_from_pollset_set(
+    grpc_pollset_set *pollset_set) {
+  grpc_polling_entity pollent;
+  pollent.pollent.pollset_set = pollset_set;
+  pollent.tag = POPS_POLLSET_SET;
+  return pollent;
+}
+
+grpc_polling_entity grpc_polling_entity_create_from_pollset(
+    grpc_pollset *pollset) {
+  grpc_polling_entity pollent;
+  pollent.pollent.pollset = pollset;
+  pollent.tag = POPS_POLLSET;
+  return pollent;
+}
+
+grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) {
+  if (pollent->tag == POPS_POLLSET) {
+    return pollent->pollent.pollset;
+  }
+  return NULL;
+}
+
+grpc_pollset_set *grpc_polling_entity_pollset_set(
+    grpc_polling_entity *pollent) {
+  if (pollent->tag == POPS_POLLSET_SET) {
+    return pollent->pollent.pollset_set;
+  }
+  return NULL;
+}
+
+bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) {
+  return pollent->tag == POPS_NONE;
+}
+
+void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
+                                            grpc_polling_entity *pollent,
+                                            grpc_pollset_set *pss_dst) {
+  if (pollent->tag == POPS_POLLSET) {
+    GPR_ASSERT(pollent->pollent.pollset != NULL);
+    grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset);
+  } else if (pollent->tag == POPS_POLLSET_SET) {
+    GPR_ASSERT(pollent->pollent.pollset_set != NULL);
+    grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst,
+                                     pollent->pollent.pollset_set);
+  } else {
+    gpr_log(GPR_ERROR, "Invalid grpc_polling_entity tag '%d'", pollent->tag);
+    abort();
+  }
+}
+
+void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx,
+                                              grpc_polling_entity *pollent,
+                                              grpc_pollset_set *pss_dst) {
+  if (pollent->tag == POPS_POLLSET) {
+    GPR_ASSERT(pollent->pollent.pollset != NULL);
+    grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset);
+  } else if (pollent->tag == POPS_POLLSET_SET) {
+    GPR_ASSERT(pollent->pollent.pollset_set != NULL);
+    grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst,
+                                     pollent->pollent.pollset_set);
+  } else {
+    gpr_log(GPR_ERROR, "Invalid grpc_polling_entity tag '%d'", pollent->tag);
+    abort();
+  }
+}
diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h
new file mode 100644
index 0000000000..e81531053c
--- /dev/null
+++ b/src/core/lib/iomgr/polling_entity.h
@@ -0,0 +1,81 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H
+#define GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H
+
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+
+/* A grpc_polling_entity is a pollset-or-pollset_set container. It allows
+ * functions that
+ * accept a pollset XOR a pollset_set to do so through an abstract interface.
+ * No ownership is taken. */
+
+typedef struct grpc_polling_entity {
+  union {
+    grpc_pollset *pollset;
+    grpc_pollset_set *pollset_set;
+  } pollent;
+  enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag;
+} grpc_polling_entity;
+
+grpc_polling_entity grpc_polling_entity_create_from_pollset_set(
+    grpc_pollset_set *pollset_set);
+grpc_polling_entity grpc_polling_entity_create_from_pollset(
+    grpc_pollset *pollset);
+
+/** If \a pollent contains a pollset, return it. Otherwise, return NULL */
+grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent);
+
+/** If \a pollent contains a pollset_set, return it. Otherwise, return NULL */
+grpc_pollset_set *grpc_polling_entity_pollset_set(grpc_polling_entity *pollent);
+
+bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent);
+
+/** Add the pollset or pollset_set in \a pollent to the destination pollset_set
+ * \a
+ * pss_dst */
+void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
+                                            grpc_polling_entity *pollent,
+                                            grpc_pollset_set *pss_dst);
+
+/** Delete the pollset or pollset_set in \a pollent from the destination
+ * pollset_set \a
+ * pss_dst */
+void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx,
+                                              grpc_polling_entity *pollent,
+                                              grpc_pollset_set *pss_dst);
+/* pollset_set specific */
+
+#endif /* GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H */
diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h
index c40a474877..8d9edc8406 100644
--- a/src/core/lib/iomgr/pollset.h
+++ b/src/core/lib/iomgr/pollset.h
@@ -81,14 +81,15 @@ void grpc_pollset_destroy(grpc_pollset *pollset);
    May call grpc_closure_list_run on grpc_closure_list, without holding the
    pollset
    lock */
-void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker **worker, gpr_timespec now,
-                       gpr_timespec deadline);
+grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                              grpc_pollset_worker **worker, gpr_timespec now,
+                              gpr_timespec deadline) GRPC_MUST_USE_RESULT;
 
 /* Break one polling thread out of polling work for this pollset.
    If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers.
    Otherwise, if specific_worker is non-NULL, then kick that worker. */
-void grpc_pollset_kick(grpc_pollset *pollset,
-                       grpc_pollset_worker *specific_worker);
+grpc_error *grpc_pollset_kick(grpc_pollset *pollset,
+                              grpc_pollset_worker *specific_worker)
+    GRPC_MUST_USE_RESULT;
 
 #endif /* GRPC_CORE_LIB_IOMGR_POLLSET_H */
diff --git a/src/core/lib/iomgr/pollset_set_windows.c b/src/core/lib/iomgr/pollset_set_windows.c
index 89f60b92fb..a35a9766fc 100644
--- a/src/core/lib/iomgr/pollset_set_windows.c
+++ b/src/core/lib/iomgr/pollset_set_windows.c
@@ -32,12 +32,15 @@
  */
 
 #include <grpc/support/port_platform.h>
+#include <stdint.h>
 
 #ifdef GPR_WINSOCK_SOCKET
 
 #include "src/core/lib/iomgr/pollset_set_windows.h"
 
-grpc_pollset_set* grpc_pollset_set_create(void) { return NULL; }
+grpc_pollset_set* grpc_pollset_set_create(void) {
+  return (grpc_pollset_set*)((intptr_t)0xdeafbeef);
+}
 
 void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {}
 
diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c
index bff5c586f8..626dd784b3 100644
--- a/src/core/lib/iomgr/pollset_windows.c
+++ b/src/core/lib/iomgr/pollset_windows.c
@@ -109,7 +109,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   pollset->shutting_down = 1;
   grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
   if (!pollset->is_iocp_worker) {
-    grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
   } else {
     pollset->on_shutdown = closure;
   }
@@ -127,9 +127,9 @@ void grpc_pollset_reset(grpc_pollset *pollset) {
   pollset->on_shutdown = NULL;
 }
 
-void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker **worker_hdl, gpr_timespec now,
-                       gpr_timespec deadline) {
+grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                              grpc_pollset_worker **worker_hdl,
+                              gpr_timespec now, gpr_timespec deadline) {
   grpc_pollset_worker worker;
   *worker_hdl = &worker;
 
@@ -167,7 +167,8 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
 
       if (pollset->shutting_down && pollset->on_shutdown != NULL) {
-        grpc_exec_ctx_enqueue(exec_ctx, pollset->on_shutdown, true, NULL);
+        grpc_exec_ctx_sched(exec_ctx, pollset->on_shutdown, GRPC_ERROR_NONE,
+                            NULL);
         pollset->on_shutdown = NULL;
       }
       goto done;
@@ -197,9 +198,11 @@ done:
   }
   gpr_cv_destroy(&worker.cv);
   *worker_hdl = NULL;
+  return GRPC_ERROR_NONE;
 }
 
-void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
+grpc_error *grpc_pollset_kick(grpc_pollset *p,
+                              grpc_pollset_worker *specific_worker) {
   if (specific_worker != NULL) {
     if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
       for (specific_worker =
@@ -233,6 +236,7 @@ void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
       p->kicked_without_pollers = 1;
     }
   }
+  return GRPC_ERROR_NONE;
 }
 
 void grpc_kick_poller(void) { grpc_iocp_kick(); }
diff --git a/src/core/lib/iomgr/resolve_address.h b/src/core/lib/iomgr/resolve_address.h
index ef198fe0f6..ddbe375755 100644
--- a/src/core/lib/iomgr/resolve_address.h
+++ b/src/core/lib/iomgr/resolve_address.h
@@ -50,24 +50,20 @@ typedef struct {
   grpc_resolved_address *addrs;
 } grpc_resolved_addresses;
 
-/* Async result callback:
-   On success: addresses is the result, and the callee must call
-   grpc_resolved_addresses_destroy when it's done with them
-   On failure: addresses is NULL */
-typedef void (*grpc_resolve_cb)(grpc_exec_ctx *exec_ctx, void *arg,
-                                grpc_resolved_addresses *addresses);
 /* Asynchronously resolve addr. Use default_port if a port isn't designated
    in addr, otherwise use the port in addr. */
 /* TODO(ctiller): add a timeout here */
 extern void (*grpc_resolve_address)(grpc_exec_ctx *exec_ctx, const char *addr,
                                     const char *default_port,
-                                    grpc_resolve_cb cb, void *arg);
+                                    grpc_closure *on_done,
+                                    grpc_resolved_addresses **addresses);
 /* Destroy resolved addresses */
 void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addresses);
 
 /* Resolve addr in a blocking fashion. Returns NULL on failure. On success,
    result must be freed with grpc_resolved_addresses_destroy. */
-extern grpc_resolved_addresses *(*grpc_blocking_resolve_address)(
-    const char *name, const char *default_port);
+extern grpc_error *(*grpc_blocking_resolve_address)(
+    const char *name, const char *default_port,
+    grpc_resolved_addresses **addresses);
 
 #endif /* GRPC_CORE_LIB_IOMGR_RESOLVE_ADDRESS_H */
diff --git a/src/core/lib/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c
index cae91eec20..4e9f978584 100644
--- a/src/core/lib/iomgr/resolve_address_posix.c
+++ b/src/core/lib/iomgr/resolve_address_posix.c
@@ -54,38 +54,33 @@
 #include "src/core/lib/support/block_annotate.h"
 #include "src/core/lib/support/string.h"
 
-typedef struct {
-  char *name;
-  char *default_port;
-  grpc_resolve_cb cb;
-  grpc_closure request_closure;
-  void *arg;
-} request;
-
-static grpc_resolved_addresses *blocking_resolve_address_impl(
-    const char *name, const char *default_port) {
+static grpc_error *blocking_resolve_address_impl(
+    const char *name, const char *default_port,
+    grpc_resolved_addresses **addresses) {
   struct addrinfo hints;
   struct addrinfo *result = NULL, *resp;
   char *host;
   char *port;
   int s;
   size_t i;
-  grpc_resolved_addresses *addrs = NULL;
+  grpc_error *err;
 
   if (name[0] == 'u' && name[1] == 'n' && name[2] == 'i' && name[3] == 'x' &&
       name[4] == ':' && name[5] != 0) {
-    return grpc_resolve_unix_domain_address(name + 5);
+    return grpc_resolve_unix_domain_address(name + 5, addresses);
   }
 
   /* parse name, splitting it into host and port parts */
   gpr_split_host_port(name, &host, &port);
   if (host == NULL) {
-    gpr_log(GPR_ERROR, "unparseable host:port: '%s'", name);
+    err = grpc_error_set_str(GRPC_ERROR_CREATE("unparseable host:port"),
+                             GRPC_ERROR_STR_TARGET_ADDRESS, name);
     goto done;
   }
   if (port == NULL) {
     if (default_port == NULL) {
-      gpr_log(GPR_ERROR, "no port in name '%s'", name);
+      err = grpc_error_set_str(GRPC_ERROR_CREATE("no port in name"),
+                               GRPC_ERROR_STR_TARGET_ADDRESS, name);
       goto done;
     }
     port = gpr_strdup(default_port);
@@ -115,23 +110,31 @@ static grpc_resolved_addresses *blocking_resolve_address_impl(
   }
 
   if (s != 0) {
-    gpr_log(GPR_ERROR, "getaddrinfo: %s", gai_strerror(s));
+    err = grpc_error_set_str(
+        grpc_error_set_str(
+            grpc_error_set_str(grpc_error_set_int(GRPC_ERROR_CREATE("OS Error"),
+                                                  GRPC_ERROR_INT_ERRNO, s),
+                               GRPC_ERROR_STR_OS_ERROR, gai_strerror(s)),
+            GRPC_ERROR_STR_SYSCALL, "getaddrinfo"),
+        GRPC_ERROR_STR_TARGET_ADDRESS, name);
     goto done;
   }
 
   /* Success path: set addrs non-NULL, fill it in */
-  addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
-  addrs->naddrs = 0;
+  *addresses = gpr_malloc(sizeof(grpc_resolved_addresses));
+  (*addresses)->naddrs = 0;
   for (resp = result; resp != NULL; resp = resp->ai_next) {
-    addrs->naddrs++;
+    (*addresses)->naddrs++;
   }
-  addrs->addrs = gpr_malloc(sizeof(grpc_resolved_address) * addrs->naddrs);
+  (*addresses)->addrs =
+      gpr_malloc(sizeof(grpc_resolved_address) * (*addresses)->naddrs);
   i = 0;
   for (resp = result; resp != NULL; resp = resp->ai_next) {
-    memcpy(&addrs->addrs[i].addr, resp->ai_addr, resp->ai_addrlen);
-    addrs->addrs[i].len = resp->ai_addrlen;
+    memcpy(&(*addresses)->addrs[i].addr, resp->ai_addr, resp->ai_addrlen);
+    (*addresses)->addrs[i].len = resp->ai_addrlen;
     i++;
   }
+  err = GRPC_ERROR_NONE;
 
 done:
   gpr_free(host);
@@ -139,45 +142,59 @@ done:
   if (result) {
     freeaddrinfo(result);
   }
-  return addrs;
+  return err;
 }
 
-grpc_resolved_addresses *(*grpc_blocking_resolve_address)(
-    const char *name, const char *default_port) = blocking_resolve_address_impl;
+grpc_error *(*grpc_blocking_resolve_address)(
+    const char *name, const char *default_port,
+    grpc_resolved_addresses **addresses) = blocking_resolve_address_impl;
+
+typedef struct {
+  char *name;
+  char *default_port;
+  grpc_closure *on_done;
+  grpc_resolved_addresses **addrs_out;
+  grpc_closure request_closure;
+  void *arg;
+} request;
 
 /* Callback to be passed to grpc_executor to asynch-ify
  * grpc_blocking_resolve_address */
-static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) {
+static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp,
+                              grpc_error *error) {
   request *r = rp;
-  grpc_resolved_addresses *resolved =
-      grpc_blocking_resolve_address(r->name, r->default_port);
-  void *arg = r->arg;
-  grpc_resolve_cb cb = r->cb;
+  grpc_exec_ctx_sched(
+      exec_ctx, r->on_done,
+      grpc_blocking_resolve_address(r->name, r->default_port, r->addrs_out),
+      NULL);
   gpr_free(r->name);
   gpr_free(r->default_port);
-  cb(exec_ctx, arg, resolved);
   gpr_free(r);
 }
 
 void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {
-  gpr_free(addrs->addrs);
+  if (addrs != NULL) {
+    gpr_free(addrs->addrs);
+  }
   gpr_free(addrs);
 }
 
 static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name,
-                                 const char *default_port, grpc_resolve_cb cb,
-                                 void *arg) {
+                                 const char *default_port,
+                                 grpc_closure *on_done,
+                                 grpc_resolved_addresses **addrs) {
   request *r = gpr_malloc(sizeof(request));
   grpc_closure_init(&r->request_closure, do_request_thread, r);
   r->name = gpr_strdup(name);
   r->default_port = gpr_strdup(default_port);
-  r->cb = cb;
-  r->arg = arg;
-  grpc_executor_enqueue(&r->request_closure, 1);
+  r->on_done = on_done;
+  r->addrs_out = addrs;
+  grpc_executor_push(&r->request_closure, GRPC_ERROR_NONE);
 }
 
 void (*grpc_resolve_address)(grpc_exec_ctx *exec_ctx, const char *name,
-                             const char *default_port, grpc_resolve_cb cb,
-                             void *arg) = resolve_address_impl;
+                             const char *default_port, grpc_closure *on_done,
+                             grpc_resolved_addresses **addrs) =
+    resolve_address_impl;
 
 #endif
diff --git a/src/core/lib/iomgr/resolve_address_windows.c b/src/core/lib/iomgr/resolve_address_windows.c
index 914736234d..2af8af82dc 100644
--- a/src/core/lib/iomgr/resolve_address_windows.c
+++ b/src/core/lib/iomgr/resolve_address_windows.c
@@ -43,7 +43,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
@@ -56,30 +56,37 @@
 typedef struct {
   char *name;
   char *default_port;
-  grpc_resolve_cb cb;
   grpc_closure request_closure;
-  void *arg;
+  grpc_closure *on_done;
+  grpc_resolved_addresses **addresses;
 } request;
 
-static grpc_resolved_addresses *blocking_resolve_address_impl(
-    const char *name, const char *default_port) {
+static grpc_error *blocking_resolve_address_impl(
+    const char *name, const char *default_port,
+    grpc_resolved_addresses **addresses) {
   struct addrinfo hints;
   struct addrinfo *result = NULL, *resp;
   char *host;
   char *port;
   int s;
   size_t i;
-  grpc_resolved_addresses *addrs = NULL;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   /* parse name, splitting it into host and port parts */
   gpr_split_host_port(name, &host, &port);
   if (host == NULL) {
-    gpr_log(GPR_ERROR, "unparseable host:port: '%s'", name);
+    char *msg;
+    gpr_asprintf(&msg, "unparseable host:port: '%s'", name);
+    error = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
     goto done;
   }
   if (port == NULL) {
     if (default_port == NULL) {
-      gpr_log(GPR_ERROR, "no port in name '%s'", name);
+      char *msg;
+      gpr_asprintf(&msg, "no port in name '%s'", name);
+      error = GRPC_ERROR_CREATE(msg);
+      gpr_free(msg);
       goto done;
     }
     port = gpr_strdup(default_port);
@@ -95,31 +102,30 @@ static grpc_resolved_addresses *blocking_resolve_address_impl(
   s = getaddrinfo(host, port, &hints, &result);
   GRPC_SCHEDULING_END_BLOCKING_REGION;
   if (s != 0) {
-    char *error_message = gpr_format_message(s);
-    gpr_log(GPR_ERROR, "getaddrinfo: %s", error_message);
-    gpr_free(error_message);
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "getaddrinfo");
     goto done;
   }
 
   /* Success path: set addrs non-NULL, fill it in */
-  addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
-  addrs->naddrs = 0;
+  (*addresses) = gpr_malloc(sizeof(grpc_resolved_addresses));
+  (*addresses)->naddrs = 0;
   for (resp = result; resp != NULL; resp = resp->ai_next) {
-    addrs->naddrs++;
+    (*addresses)->naddrs++;
   }
-  addrs->addrs = gpr_malloc(sizeof(grpc_resolved_address) * addrs->naddrs);
+  (*addresses)->addrs =
+      gpr_malloc(sizeof(grpc_resolved_address) * (*addresses)->naddrs);
   i = 0;
   for (resp = result; resp != NULL; resp = resp->ai_next) {
-    memcpy(&addrs->addrs[i].addr, resp->ai_addr, resp->ai_addrlen);
-    addrs->addrs[i].len = resp->ai_addrlen;
+    memcpy(&(*addresses)->addrs[i].addr, resp->ai_addr, resp->ai_addrlen);
+    (*addresses)->addrs[i].len = resp->ai_addrlen;
     i++;
   }
 
   {
-    for (i = 0; i < addrs->naddrs; i++) {
+    for (i = 0; i < (*addresses)->naddrs; i++) {
       char *buf;
-      grpc_sockaddr_to_string(&buf, (struct sockaddr *)&addrs->addrs[i].addr,
-                              0);
+      grpc_sockaddr_to_string(
+          &buf, (struct sockaddr *)&(*addresses)->addrs[i].addr, 0);
       gpr_free(buf);
     }
   }
@@ -130,45 +136,53 @@ done:
   if (result) {
     freeaddrinfo(result);
   }
-  return addrs;
+  return error;
 }
 
-grpc_resolved_addresses *(*grpc_blocking_resolve_address)(
-    const char *name, const char *default_port) = blocking_resolve_address_impl;
+grpc_error *(*grpc_blocking_resolve_address)(
+    const char *name, const char *default_port,
+    grpc_resolved_addresses **addresses) = blocking_resolve_address_impl;
 
 /* Callback to be passed to grpc_executor to asynch-ify
  * grpc_blocking_resolve_address */
-static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) {
+static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp,
+                              grpc_error *error) {
   request *r = rp;
-  grpc_resolved_addresses *resolved =
-      grpc_blocking_resolve_address(r->name, r->default_port);
-  void *arg = r->arg;
-  grpc_resolve_cb cb = r->cb;
+  if (error == GRPC_ERROR_NONE) {
+    error =
+        grpc_blocking_resolve_address(r->name, r->default_port, r->addresses);
+  } else {
+    GRPC_ERROR_REF(error);
+  }
+  grpc_exec_ctx_sched(exec_ctx, r->on_done, error, NULL);
   gpr_free(r->name);
   gpr_free(r->default_port);
-  cb(exec_ctx, arg, resolved);
   gpr_free(r);
 }
 
 void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {
-  gpr_free(addrs->addrs);
+  if (addrs != NULL) {
+    gpr_free(addrs->addrs);
+  }
   gpr_free(addrs);
 }
 
 static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name,
-                                 const char *default_port, grpc_resolve_cb cb,
-                                 void *arg) {
+                                 const char *default_port,
+                                 grpc_closure *on_done,
+                                 grpc_resolved_addresses **addresses) {
   request *r = gpr_malloc(sizeof(request));
   grpc_closure_init(&r->request_closure, do_request_thread, r);
   r->name = gpr_strdup(name);
   r->default_port = gpr_strdup(default_port);
-  r->cb = cb;
-  r->arg = arg;
-  grpc_executor_enqueue(&r->request_closure, 1);
+  r->on_done = on_done;
+  r->addresses = addresses;
+  grpc_executor_push(&r->request_closure, GRPC_ERROR_NONE);
 }
 
 void (*grpc_resolve_address)(grpc_exec_ctx *exec_ctx, const char *name,
-                             const char *default_port, grpc_resolve_cb cb,
-                             void *arg) = resolve_address_impl;
+                             const char *default_port, grpc_closure *on_done,
+                             grpc_resolved_addresses **addresses) =
+    resolve_address_impl;
 
 #endif
diff --git a/src/core/lib/iomgr/sockaddr.h b/src/core/lib/iomgr/sockaddr.h
index 891a2f094f..5563d0b8a6 100644
--- a/src/core/lib/iomgr/sockaddr.h
+++ b/src/core/lib/iomgr/sockaddr.h
@@ -36,8 +36,8 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#ifdef GPR_WINDOWS
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 #endif
 
 #ifdef GPR_POSIX_SOCKETADDR
diff --git a/src/core/lib/iomgr/sockaddr_win32.h b/src/core/lib/iomgr/sockaddr_windows.h
index 02aeae7619..971db5b32b 100644
--- a/src/core/lib/iomgr/sockaddr_win32.h
+++ b/src/core/lib/iomgr/sockaddr_windows.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H
-#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H
+#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H
+#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H
 
 #include <winsock2.h>
 #include <ws2tcpip.h>
@@ -40,4 +40,4 @@
 // must be included after the above
 #include <mswsock.h>
 
-#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H */
+#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H */
diff --git a/src/core/lib/iomgr/socket_utils_common_posix.c b/src/core/lib/iomgr/socket_utils_common_posix.c
index fa83ceef30..d2f6261e2a 100644
--- a/src/core/lib/iomgr/socket_utils_common_posix.c
+++ b/src/core/lib/iomgr/socket_utils_common_posix.c
@@ -49,6 +49,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
@@ -57,10 +58,10 @@
 #include "src/core/lib/support/string.h"
 
 /* set a socket to non blocking mode */
-int grpc_set_socket_nonblocking(int fd, int non_blocking) {
+grpc_error *grpc_set_socket_nonblocking(int fd, int non_blocking) {
   int oldflags = fcntl(fd, F_GETFL, 0);
   if (oldflags < 0) {
-    return 0;
+    return GRPC_OS_ERROR(errno, "fcntl");
   }
 
   if (non_blocking) {
@@ -70,52 +71,71 @@ int grpc_set_socket_nonblocking(int fd, int non_blocking) {
   }
 
   if (fcntl(fd, F_SETFL, oldflags) != 0) {
-    return 0;
+    return GRPC_OS_ERROR(errno, "fcntl");
   }
 
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
-int grpc_set_socket_no_sigpipe_if_possible(int fd) {
+grpc_error *grpc_set_socket_no_sigpipe_if_possible(int fd) {
 #ifdef GPR_HAVE_SO_NOSIGPIPE
   int val = 1;
   int newval;
   socklen_t intlen = sizeof(newval);
-  return 0 == setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &val, sizeof(val)) &&
-         0 == getsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &newval, &intlen) &&
-         (newval != 0) == val;
-#else
-  return 1;
+  if (0 != setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &val, sizeof(val))) {
+    return GRPC_OS_ERROR(errno, "setsockopt(SO_NOSIGPIPE)");
+  }
+  if (0 != getsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &newval, &intlen)) {
+    return GRPC_OS_ERROR(errno, "getsockopt(SO_NOSIGPIPE)");
+  }
+  if ((newval != 0) != (val != 0)) {
+    return GRPC_ERROR_CREATE("Failed to set SO_NOSIGPIPE");
+  }
 #endif
+  return GRPC_ERROR_NONE;
 }
 
-int grpc_set_socket_ip_pktinfo_if_possible(int fd) {
+grpc_error *grpc_set_socket_ip_pktinfo_if_possible(int fd) {
 #ifdef GPR_HAVE_IP_PKTINFO
   int get_local_ip = 1;
-  return 0 == setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &get_local_ip,
-                         sizeof(get_local_ip));
-#else
-  (void)fd;
-  return 1;
+  if (0 != setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &get_local_ip,
+                      sizeof(get_local_ip))) {
+    return GRPC_OS_ERROR(errno, "setsockopt(IP_PKTINFO)");
+  }
 #endif
+  return GRPC_ERROR_NONE;
 }
 
-int grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd) {
+grpc_error *grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd) {
 #ifdef GPR_HAVE_IPV6_RECVPKTINFO
   int get_local_ip = 1;
-  return 0 == setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &get_local_ip,
-                         sizeof(get_local_ip));
-#else
-  (void)fd;
-  return 1;
+  if (0 != setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &get_local_ip,
+                      sizeof(get_local_ip))) {
+    return GRPC_OS_ERROR(errno, "setsockopt(IPV6_RECVPKTINFO)");
+  }
 #endif
+  return GRPC_ERROR_NONE;
+}
+
+grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes) {
+  return 0 == setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffer_size_bytes,
+                         sizeof(buffer_size_bytes))
+             ? GRPC_ERROR_NONE
+             : GRPC_OS_ERROR(errno, "setsockopt(SO_SNDBUF)");
+}
+
+grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes) {
+  return 0 == setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &buffer_size_bytes,
+                         sizeof(buffer_size_bytes))
+             ? GRPC_ERROR_NONE
+             : GRPC_OS_ERROR(errno, "setsockopt(SO_RCVBUF)");
 }
 
 /* set a socket to close on exec */
-int grpc_set_socket_cloexec(int fd, int close_on_exec) {
+grpc_error *grpc_set_socket_cloexec(int fd, int close_on_exec) {
   int oldflags = fcntl(fd, F_GETFD, 0);
   if (oldflags < 0) {
-    return 0;
+    return GRPC_OS_ERROR(errno, "fcntl");
   }
 
   if (close_on_exec) {
@@ -125,30 +145,67 @@ int grpc_set_socket_cloexec(int fd, int close_on_exec) {
   }
 
   if (fcntl(fd, F_SETFD, oldflags) != 0) {
-    return 0;
+    return GRPC_OS_ERROR(errno, "fcntl");
   }
 
-  return 1;
+  return GRPC_ERROR_NONE;
 }
 
 /* set a socket to reuse old addresses */
-int grpc_set_socket_reuse_addr(int fd, int reuse) {
+grpc_error *grpc_set_socket_reuse_addr(int fd, int reuse) {
   int val = (reuse != 0);
   int newval;
   socklen_t intlen = sizeof(newval);
-  return 0 == setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) &&
-         0 == getsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &newval, &intlen) &&
-         (newval != 0) == val;
+  if (0 != setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val))) {
+    return GRPC_OS_ERROR(errno, "setsockopt(SO_REUSEADDR)");
+  }
+  if (0 != getsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &newval, &intlen)) {
+    return GRPC_OS_ERROR(errno, "getsockopt(SO_REUSEADDR)");
+  }
+  if ((newval != 0) != val) {
+    return GRPC_ERROR_CREATE("Failed to set SO_REUSEADDR");
+  }
+
+  return GRPC_ERROR_NONE;
+}
+
+/* set a socket to reuse old addresses */
+grpc_error *grpc_set_socket_reuse_port(int fd, int reuse) {
+#ifndef SO_REUSEPORT
+  return GRPC_ERROR_CREATE("SO_REUSEPORT unavailable on compiling system");
+#else
+  int val = (reuse != 0);
+  int newval;
+  socklen_t intlen = sizeof(newval);
+  if (0 != setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val))) {
+    return GRPC_OS_ERROR(errno, "setsockopt(SO_REUSEPORT)");
+  }
+  if (0 != getsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &newval, &intlen)) {
+    return GRPC_OS_ERROR(errno, "getsockopt(SO_REUSEPORT)");
+  }
+  if ((newval != 0) != val) {
+    return GRPC_ERROR_CREATE("Failed to set SO_REUSEPORT");
+  }
+
+  return GRPC_ERROR_NONE;
+#endif
 }
 
 /* disable nagle */
-int grpc_set_socket_low_latency(int fd, int low_latency) {
+grpc_error *grpc_set_socket_low_latency(int fd, int low_latency) {
   int val = (low_latency != 0);
   int newval;
   socklen_t intlen = sizeof(newval);
-  return 0 == setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) &&
-         0 == getsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &newval, &intlen) &&
-         (newval != 0) == val;
+  if (0 != setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val))) {
+    return GRPC_OS_ERROR(errno, "setsockopt(TCP_NODELAY)");
+  }
+  if (0 != getsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &newval, &intlen)) {
+    return GRPC_OS_ERROR(errno, "getsockopt(TCP_NODELAY)");
+  }
+  if ((newval != 0) != val) {
+    return GRPC_ERROR_CREATE("Failed to set TCP_NODELAY");
+  }
+  return GRPC_ERROR_NONE;
 }
 
 static gpr_once g_probe_ipv6_once = GPR_ONCE_INIT;
@@ -196,35 +253,47 @@ static int set_socket_dualstack(int fd) {
   }
 }
 
-int grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
-                                 int protocol, grpc_dualstack_mode *dsmode) {
+static grpc_error *error_for_fd(int fd, const struct sockaddr *addr) {
+  if (fd >= 0) return GRPC_ERROR_NONE;
+  char *addr_str;
+  grpc_sockaddr_to_string(&addr_str, addr, 0);
+  grpc_error *err = grpc_error_set_str(GRPC_OS_ERROR(errno, "socket"),
+                                       GRPC_ERROR_STR_TARGET_ADDRESS, addr_str);
+  gpr_free(addr_str);
+  return err;
+}
+
+grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
+                                         int protocol,
+                                         grpc_dualstack_mode *dsmode,
+                                         int *newfd) {
   int family = addr->sa_family;
   if (family == AF_INET6) {
-    int fd;
     if (grpc_ipv6_loopback_available()) {
-      fd = socket(family, type, protocol);
+      *newfd = socket(family, type, protocol);
     } else {
-      fd = -1;
+      *newfd = -1;
       errno = EAFNOSUPPORT;
     }
     /* Check if we've got a valid dualstack socket. */
-    if (fd >= 0 && set_socket_dualstack(fd)) {
+    if (*newfd >= 0 && set_socket_dualstack(*newfd)) {
       *dsmode = GRPC_DSMODE_DUALSTACK;
-      return fd;
+      return GRPC_ERROR_NONE;
     }
     /* If this isn't an IPv4 address, then return whatever we've got. */
     if (!grpc_sockaddr_is_v4mapped(addr, NULL)) {
       *dsmode = GRPC_DSMODE_IPV6;
-      return fd;
+      return error_for_fd(*newfd, addr);
     }
     /* Fall back to AF_INET. */
-    if (fd >= 0) {
-      close(fd);
+    if (*newfd >= 0) {
+      close(*newfd);
     }
     family = AF_INET;
   }
   *dsmode = family == AF_INET ? GRPC_DSMODE_IPV4 : GRPC_DSMODE_NONE;
-  return socket(family, type, protocol);
+  *newfd = socket(family, type, protocol);
+  return error_for_fd(*newfd, addr);
 }
 
 #endif
diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h
index a8f6e5e658..7bcc2219ae 100644
--- a/src/core/lib/iomgr/socket_utils_posix.h
+++ b/src/core/lib/iomgr/socket_utils_posix.h
@@ -37,21 +37,26 @@
 #include <sys/socket.h>
 #include <unistd.h>
 
+#include "src/core/lib/iomgr/error.h"
+
 /* a wrapper for accept or accept4 */
 int grpc_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen,
                  int nonblock, int cloexec);
 
 /* set a socket to non blocking mode */
-int grpc_set_socket_nonblocking(int fd, int non_blocking);
+grpc_error *grpc_set_socket_nonblocking(int fd, int non_blocking);
 
 /* set a socket to close on exec */
-int grpc_set_socket_cloexec(int fd, int close_on_exec);
+grpc_error *grpc_set_socket_cloexec(int fd, int close_on_exec);
 
 /* set a socket to reuse old addresses */
-int grpc_set_socket_reuse_addr(int fd, int reuse);
+grpc_error *grpc_set_socket_reuse_addr(int fd, int reuse);
 
 /* disable nagle */
-int grpc_set_socket_low_latency(int fd, int low_latency);
+grpc_error *grpc_set_socket_low_latency(int fd, int low_latency);
+
+/* set SO_REUSEPORT */
+grpc_error *grpc_set_socket_reuse_port(int fd, int reuse);
 
 /* Returns true if this system can create AF_INET6 sockets bound to ::1.
    The value is probed once, and cached for the life of the process.
@@ -64,19 +69,22 @@ int grpc_set_socket_low_latency(int fd, int low_latency);
 int grpc_ipv6_loopback_available(void);
 
 /* Tries to set SO_NOSIGPIPE if available on this platform.
-   Returns 1 on success, 0 on failure.
    If SO_NO_SIGPIPE is not available, returns 1. */
-int grpc_set_socket_no_sigpipe_if_possible(int fd);
+grpc_error *grpc_set_socket_no_sigpipe_if_possible(int fd);
 
 /* Tries to set IP_PKTINFO if available on this platform.
-   Returns 1 on success, 0 on failure.
    If IP_PKTINFO is not available, returns 1. */
-int grpc_set_socket_ip_pktinfo_if_possible(int fd);
+grpc_error *grpc_set_socket_ip_pktinfo_if_possible(int fd);
 
 /* Tries to set IPV6_RECVPKTINFO if available on this platform.
-   Returns 1 on success, 0 on failure.
    If IPV6_RECVPKTINFO is not available, returns 1. */
-int grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd);
+grpc_error *grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd);
+
+/* Tries to set the socket's send buffer to given size. */
+grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes);
+
+/* Tries to set the socket's receive buffer to given size. */
+grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes);
 
 /* An enum to keep track of IPv4/IPv6 socket modes.
 
@@ -117,7 +125,9 @@ extern int grpc_forbid_dualstack_sockets_for_testing;
      IPv4, so that bind() or connect() see the correct family.
    Also, it's important to distinguish between DUALSTACK and IPV6 when
    listening on the [::] wildcard address. */
-int grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
-                                 int protocol, grpc_dualstack_mode *dsmode);
+grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type,
+                                         int protocol,
+                                         grpc_dualstack_mode *dsmode,
+                                         int *newfd);
 
 #endif /* GRPC_CORE_LIB_IOMGR_SOCKET_UTILS_POSIX_H */
diff --git a/src/core/lib/iomgr/socket_windows.c b/src/core/lib/iomgr/socket_windows.c
index ebd77e0372..d7d5f6f157 100644
--- a/src/core/lib/iomgr/socket_windows.c
+++ b/src/core/lib/iomgr/socket_windows.c
@@ -42,7 +42,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/iomgr/iocp_windows.h"
@@ -91,10 +91,69 @@ void grpc_winsocket_shutdown(grpc_winsocket *winsocket) {
   closesocket(winsocket->socket);
 }
 
-void grpc_winsocket_destroy(grpc_winsocket *winsocket) {
+static void destroy(grpc_winsocket *winsocket) {
   grpc_iomgr_unregister_object(&winsocket->iomgr_object);
   gpr_mu_destroy(&winsocket->state_mu);
   gpr_free(winsocket);
 }
 
+static bool check_destroyable(grpc_winsocket *winsocket) {
+  return winsocket->destroy_called == true &&
+         winsocket->write_info.closure == NULL &&
+         winsocket->read_info.closure == NULL;
+}
+
+void grpc_winsocket_destroy(grpc_winsocket *winsocket) {
+  gpr_mu_lock(&winsocket->state_mu);
+  GPR_ASSERT(!winsocket->destroy_called);
+  winsocket->destroy_called = true;
+  bool should_destroy = check_destroyable(winsocket);
+  gpr_mu_unlock(&winsocket->state_mu);
+  if (should_destroy) destroy(winsocket);
+}
+
+/* Calling notify_on_read or write means either of two things:
+-) The IOCP already completed in the background, and we need to call
+the callback now.
+-) The IOCP hasn't completed yet, and we're queuing it for later. */
+static void socket_notify_on_iocp(grpc_exec_ctx *exec_ctx,
+                                  grpc_winsocket *socket, grpc_closure *closure,
+                                  grpc_winsocket_callback_info *info) {
+  GPR_ASSERT(info->closure == NULL);
+  gpr_mu_lock(&socket->state_mu);
+  if (info->has_pending_iocp) {
+    info->has_pending_iocp = 0;
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
+  } else {
+    info->closure = closure;
+  }
+  gpr_mu_unlock(&socket->state_mu);
+}
+
+void grpc_socket_notify_on_write(grpc_exec_ctx *exec_ctx,
+                                 grpc_winsocket *socket,
+                                 grpc_closure *closure) {
+  socket_notify_on_iocp(exec_ctx, socket, closure, &socket->write_info);
+}
+
+void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket,
+                                grpc_closure *closure) {
+  socket_notify_on_iocp(exec_ctx, socket, closure, &socket->read_info);
+}
+
+void grpc_socket_become_ready(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket,
+                              grpc_winsocket_callback_info *info) {
+  GPR_ASSERT(!info->has_pending_iocp);
+  gpr_mu_lock(&socket->state_mu);
+  if (info->closure) {
+    grpc_exec_ctx_sched(exec_ctx, info->closure, GRPC_ERROR_NONE, NULL);
+    info->closure = NULL;
+  } else {
+    info->has_pending_iocp = 1;
+  }
+  bool should_destroy = check_destroyable(socket);
+  gpr_mu_unlock(&socket->state_mu);
+  if (should_destroy) destroy(socket);
+}
+
 #endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/lib/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h
index 73c4384987..490d0e0a06 100644
--- a/src/core/lib/iomgr/socket_windows.h
+++ b/src/core/lib/iomgr/socket_windows.h
@@ -81,6 +81,7 @@ typedef struct grpc_winsocket_callback_info {
    is closer to what happens in posix world. */
 typedef struct grpc_winsocket {
   SOCKET socket;
+  bool destroy_called;
 
   grpc_winsocket_callback_info write_info;
   grpc_winsocket_callback_info read_info;
@@ -108,4 +109,16 @@ void grpc_winsocket_shutdown(grpc_winsocket *socket);
 /* Destroy a socket. Should only be called if there's no pending operation. */
 void grpc_winsocket_destroy(grpc_winsocket *socket);
 
+void grpc_socket_notify_on_write(grpc_exec_ctx *exec_ctx,
+                                 grpc_winsocket *winsocket,
+                                 grpc_closure *closure);
+
+void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx,
+                                grpc_winsocket *winsocket,
+                                grpc_closure *closure);
+
+void grpc_socket_become_ready(grpc_exec_ctx *exec_ctx,
+                              grpc_winsocket *winsocket,
+                              grpc_winsocket_callback_info *ci);
+
 #endif /* GRPC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H */
diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index e93d5734a0..80c7a3f128 100644
--- a/src/core/lib/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -71,33 +71,39 @@ typedef struct {
   grpc_closure *closure;
 } async_connect;
 
-static int prepare_socket(const struct sockaddr *addr, int fd) {
-  if (fd < 0) {
-    goto error;
+static grpc_error *prepare_socket(const struct sockaddr *addr, int fd) {
+  grpc_error *err = GRPC_ERROR_NONE;
+
+  GPR_ASSERT(fd >= 0);
+
+  err = grpc_set_socket_nonblocking(fd, 1);
+  if (err != GRPC_ERROR_NONE) goto error;
+  err = grpc_set_socket_cloexec(fd, 1);
+  if (err != GRPC_ERROR_NONE) goto error;
+  if (!grpc_is_unix_socket(addr)) {
+    err = grpc_set_socket_low_latency(fd, 1);
+    if (err != GRPC_ERROR_NONE) goto error;
   }
-
-  if (!grpc_set_socket_nonblocking(fd, 1) || !grpc_set_socket_cloexec(fd, 1) ||
-      (!grpc_is_unix_socket(addr) && !grpc_set_socket_low_latency(fd, 1)) ||
-      !grpc_set_socket_no_sigpipe_if_possible(fd)) {
-    gpr_log(GPR_ERROR, "Unable to configure socket %d: %s", fd,
-            strerror(errno));
-    goto error;
-  }
-  return 1;
+  err = grpc_set_socket_no_sigpipe_if_possible(fd);
+  if (err != GRPC_ERROR_NONE) goto error;
+  goto done;
 
 error:
   if (fd >= 0) {
     close(fd);
   }
-  return 0;
+done:
+  return err;
 }
 
-static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
+static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   int done;
   async_connect *ac = acp;
   if (grpc_tcp_trace) {
-    gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: success=%d", ac->addr_str,
-            success);
+    const char *str = grpc_error_string(error);
+    gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", ac->addr_str,
+            str);
+    grpc_error_free_string(str);
   }
   gpr_mu_lock(&ac->mu);
   if (ac->fd != NULL) {
@@ -112,7 +118,7 @@ static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
   }
 }
 
-static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
+static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   async_connect *ac = acp;
   int so_error = 0;
   socklen_t so_error_size;
@@ -122,9 +128,13 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
   grpc_closure *closure = ac->closure;
   grpc_fd *fd;
 
+  GRPC_ERROR_REF(error);
+
   if (grpc_tcp_trace) {
-    gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: success=%d",
-            ac->addr_str, success);
+    const char *str = grpc_error_string(error);
+    gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: error=%s",
+            ac->addr_str, str);
+    grpc_error_free_string(str);
   }
 
   gpr_mu_lock(&ac->mu);
@@ -136,15 +146,14 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
   grpc_timer_cancel(exec_ctx, &ac->alarm);
 
   gpr_mu_lock(&ac->mu);
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     do {
       so_error_size = sizeof(so_error);
       err = getsockopt(grpc_fd_wrapped_fd(fd), SOL_SOCKET, SO_ERROR, &so_error,
                        &so_error_size);
     } while (err < 0 && errno == EINTR);
     if (err < 0) {
-      gpr_log(GPR_ERROR, "failed to connect to '%s': getsockopt(ERROR): %s",
-              ac->addr_str, strerror(errno));
+      error = GRPC_OS_ERROR(errno, "getsockopt");
       goto finish;
     } else if (so_error != 0) {
       if (so_error == ENOBUFS) {
@@ -169,14 +178,12 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
       } else {
         switch (so_error) {
           case ECONNREFUSED:
-            gpr_log(
-                GPR_ERROR,
-                "failed to connect to '%s': socket error: connection refused",
-                ac->addr_str);
+            error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errno);
+            error = grpc_error_set_str(error, GRPC_ERROR_STR_OS_ERROR,
+                                       "Connection refused");
             break;
           default:
-            gpr_log(GPR_ERROR, "failed to connect to '%s': socket error: %d",
-                    ac->addr_str, so_error);
+            error = GRPC_OS_ERROR(errno, "getsockopt(SO_ERROR)");
             break;
         }
         goto finish;
@@ -188,8 +195,8 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
       goto finish;
     }
   } else {
-    gpr_log(GPR_ERROR, "failed to connect to '%s': timeout occurred",
-            ac->addr_str);
+    error =
+        grpc_error_set_str(error, GRPC_ERROR_STR_OS_ERROR, "Timeout occurred");
     goto finish;
   }
 
@@ -203,12 +210,18 @@ finish:
   }
   done = (--ac->refs == 0);
   gpr_mu_unlock(&ac->mu);
+  if (error != GRPC_ERROR_NONE) {
+    error = grpc_error_set_str(error, GRPC_ERROR_STR_DESCRIPTION,
+                               "Failed to connect to remote host");
+    error =
+        grpc_error_set_str(error, GRPC_ERROR_STR_TARGET_ADDRESS, ac->addr_str);
+  }
   if (done) {
     gpr_mu_destroy(&ac->mu);
     gpr_free(ac->addr_str);
     gpr_free(ac);
   }
-  grpc_exec_ctx_enqueue(exec_ctx, closure, *ep != NULL, NULL);
+  grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
 }
 
 static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
@@ -225,6 +238,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   grpc_fd *fdobj;
   char *name;
   char *addr_str;
+  grpc_error *error;
 
   *ep = NULL;
 
@@ -234,9 +248,10 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
     addr_len = sizeof(addr6_v4mapped);
   }
 
-  fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode);
-  if (fd < 0) {
-    gpr_log(GPR_ERROR, "Unable to create socket: %s", strerror(errno));
+  error = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd);
+  if (error != GRPC_ERROR_NONE) {
+    grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
+    return;
   }
   if (dsmode == GRPC_DSMODE_IPV4) {
     /* If we got an AF_INET socket, map the address back to IPv4. */
@@ -244,8 +259,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
     addr = (struct sockaddr *)&addr4_copy;
     addr_len = sizeof(addr4_copy);
   }
-  if (!prepare_socket(addr, fd)) {
-    grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
+  if ((error = prepare_socket(addr, fd)) != GRPC_ERROR_NONE) {
+    grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
     return;
   }
 
@@ -261,14 +276,14 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
 
   if (err >= 0) {
     *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str);
-    grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
     goto done;
   }
 
   if (errno != EWOULDBLOCK && errno != EINPROGRESS) {
-    gpr_log(GPR_ERROR, "connect error to '%s': %s", addr_str, strerror(errno));
     grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error");
-    grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"),
+                        NULL);
     goto done;
   }
 
diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c
index 66f9ff7a46..562cb9c6bf 100644
--- a/src/core/lib/iomgr/tcp_client_windows.c
+++ b/src/core/lib/iomgr/tcp_client_windows.c
@@ -35,11 +35,11 @@
 
 #ifdef GPR_WINSOCK_SOCKET
 
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/useful.h>
 
@@ -75,23 +75,25 @@ static void async_connect_unlock_and_cleanup(async_connect *ac,
   if (socket != NULL) grpc_winsocket_destroy(socket);
 }
 
-static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, bool occured) {
+static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   async_connect *ac = acp;
   gpr_mu_lock(&ac->mu);
-  if (ac->socket != NULL) {
-    grpc_winsocket_shutdown(ac->socket);
+  grpc_winsocket *socket = ac->socket;
+  ac->socket = NULL;
+  if (socket != NULL) {
+    grpc_winsocket_shutdown(socket);
   }
-  async_connect_unlock_and_cleanup(ac, ac->socket);
+  async_connect_unlock_and_cleanup(ac, socket);
 }
 
-static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, bool from_iocp) {
+static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   async_connect *ac = acp;
-  SOCKET sock = ac->socket->socket;
   grpc_endpoint **ep = ac->endpoint;
   GPR_ASSERT(*ep == NULL);
-  grpc_winsocket_callback_info *info = &ac->socket->write_info;
   grpc_closure *on_done = ac->on_done;
 
+  GRPC_ERROR_REF(error);
+
   gpr_mu_lock(&ac->mu);
   grpc_winsocket *socket = ac->socket;
   ac->socket = NULL;
@@ -101,17 +103,15 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, bool from_iocp) {
 
   gpr_mu_lock(&ac->mu);
 
-  if (from_iocp && socket != NULL) {
+  if (error == GRPC_ERROR_NONE && socket != NULL) {
     DWORD transfered_bytes = 0;
     DWORD flags;
-    BOOL wsa_success = WSAGetOverlappedResult(sock, &info->overlapped,
-                                              &transfered_bytes, FALSE, &flags);
+    BOOL wsa_success =
+        WSAGetOverlappedResult(socket->socket, &socket->write_info.overlapped,
+                               &transfered_bytes, FALSE, &flags);
     GPR_ASSERT(transfered_bytes == 0);
     if (!wsa_success) {
-      char *utf8_message = gpr_format_message(WSAGetLastError());
-      gpr_log(GPR_ERROR, "on_connect error connecting to '%s': %s",
-              ac->addr_name, utf8_message);
-      gpr_free(utf8_message);
+      error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx");
     } else {
       *ep = grpc_tcp_create(socket, ac->addr_name);
       socket = NULL;
@@ -121,7 +121,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, bool from_iocp) {
   async_connect_unlock_and_cleanup(ac, socket);
   /* If the connection was aborted, the callback was already called when
      the deadline was met. */
-  on_done->cb(exec_ctx, on_done->cb_arg, *ep != NULL);
+  grpc_exec_ctx_sched(exec_ctx, on_done, error, NULL);
 }
 
 /* Tries to issue one async connection, then schedules both an IOCP
@@ -141,10 +141,8 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
   LPFN_CONNECTEX ConnectEx;
   GUID guid = WSAID_CONNECTEX;
   DWORD ioctl_num_bytes;
-  const char *message = NULL;
-  char *utf8_message;
   grpc_winsocket_callback_info *info;
-  int last_error;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   *endpoint = NULL;
 
@@ -157,12 +155,12 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
   sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0,
                    WSA_FLAG_OVERLAPPED);
   if (sock == INVALID_SOCKET) {
-    message = "Unable to create socket: %s";
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket");
     goto failure;
   }
 
-  if (!grpc_tcp_prepare_socket(sock)) {
-    message = "Unable to set socket options: %s";
+  error = grpc_tcp_prepare_socket(sock);
+  if (error != GRPC_ERROR_NONE) {
     goto failure;
   }
 
@@ -173,7 +171,8 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
                &ConnectEx, sizeof(ConnectEx), &ioctl_num_bytes, NULL, NULL);
 
   if (status != 0) {
-    message = "Unable to retrieve ConnectEx pointer: %s";
+    error = GRPC_WSA_ERROR(WSAGetLastError(),
+                           "WSAIoctl(SIO_GET_EXTENSION_FUNCTION_POINTER)");
     goto failure;
   }
 
@@ -181,7 +180,7 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
 
   status = bind(sock, (struct sockaddr *)&local_address, sizeof(local_address));
   if (status != 0) {
-    message = "Unable to bind socket: %s";
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "bind");
     goto failure;
   }
 
@@ -193,9 +192,9 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
   /* It wouldn't be unusual to get a success immediately. But we'll still get
      an IOCP notification, so let's ignore it. */
   if (!success) {
-    int error = WSAGetLastError();
-    if (error != ERROR_IO_PENDING) {
-      message = "ConnectEx failed: %s";
+    int last_error = WSAGetLastError();
+    if (last_error != ERROR_IO_PENDING) {
+      error = GRPC_WSA_ERROR(last_error, "ConnectEx");
       goto failure;
     }
   }
@@ -215,17 +214,18 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
   return;
 
 failure:
-  last_error = WSAGetLastError();
-  utf8_message = gpr_format_message(last_error);
-  gpr_log(GPR_ERROR, message, utf8_message);
-  gpr_log(GPR_ERROR, "last error = %d", last_error);
-  gpr_free(utf8_message);
+  GPR_ASSERT(error != GRPC_ERROR_NONE);
+  char *target_uri = grpc_sockaddr_to_uri(addr);
+  grpc_error *final_error = grpc_error_set_str(
+      GRPC_ERROR_CREATE_REFERENCING("Failed to connect", &error, 1),
+      GRPC_ERROR_STR_TARGET_ADDRESS, target_uri);
+  GRPC_ERROR_UNREF(error);
   if (socket != NULL) {
     grpc_winsocket_destroy(socket);
   } else if (sock != INVALID_SOCKET) {
     closesocket(sock);
   }
-  grpc_exec_ctx_enqueue(exec_ctx, on_done, false, NULL);
+  grpc_exec_ctx_sched(exec_ctx, on_done, final_error, NULL);
 }
 
 #endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index 7210aef5d5..2ab45e33ce 100644
--- a/src/core/lib/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -35,9 +35,11 @@
 
 #ifdef GPR_POSIX_SOCKET
 
+#include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/tcp_posix.h"
 
 #include <errno.h>
+#include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/socket.h>
@@ -74,7 +76,7 @@ typedef struct {
   grpc_endpoint base;
   grpc_fd *em_fd;
   int fd;
-  int finished_edge;
+  bool finished_edge;
   msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */
   size_t slice_size;
   gpr_refcount refcount;
@@ -101,9 +103,9 @@ typedef struct {
 } grpc_tcp;
 
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                            bool success);
+                            grpc_error *error);
 static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                             bool success);
+                             grpc_error *error);
 
 static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
   grpc_tcp *tcp = (grpc_tcp *)ep;
@@ -151,27 +153,31 @@ static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); }
 #endif
 
 static void tcp_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
+  grpc_network_status_unregister_endpoint(ep);
   grpc_tcp *tcp = (grpc_tcp *)ep;
   TCP_UNREF(exec_ctx, tcp, "destroy");
 }
 
-static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, int success) {
+static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
+                         grpc_error *error) {
   grpc_closure *cb = tcp->read_cb;
 
   if (grpc_tcp_trace) {
     size_t i;
-    gpr_log(GPR_DEBUG, "read: success=%d", success);
+    const char *str = grpc_error_string(error);
+    gpr_log(GPR_DEBUG, "read: error=%s", str);
+    grpc_error_free_string(str);
     for (i = 0; i < tcp->incoming_buffer->count; i++) {
       char *dump = gpr_dump_slice(tcp->incoming_buffer->slices[i],
                                   GPR_DUMP_HEX | GPR_DUMP_ASCII);
-      gpr_log(GPR_DEBUG, "READ %p: %s", tcp, dump);
+      gpr_log(GPR_DEBUG, "READ %p (peer=%s): %s", tcp, tcp->peer_string, dump);
       gpr_free(dump);
     }
   }
 
   tcp->read_cb = NULL;
   tcp->incoming_buffer = NULL;
-  cb->cb(exec_ctx, cb->cb_arg, success);
+  grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
 }
 
 #define MAX_READ_IOVEC 4
@@ -219,15 +225,14 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
       /* We've consumed the edge, request a new one */
       grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure);
     } else {
-      /* TODO(klempner): Log interesting errors */
       gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer);
-      call_read_cb(exec_ctx, tcp, 0);
+      call_read_cb(exec_ctx, tcp, GRPC_OS_ERROR(errno, "recvmsg"));
       TCP_UNREF(exec_ctx, tcp, "read");
     }
   } else if (read_bytes == 0) {
     /* 0 read size ==> end of stream */
     gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer);
-    call_read_cb(exec_ctx, tcp, 0);
+    call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("EOF"));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
     GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
@@ -240,7 +245,7 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
       ++tcp->iov_size;
     }
     GPR_ASSERT((size_t)read_bytes == tcp->incoming_buffer->length);
-    call_read_cb(exec_ctx, tcp, 1);
+    call_read_cb(exec_ctx, tcp, GRPC_ERROR_NONE);
     TCP_UNREF(exec_ctx, tcp, "read");
   }
 
@@ -248,13 +253,13 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
 }
 
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                            bool success) {
+                            grpc_error *error) {
   grpc_tcp *tcp = (grpc_tcp *)arg;
   GPR_ASSERT(!tcp->finished_edge);
 
-  if (!success) {
+  if (error != GRPC_ERROR_NONE) {
     gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer);
-    call_read_cb(exec_ctx, tcp, 0);
+    call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
     tcp_continue_read(exec_ctx, tcp);
@@ -271,17 +276,16 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   gpr_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer);
   TCP_REF(tcp, "read");
   if (tcp->finished_edge) {
-    tcp->finished_edge = 0;
+    tcp->finished_edge = false;
     grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, &tcp->read_closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &tcp->read_closure, GRPC_ERROR_NONE, NULL);
   }
 }
 
-typedef enum { FLUSH_DONE, FLUSH_PENDING, FLUSH_ERROR } flush_result;
-
+/* returns true if done, false if pending; if returning true, *error is set */
 #define MAX_WRITE_IOVEC 16
-static flush_result tcp_flush(grpc_tcp *tcp) {
+static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) {
   struct msghdr msg;
   struct iovec iov[MAX_WRITE_IOVEC];
   msg_iovlen_type iov_size;
@@ -331,10 +335,10 @@ static flush_result tcp_flush(grpc_tcp *tcp) {
       if (errno == EAGAIN) {
         tcp->outgoing_slice_idx = unwind_slice_idx;
         tcp->outgoing_byte_idx = unwind_byte_idx;
-        return FLUSH_PENDING;
+        return false;
       } else {
-        /* TODO(klempner): Log some of these */
-        return FLUSH_ERROR;
+        *error = GRPC_OS_ERROR(errno, "sendmsg");
+        return true;
       }
     }
 
@@ -355,42 +359,42 @@ static flush_result tcp_flush(grpc_tcp *tcp) {
     }
 
     if (tcp->outgoing_slice_idx == tcp->outgoing_buffer->count) {
-      return FLUSH_DONE;
+      *error = GRPC_ERROR_NONE;
+      return true;
     }
   };
 }
 
 static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                             bool success) {
+                             grpc_error *error) {
   grpc_tcp *tcp = (grpc_tcp *)arg;
-  flush_result status;
   grpc_closure *cb;
 
-  if (!success) {
+  if (error != GRPC_ERROR_NONE) {
     cb = tcp->write_cb;
     tcp->write_cb = NULL;
-    cb->cb(exec_ctx, cb->cb_arg, 0);
+    cb->cb(exec_ctx, cb->cb_arg, error);
     TCP_UNREF(exec_ctx, tcp, "write");
     return;
   }
 
-  status = tcp_flush(tcp);
-  if (status == FLUSH_PENDING) {
+  if (!tcp_flush(tcp, &error)) {
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
     cb = tcp->write_cb;
     tcp->write_cb = NULL;
     GPR_TIMER_BEGIN("tcp_handle_write.cb", 0);
-    cb->cb(exec_ctx, cb->cb_arg, status == FLUSH_DONE);
+    cb->cb(exec_ctx, cb->cb_arg, error);
     GPR_TIMER_END("tcp_handle_write.cb", 0);
     TCP_UNREF(exec_ctx, tcp, "write");
+    GRPC_ERROR_UNREF(error);
   }
 }
 
 static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                       gpr_slice_buffer *buf, grpc_closure *cb) {
   grpc_tcp *tcp = (grpc_tcp *)ep;
-  flush_result status;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   if (grpc_tcp_trace) {
     size_t i;
@@ -398,7 +402,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     for (i = 0; i < buf->count; i++) {
       char *data =
           gpr_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
-      gpr_log(GPR_DEBUG, "WRITE %p: %s", tcp, data);
+      gpr_log(GPR_DEBUG, "WRITE %p (peer=%s): %s", tcp, tcp->peer_string, data);
       gpr_free(data);
     }
   }
@@ -408,20 +412,22 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
   if (buf->length == 0) {
     GPR_TIMER_END("tcp_write", 0);
-    grpc_exec_ctx_enqueue(exec_ctx, cb, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb, grpc_fd_is_shutdown(tcp->em_fd)
+                                          ? GRPC_ERROR_CREATE("EOF")
+                                          : GRPC_ERROR_NONE,
+                        NULL);
     return;
   }
   tcp->outgoing_buffer = buf;
   tcp->outgoing_slice_idx = 0;
   tcp->outgoing_byte_idx = 0;
 
-  status = tcp_flush(tcp);
-  if (status == FLUSH_PENDING) {
+  if (!tcp_flush(tcp, &error)) {
     TCP_REF(tcp, "write");
     tcp->write_cb = cb;
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, cb, status == FLUSH_DONE, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
   }
 
   GPR_TIMER_END("tcp_write", 0);
@@ -461,7 +467,7 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
   tcp->incoming_buffer = NULL;
   tcp->slice_size = slice_size;
   tcp->iov_size = 1;
-  tcp->finished_edge = 1;
+  tcp->finished_edge = true;
   /* paired with unref in grpc_tcp_destroy */
   gpr_ref_init(&tcp->refcount, 1);
   tcp->em_fd = em_fd;
@@ -470,6 +476,8 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
   tcp->write_closure.cb = tcp_handle_write;
   tcp->write_closure.cb_arg = tcp;
   gpr_slice_buffer_init(&tcp->last_read_buffer);
+  /* Tell network status tracker about new endpoint */
+  grpc_network_status_register_endpoint(&tcp->base);
 
   return &tcp->base;
 }
diff --git a/src/core/lib/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h
index 99b9f29729..875a6ca14a 100644
--- a/src/core/lib/iomgr/tcp_server.h
+++ b/src/core/lib/iomgr/tcp_server.h
@@ -34,6 +34,8 @@
 #ifndef GRPC_CORE_LIB_IOMGR_TCP_SERVER_H
 #define GRPC_CORE_LIB_IOMGR_TCP_SERVER_H
 
+#include <grpc/grpc.h>
+
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
 
@@ -52,12 +54,15 @@ typedef struct grpc_tcp_server_acceptor {
 /* Called for newly connected TCP connections. */
 typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_endpoint *ep,
+                                   grpc_pollset *accepting_pollset,
                                    grpc_tcp_server_acceptor *acceptor);
 
 /* Create a server, initially not bound to any ports. The caller owns one ref.
    If shutdown_complete is not NULL, it will be used by
    grpc_tcp_server_unref() when the ref count reaches zero. */
-grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete);
+grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
+                                   const grpc_channel_args *args,
+                                   grpc_tcp_server **server);
 
 /* Start listening to bound ports */
 void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
@@ -73,8 +78,8 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
    but not dualstack sockets. */
 /* TODO(ctiller): deprecate this, and make grpc_tcp_server_add_ports to handle
                   all of the multiple socket port matching logic in one place */
-int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
-                             size_t addr_len);
+grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
+                                     size_t addr_len, int *out_port);
 
 /* Number of fds at the given port_index, or 0 if port_index is out of
    bounds. */
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index aaeb384f6e..a1a463550a 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -59,6 +59,8 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
+#include <grpc/support/useful.h>
+
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/socket_utils_posix.h"
@@ -110,8 +112,10 @@ struct grpc_tcp_server {
   /* destroyed port count: how many ports are completely destroyed */
   size_t destroyed_ports;
 
-  /* is this server shutting down? (boolean) */
-  int shutdown;
+  /* is this server shutting down? */
+  bool shutdown;
+  /* use SO_REUSEPORT */
+  bool so_reuseport;
 
   /* linked list of server ports */
   grpc_tcp_listener *head;
@@ -128,15 +132,47 @@ struct grpc_tcp_server {
   grpc_pollset **pollsets;
   /* number of pollsets in the pollsets array */
   size_t pollset_count;
+
+  /* next pollset to assign a channel to */
+  size_t next_pollset_to_assign;
 };
 
-grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
+static gpr_once check_init = GPR_ONCE_INIT;
+static bool has_so_reuseport;
+
+static void init(void) {
+  int s = socket(AF_INET, SOCK_STREAM, 0);
+  if (s >= 0) {
+    has_so_reuseport = GRPC_LOG_IF_ERROR("check for SO_REUSEPORT",
+                                         grpc_set_socket_reuse_port(s, 1));
+    close(s);
+  }
+}
+
+grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
+                                   const grpc_channel_args *args,
+                                   grpc_tcp_server **server) {
+  gpr_once_init(&check_init, init);
+
   grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server));
+  s->so_reuseport = has_so_reuseport;
+  for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) {
+    if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) {
+      if (args->args[i].type == GRPC_ARG_INTEGER) {
+        s->so_reuseport =
+            has_so_reuseport && (args->args[i].value.integer != 0);
+      } else {
+        gpr_free(s);
+        return GRPC_ERROR_CREATE(GRPC_ARG_ALLOW_REUSEPORT
+                                 " must be an integer");
+      }
+    }
+  }
   gpr_ref_init(&s->refs, 1);
   gpr_mu_init(&s->mu);
   s->active_ports = 0;
   s->destroyed_ports = 0;
-  s->shutdown = 0;
+  s->shutdown = false;
   s->shutdown_starting.head = NULL;
   s->shutdown_starting.tail = NULL;
   s->shutdown_complete = shutdown_complete;
@@ -145,12 +181,14 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
   s->head = NULL;
   s->tail = NULL;
   s->nports = 0;
-  return s;
+  s->next_pollset_to_assign = 0;
+  *server = s;
+  return GRPC_ERROR_NONE;
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (s->shutdown_complete != NULL) {
-    grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }
 
   gpr_mu_destroy(&s->mu);
@@ -165,7 +203,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 }
 
 static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server,
-                           bool success) {
+                           grpc_error *error) {
   grpc_tcp_server *s = server;
   gpr_mu_lock(&s->mu);
   s->destroyed_ports++;
@@ -210,7 +248,7 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   gpr_mu_lock(&s->mu);
 
   GPR_ASSERT(!s->shutdown);
-  s->shutdown = 1;
+  s->shutdown = true;
 
   /* shutdown all fd's */
   if (s->active_ports) {
@@ -259,64 +297,81 @@ static int get_max_accept_queue_size(void) {
 }
 
 /* Prepare a recently-created socket for listening. */
-static int prepare_socket(int fd, const struct sockaddr *addr,
-                          size_t addr_len) {
+static grpc_error *prepare_socket(int fd, const struct sockaddr *addr,
+                                  size_t addr_len, bool so_reuseport,
+                                  int *port) {
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
+  grpc_error *err = GRPC_ERROR_NONE;
 
-  if (fd < 0) {
-    goto error;
+  GPR_ASSERT(fd >= 0);
+
+  if (so_reuseport) {
+    err = grpc_set_socket_reuse_port(fd, 1);
+    if (err != GRPC_ERROR_NONE) goto error;
   }
 
-  if (!grpc_set_socket_nonblocking(fd, 1) || !grpc_set_socket_cloexec(fd, 1) ||
-      (!grpc_is_unix_socket(addr) && (!grpc_set_socket_low_latency(fd, 1) ||
-                                      !grpc_set_socket_reuse_addr(fd, 1))) ||
-      !grpc_set_socket_no_sigpipe_if_possible(fd)) {
-    gpr_log(GPR_ERROR, "Unable to configure socket %d: %s", fd,
-            strerror(errno));
-    goto error;
+  err = grpc_set_socket_nonblocking(fd, 1);
+  if (err != GRPC_ERROR_NONE) goto error;
+  err = grpc_set_socket_cloexec(fd, 1);
+  if (err != GRPC_ERROR_NONE) goto error;
+  if (!grpc_is_unix_socket(addr)) {
+    err = grpc_set_socket_low_latency(fd, 1);
+    if (err != GRPC_ERROR_NONE) goto error;
+    err = grpc_set_socket_reuse_addr(fd, 1);
+    if (err != GRPC_ERROR_NONE) goto error;
   }
+  err = grpc_set_socket_no_sigpipe_if_possible(fd);
+  if (err != GRPC_ERROR_NONE) goto error;
 
   GPR_ASSERT(addr_len < ~(socklen_t)0);
   if (bind(fd, addr, (socklen_t)addr_len) < 0) {
-    char *addr_str;
-    grpc_sockaddr_to_string(&addr_str, addr, 0);
-    gpr_log(GPR_ERROR, "bind addr=%s: %s", addr_str, strerror(errno));
-    gpr_free(addr_str);
+    err = GRPC_OS_ERROR(errno, "bind");
     goto error;
   }
 
   if (listen(fd, get_max_accept_queue_size()) < 0) {
-    gpr_log(GPR_ERROR, "listen: %s", strerror(errno));
+    err = GRPC_OS_ERROR(errno, "listen");
     goto error;
   }
 
   sockname_len = sizeof(sockname_temp);
   if (getsockname(fd, (struct sockaddr *)&sockname_temp, &sockname_len) < 0) {
+    err = GRPC_OS_ERROR(errno, "getsockname");
     goto error;
   }
 
-  return grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
+  *port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
+  return GRPC_ERROR_NONE;
 
 error:
+  GPR_ASSERT(err != GRPC_ERROR_NONE);
   if (fd >= 0) {
     close(fd);
   }
-  return -1;
+  grpc_error *ret = grpc_error_set_int(
+      GRPC_ERROR_CREATE_REFERENCING("Unable to configure socket", &err, 1),
+      GRPC_ERROR_INT_FD, fd);
+  GRPC_ERROR_UNREF(err);
+  return ret;
 }
 
 /* event manager callback when reads are ready */
-static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
   grpc_tcp_listener *sp = arg;
   grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index,
                                        sp->fd_index};
+  grpc_pollset *read_notifier_pollset = NULL;
   grpc_fd *fdobj;
-  size_t i;
 
-  if (!success) {
+  if (err != GRPC_ERROR_NONE) {
     goto error;
   }
 
+  read_notifier_pollset =
+      sp->server->pollsets[(sp->server->next_pollset_to_assign++) %
+                           sp->server->pollset_count];
+
   /* loop until accept4 returns EAGAIN, and then re-arm notification */
   for (;;) {
     struct sockaddr_storage addr;
@@ -349,16 +404,18 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     }
 
     fdobj = grpc_fd_create(fd, name);
-    /* TODO(ctiller): revise this when we have server-side sharding
-       of channels -- we certainly should not be automatically adding every
-       incoming channel to every pollset owned by the server */
-    for (i = 0; i < sp->server->pollset_count; i++) {
-      grpc_pollset_add_fd(exec_ctx, sp->server->pollsets[i], fdobj);
+
+    if (read_notifier_pollset == NULL) {
+      gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd");
+      goto error;
     }
+
+    grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj);
+
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
         grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
-        &acceptor);
+        read_notifier_pollset, &acceptor);
 
     gpr_free(name);
     gpr_free(addr_str);
@@ -376,18 +433,19 @@ error:
   }
 }
 
-static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd,
-                                               const struct sockaddr *addr,
-                                               size_t addr_len,
-                                               unsigned port_index,
-                                               unsigned fd_index) {
+static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd,
+                                        const struct sockaddr *addr,
+                                        size_t addr_len, unsigned port_index,
+                                        unsigned fd_index,
+                                        grpc_tcp_listener **listener) {
   grpc_tcp_listener *sp = NULL;
-  int port;
+  int port = -1;
   char *addr_str;
   char *name;
 
-  port = prepare_socket(fd, addr, addr_len);
-  if (port >= 0) {
+  grpc_error *err = prepare_socket(fd, addr, addr_len, s->so_reuseport, &port);
+  if (err == GRPC_ERROR_NONE) {
+    GPR_ASSERT(port > 0);
     grpc_sockaddr_to_string(&addr_str, (struct sockaddr *)&addr, 1);
     gpr_asprintf(&name, "tcp-server-listener:%s", addr_str);
     gpr_mu_lock(&s->mu);
@@ -417,11 +475,58 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd,
     gpr_free(name);
   }
 
-  return sp;
+  *listener = sp;
+  return err;
 }
 
-int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
-                             size_t addr_len) {
+static grpc_error *clone_port(grpc_tcp_listener *listener, unsigned count) {
+  grpc_tcp_listener *sp = NULL;
+  char *addr_str;
+  char *name;
+  grpc_error *err;
+
+  for (grpc_tcp_listener *l = listener->next; l && l->is_sibling; l = l->next) {
+    l->fd_index += count;
+  }
+
+  for (unsigned i = 0; i < count; i++) {
+    int fd, port;
+    grpc_dualstack_mode dsmode;
+    err = grpc_create_dualstack_socket(&listener->addr.sockaddr, SOCK_STREAM, 0,
+                                       &dsmode, &fd);
+    if (err != GRPC_ERROR_NONE) return err;
+    err = prepare_socket(fd, &listener->addr.sockaddr, listener->addr_len, true,
+                         &port);
+    if (err != GRPC_ERROR_NONE) return err;
+    listener->server->nports++;
+    grpc_sockaddr_to_string(&addr_str, &listener->addr.sockaddr, 1);
+    gpr_asprintf(&name, "tcp-server-listener:%s/clone-%d", addr_str, i);
+    sp = gpr_malloc(sizeof(grpc_tcp_listener));
+    sp->next = listener->next;
+    listener->next = sp;
+    sp->server = listener->server;
+    sp->fd = fd;
+    sp->emfd = grpc_fd_create(fd, name);
+    memcpy(sp->addr.untyped, listener->addr.untyped, listener->addr_len);
+    sp->addr_len = listener->addr_len;
+    sp->port = port;
+    sp->port_index = listener->port_index;
+    sp->fd_index = listener->fd_index + count - i;
+    sp->is_sibling = 1;
+    sp->sibling = listener->is_sibling ? listener->sibling : listener;
+    GPR_ASSERT(sp->emfd);
+    while (listener->server->tail->next != NULL) {
+      listener->server->tail = listener->server->tail->next;
+    }
+    gpr_free(addr_str);
+    gpr_free(name);
+  }
+
+  return GRPC_ERROR_NONE;
+}
+
+grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
+                                     size_t addr_len, int *out_port) {
   grpc_tcp_listener *sp;
   grpc_tcp_listener *sp2 = NULL;
   int fd;
@@ -436,6 +541,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
   int port;
   unsigned port_index = 0;
   unsigned fd_index = 0;
+  grpc_error *errs[2] = {GRPC_ERROR_NONE, GRPC_ERROR_NONE};
   if (s->tail != NULL) {
     port_index = s->tail->port_index + 1;
   }
@@ -474,33 +580,35 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
     /* Try listening on IPv6 first. */
     addr = (struct sockaddr *)&wild6;
     addr_len = sizeof(wild6);
-    fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode);
-    sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index);
-    if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) {
-      goto done;
-    }
-    if (sp != NULL) {
-      ++fd_index;
-    }
-    /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */
-    if (port == 0 && sp != NULL) {
-      grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port);
+    errs[0] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd);
+    if (errs[0] == GRPC_ERROR_NONE) {
+      errs[0] = add_socket_to_server(s, fd, addr, addr_len, port_index,
+                                     fd_index, &sp);
+      if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) {
+        goto done;
+      }
+      if (sp != NULL) {
+        ++fd_index;
+      }
+      /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */
+      if (port == 0 && sp != NULL) {
+        grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port);
+      }
     }
     addr = (struct sockaddr *)&wild4;
     addr_len = sizeof(wild4);
   }
 
-  fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode);
-  if (fd < 0) {
-    gpr_log(GPR_ERROR, "Unable to create socket: %s", strerror(errno));
-  } else {
+  errs[1] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd);
+  if (errs[1] == GRPC_ERROR_NONE) {
     if (dsmode == GRPC_DSMODE_IPV4 &&
         grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) {
       addr = (struct sockaddr *)&addr4_copy;
       addr_len = sizeof(addr4_copy);
     }
     sp2 = sp;
-    sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index);
+    errs[1] =
+        add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index, &sp);
     if (sp2 != NULL && sp != NULL) {
       sp2->sibling = sp;
       sp->is_sibling = 1;
@@ -510,9 +618,21 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
 done:
   gpr_free(allocated_addr);
   if (sp != NULL) {
-    return sp->port;
+    *out_port = sp->port;
+    GRPC_ERROR_UNREF(errs[0]);
+    GRPC_ERROR_UNREF(errs[1]);
+    return GRPC_ERROR_NONE;
   } else {
-    return -1;
+    *out_port = -1;
+    char *addr_str = grpc_sockaddr_to_uri(addr);
+    grpc_error *err = grpc_error_set_str(
+        GRPC_ERROR_CREATE_REFERENCING("Failed to add port to server", errs,
+                                      GPR_ARRAY_SIZE(errs)),
+        GRPC_ERROR_STR_TARGET_ADDRESS, addr_str);
+    GRPC_ERROR_UNREF(errs[0]);
+    GRPC_ERROR_UNREF(errs[1]);
+    gpr_free(addr_str);
+    return err;
   }
 }
 
@@ -561,14 +681,29 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
   s->on_accept_cb_arg = on_accept_cb_arg;
   s->pollsets = pollsets;
   s->pollset_count = pollset_count;
-  for (sp = s->head; sp; sp = sp->next) {
-    for (i = 0; i < pollset_count; i++) {
-      grpc_pollset_add_fd(exec_ctx, pollsets[i], sp->emfd);
+  sp = s->head;
+  while (sp != NULL) {
+    if (s->so_reuseport && pollset_count > 1) {
+      GPR_ASSERT(GRPC_LOG_IF_ERROR(
+          "clone_port", clone_port(sp, (unsigned)(pollset_count - 1))));
+      for (i = 0; i < pollset_count; i++) {
+        grpc_pollset_add_fd(exec_ctx, pollsets[i], sp->emfd);
+        sp->read_closure.cb = on_read;
+        sp->read_closure.cb_arg = sp;
+        grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure);
+        s->active_ports++;
+        sp = sp->next;
+      }
+    } else {
+      for (i = 0; i < pollset_count; i++) {
+        grpc_pollset_add_fd(exec_ctx, pollsets[i], sp->emfd);
+      }
+      sp->read_closure.cb = on_read;
+      sp->read_closure.cb_arg = sp;
+      grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure);
+      s->active_ports++;
+      sp = sp->next;
     }
-    sp->read_closure.cb = on_read;
-    sp->read_closure.cb_arg = sp;
-    grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure);
-    s->active_ports++;
   }
   gpr_mu_unlock(&s->mu);
 }
@@ -581,7 +716,8 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
 void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
                                            grpc_closure *shutdown_starting) {
   gpr_mu_lock(&s->mu);
-  grpc_closure_list_add(&s->shutdown_starting, shutdown_starting, 1);
+  grpc_closure_list_append(&s->shutdown_starting, shutdown_starting,
+                           GRPC_ERROR_NONE);
   gpr_mu_unlock(&s->mu);
 }
 
diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 125f521d87..7b0966704c 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -41,7 +41,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
@@ -102,7 +102,9 @@ struct grpc_tcp_server {
 
 /* Public function. Allocates the proper data structures to hold a
    grpc_tcp_server. */
-grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
+grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
+                                   const grpc_channel_args *args,
+                                   grpc_tcp_server **server) {
   grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server));
   gpr_ref_init(&s->refs, 1);
   gpr_mu_init(&s->mu);
@@ -114,12 +116,13 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
   s->shutdown_starting.head = NULL;
   s->shutdown_starting.tail = NULL;
   s->shutdown_complete = shutdown_complete;
-  return s;
+  *server = s;
+  return GRPC_ERROR_NONE;
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (s->shutdown_complete != NULL) {
-    grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }
 
   /* Now that the accepts have been aborted, we can destroy the sockets.
@@ -143,7 +146,8 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
 void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
                                            grpc_closure *shutdown_starting) {
   gpr_mu_lock(&s->mu);
-  grpc_closure_list_add(&s->shutdown_starting, shutdown_starting, 1);
+  grpc_closure_list_append(&s->shutdown_starting, shutdown_starting,
+                           GRPC_ERROR_NONE);
   gpr_mu_unlock(&s->mu);
 }
 
@@ -187,51 +191,49 @@ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 }
 
 /* Prepare (bind) a recently-created socket for listening. */
-static int prepare_socket(SOCKET sock, const struct sockaddr *addr,
-                          size_t addr_len) {
+static grpc_error *prepare_socket(SOCKET sock, const struct sockaddr *addr,
+                                  size_t addr_len, int *port) {
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
+  grpc_error *error = GRPC_ERROR_NONE;
 
-  if (sock == INVALID_SOCKET) goto error;
-
-  if (!grpc_tcp_prepare_socket(sock)) {
-    char *utf8_message = gpr_format_message(WSAGetLastError());
-    gpr_log(GPR_ERROR, "Unable to prepare socket: %s", utf8_message);
-    gpr_free(utf8_message);
-    goto error;
+  error = grpc_tcp_prepare_socket(sock);
+  if (error != GRPC_ERROR_NONE) {
+    goto failure;
   }
 
   if (bind(sock, addr, (int)addr_len) == SOCKET_ERROR) {
-    char *addr_str;
-    char *utf8_message = gpr_format_message(WSAGetLastError());
-    grpc_sockaddr_to_string(&addr_str, addr, 0);
-    gpr_log(GPR_ERROR, "bind addr=%s: %s", addr_str, utf8_message);
-    gpr_free(utf8_message);
-    gpr_free(addr_str);
-    goto error;
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "bind");
+    goto failure;
   }
 
   if (listen(sock, SOMAXCONN) == SOCKET_ERROR) {
-    char *utf8_message = gpr_format_message(WSAGetLastError());
-    gpr_log(GPR_ERROR, "listen: %s", utf8_message);
-    gpr_free(utf8_message);
-    goto error;
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "listen");
+    goto failure;
   }
 
   sockname_len = sizeof(sockname_temp);
   if (getsockname(sock, (struct sockaddr *)&sockname_temp, &sockname_len) ==
       SOCKET_ERROR) {
-    char *utf8_message = gpr_format_message(WSAGetLastError());
-    gpr_log(GPR_ERROR, "getsockname: %s", utf8_message);
-    gpr_free(utf8_message);
-    goto error;
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "getsockname");
+    goto failure;
   }
 
-  return grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
+  *port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
+  return GRPC_ERROR_NONE;
 
-error:
+failure:
+  GPR_ASSERT(error != GRPC_ERROR_NONE);
+  char *tgtaddr = grpc_sockaddr_to_uri(addr);
+  grpc_error *final_error = grpc_error_set_int(
+      grpc_error_set_str(GRPC_ERROR_CREATE_REFERENCING(
+                             "Failed to prepare server socket", &error, 1),
+                         GRPC_ERROR_STR_TARGET_ADDRESS, tgtaddr),
+      GRPC_ERROR_INT_FD, (intptr_t)sock);
+  gpr_free(tgtaddr);
+  GRPC_ERROR_UNREF(error);
   if (sock != INVALID_SOCKET) closesocket(sock);
-  return -1;
+  return error;
 }
 
 static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx,
@@ -251,26 +253,23 @@ static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx,
 
 /* In order to do an async accept, we need to create a socket first which
    will be the one assigned to the new incoming connection. */
-static void start_accept(grpc_exec_ctx *exec_ctx, grpc_tcp_listener *port) {
+static grpc_error *start_accept(grpc_exec_ctx *exec_ctx,
+                                grpc_tcp_listener *port) {
   SOCKET sock = INVALID_SOCKET;
-  char *message;
-  char *utf8_message;
   BOOL success;
   DWORD addrlen = sizeof(struct sockaddr_in6) + 16;
   DWORD bytes_received = 0;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0,
                    WSA_FLAG_OVERLAPPED);
-
   if (sock == INVALID_SOCKET) {
-    message = "Unable to create socket: %s";
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket");
     goto failure;
   }
 
-  if (!grpc_tcp_prepare_socket(sock)) {
-    message = "Unable to prepare socket: %s";
-    goto failure;
-  }
+  error = grpc_tcp_prepare_socket(sock);
+  if (error != GRPC_ERROR_NONE) goto failure;
 
   /* Start the "accept" asynchronously. */
   success = port->AcceptEx(port->socket->socket, sock, port->addresses, 0,
@@ -280,9 +279,9 @@ static void start_accept(grpc_exec_ctx *exec_ctx, grpc_tcp_listener *port) {
   /* It is possible to get an accept immediately without delay. However, we
      will still get an IOCP notification for it. So let's just ignore it. */
   if (!success) {
-    int error = WSAGetLastError();
-    if (error != ERROR_IO_PENDING) {
-      message = "AcceptEx failed: %s";
+    int last_error = WSAGetLastError();
+    if (last_error != ERROR_IO_PENDING) {
+      error = GRPC_WSA_ERROR(last_error, "AcceptEx");
       goto failure;
     }
   }
@@ -291,9 +290,10 @@ static void start_accept(grpc_exec_ctx *exec_ctx, grpc_tcp_listener *port) {
      immediately process an accept that happened in the meantime. */
   port->new_socket = sock;
   grpc_socket_notify_on_read(exec_ctx, port->socket, &port->on_accept);
-  return;
+  return error;
 
 failure:
+  GPR_ASSERT(error != GRPC_ERROR_NONE);
   if (port->shutting_down) {
     /* We are abandoning the listener port, take that into account to prevent
        occasional hangs on shutdown. The hang happens when sp->shutting_down
@@ -301,16 +301,15 @@ failure:
        but we fail there because the listening port has been closed in the
        meantime. */
     decrement_active_ports_and_notify(exec_ctx, port);
-    return;
+    GRPC_ERROR_UNREF(error);
+    return GRPC_ERROR_NONE;
   }
-  utf8_message = gpr_format_message(WSAGetLastError());
-  gpr_log(GPR_ERROR, message, utf8_message);
-  gpr_free(utf8_message);
   if (sock != INVALID_SOCKET) closesocket(sock);
+  return error;
 }
 
 /* Event manager callback when reads are ready. */
-static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, bool from_iocp) {
+static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_tcp_listener *sp = arg;
   grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index, 0};
   SOCKET sock = sp->new_socket;
@@ -328,7 +327,10 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, bool from_iocp) {
   /* The general mechanism for shutting down is to queue abortion calls. While
      this is necessary in the read/write case, it's useless for the accept
      case. We only need to adjust the pending callback count */
-  if (!from_iocp) {
+  if (error != GRPC_ERROR_NONE) {
+    const char *msg = grpc_error_string(error);
+    gpr_log(GPR_INFO, "Skipping on_accept due to error: %s", msg);
+    grpc_error_free_string(msg);
     return;
   }
 
@@ -379,28 +381,28 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, bool from_iocp) {
 
   /* The only time we should call our callback, is where we successfully
      managed to accept a connection, and created an endpoint. */
-  if (ep)
-    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep,
+  if (ep) {
+    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
                              &acceptor);
+  }
   /* As we were notified from the IOCP of one and exactly one accept,
      the former socked we created has now either been destroy or assigned
      to the new connection. We need to create a new one for the next
      connection. */
-  start_accept(exec_ctx, sp);
+  GPR_ASSERT(GRPC_LOG_IF_ERROR("start_accept", start_accept(exec_ctx, sp)));
 }
 
-static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock,
-                                               const struct sockaddr *addr,
-                                               size_t addr_len,
-                                               unsigned port_index) {
+static grpc_error *add_socket_to_server(grpc_tcp_server *s, SOCKET sock,
+                                        const struct sockaddr *addr,
+                                        size_t addr_len, unsigned port_index,
+                                        grpc_tcp_listener **listener) {
   grpc_tcp_listener *sp = NULL;
-  int port;
+  int port = -1;
   int status;
   GUID guid = WSAID_ACCEPTEX;
   DWORD ioctl_num_bytes;
   LPFN_ACCEPTEX AcceptEx;
-
-  if (sock == INVALID_SOCKET) return NULL;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   /* We need to grab the AcceptEx pointer for that port, as it may be
      interface-dependent. We'll cache it to avoid doing that again. */
@@ -416,44 +418,49 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock,
     return NULL;
   }
 
-  port = prepare_socket(sock, addr, addr_len);
-  if (port >= 0) {
-    gpr_mu_lock(&s->mu);
-    GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server");
-    sp = gpr_malloc(sizeof(grpc_tcp_listener));
-    sp->next = NULL;
-    if (s->head == NULL) {
-      s->head = sp;
-    } else {
-      s->tail->next = sp;
-    }
-    s->tail = sp;
-    sp->server = s;
-    sp->socket = grpc_winsocket_create(sock, "listener");
-    sp->shutting_down = 0;
-    sp->AcceptEx = AcceptEx;
-    sp->new_socket = INVALID_SOCKET;
-    sp->port = port;
-    sp->port_index = port_index;
-    grpc_closure_init(&sp->on_accept, on_accept, sp);
-    GPR_ASSERT(sp->socket);
-    gpr_mu_unlock(&s->mu);
+  error = prepare_socket(sock, addr, addr_len, &port);
+  if (error != GRPC_ERROR_NONE) {
+    return error;
+  }
+
+  GPR_ASSERT(port >= 0);
+  gpr_mu_lock(&s->mu);
+  GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server");
+  sp = gpr_malloc(sizeof(grpc_tcp_listener));
+  sp->next = NULL;
+  if (s->head == NULL) {
+    s->head = sp;
+  } else {
+    s->tail->next = sp;
   }
+  s->tail = sp;
+  sp->server = s;
+  sp->socket = grpc_winsocket_create(sock, "listener");
+  sp->shutting_down = 0;
+  sp->AcceptEx = AcceptEx;
+  sp->new_socket = INVALID_SOCKET;
+  sp->port = port;
+  sp->port_index = port_index;
+  grpc_closure_init(&sp->on_accept, on_accept, sp);
+  GPR_ASSERT(sp->socket);
+  gpr_mu_unlock(&s->mu);
+  *listener = sp;
 
-  return sp;
+  return GRPC_ERROR_NONE;
 }
 
-int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
-                             size_t addr_len) {
-  grpc_tcp_listener *sp;
+grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
+                                     size_t addr_len, int *port) {
+  grpc_tcp_listener *sp = NULL;
   SOCKET sock;
   struct sockaddr_in6 addr6_v4mapped;
   struct sockaddr_in6 wildcard;
   struct sockaddr *allocated_addr = NULL;
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
-  int port;
   unsigned port_index = 0;
+  grpc_error *error = GRPC_ERROR_NONE;
+
   if (s->tail != NULL) {
     port_index = s->tail->port_index + 1;
   }
@@ -465,11 +472,11 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
       sockname_len = sizeof(sockname_temp);
       if (0 == getsockname(sp->socket->socket,
                            (struct sockaddr *)&sockname_temp, &sockname_len)) {
-        port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
-        if (port > 0) {
+        *port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
+        if (*port > 0) {
           allocated_addr = gpr_malloc(addr_len);
           memcpy(allocated_addr, addr, addr_len);
-          grpc_sockaddr_set_port(allocated_addr, port);
+          grpc_sockaddr_set_port(allocated_addr, *port);
           addr = allocated_addr;
           break;
         }
@@ -483,8 +490,8 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
   }
 
   /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */
-  if (grpc_sockaddr_is_wildcard(addr, &port)) {
-    grpc_sockaddr_make_wildcard6(port, &wildcard);
+  if (grpc_sockaddr_is_wildcard(addr, port)) {
+    grpc_sockaddr_make_wildcard6(*port, &wildcard);
 
     addr = (struct sockaddr *)&wildcard;
     addr_len = sizeof(wildcard);
@@ -493,19 +500,26 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
   sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0,
                    WSA_FLAG_OVERLAPPED);
   if (sock == INVALID_SOCKET) {
-    char *utf8_message = gpr_format_message(WSAGetLastError());
-    gpr_log(GPR_ERROR, "unable to create socket: %s", utf8_message);
-    gpr_free(utf8_message);
+    error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket");
+    goto done;
   }
 
-  sp = add_socket_to_server(s, sock, addr, addr_len, port_index);
+  error = add_socket_to_server(s, sock, addr, addr_len, port_index, &sp);
+
+done:
   gpr_free(allocated_addr);
 
-  if (sp) {
-    return sp->port;
+  if (error != GRPC_ERROR_NONE) {
+    grpc_error *error_out = GRPC_ERROR_CREATE_REFERENCING(
+        "Failed to add port to server", &error, 1);
+    GRPC_ERROR_UNREF(error);
+    error = error_out;
+    *port = -1;
   } else {
-    return -1;
+    GPR_ASSERT(sp != NULL);
+    *port = sp->port;
   }
+  return error;
 }
 
 void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
@@ -520,7 +534,7 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
   s->on_accept_cb = on_accept_cb;
   s->on_accept_cb_arg = on_accept_cb_arg;
   for (sp = s->head; sp; sp = sp->next) {
-    start_accept(exec_ctx, sp);
+    GPR_ASSERT(GRPC_LOG_IF_ERROR("start_accept", start_accept(exec_ctx, sp)));
     s->active_ports++;
   }
   gpr_mu_unlock(&s->mu);
diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c
index 551149e1a6..37ab59021e 100644
--- a/src/core/lib/iomgr/tcp_windows.c
+++ b/src/core/lib/iomgr/tcp_windows.c
@@ -37,11 +37,12 @@
 
 #include <limits.h>
 
-#include "src/core/lib/iomgr/sockaddr_win32.h"
+#include "src/core/lib/iomgr/network_status_tracker.h"
+#include "src/core/lib/iomgr/sockaddr_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
@@ -61,27 +62,34 @@
 #define GRPC_FIONBIO FIONBIO
 #endif
 
-static int set_non_block(SOCKET sock) {
+static grpc_error *set_non_block(SOCKET sock) {
   int status;
   uint32_t param = 1;
   DWORD ret;
   status = WSAIoctl(sock, GRPC_FIONBIO, &param, sizeof(param), NULL, 0, &ret,
                     NULL, NULL);
-  return status == 0;
+  return status == 0
+             ? GRPC_ERROR_NONE
+             : GRPC_WSA_ERROR(WSAGetLastError(), "WSAIoctl(GRPC_FIONBIO)");
 }
 
-static int set_dualstack(SOCKET sock) {
+static grpc_error *set_dualstack(SOCKET sock) {
   int status;
   unsigned long param = 0;
   status = setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char *)&param,
                       sizeof(param));
-  return status == 0;
+  return status == 0
+             ? GRPC_ERROR_NONE
+             : GRPC_WSA_ERROR(WSAGetLastError(), "setsockopt(IPV6_V6ONLY)");
 }
 
-int grpc_tcp_prepare_socket(SOCKET sock) {
-  if (!set_non_block(sock)) return 0;
-  if (!set_dualstack(sock)) return 0;
-  return 1;
+grpc_error *grpc_tcp_prepare_socket(SOCKET sock) {
+  grpc_error *err;
+  err = set_non_block(sock);
+  if (err != GRPC_ERROR_NONE) return err;
+  err = set_dualstack(sock);
+  if (err != GRPC_ERROR_NONE) return err;
+  return GRPC_ERROR_NONE;
 }
 
 typedef struct grpc_tcp {
@@ -148,39 +156,35 @@ static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); }
 #endif
 
 /* Asynchronous callback from the IOCP, or the background thread. */
-static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, bool success) {
+static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) {
   grpc_tcp *tcp = tcpp;
   grpc_closure *cb = tcp->read_cb;
   grpc_winsocket *socket = tcp->socket;
   gpr_slice sub;
   grpc_winsocket_callback_info *info = &socket->read_info;
 
-  if (success) {
+  GRPC_ERROR_REF(error);
+
+  if (error == GRPC_ERROR_NONE) {
     if (info->wsa_error != 0 && !tcp->shutting_down) {
-      if (info->wsa_error != WSAECONNRESET) {
-        char *utf8_message = gpr_format_message(info->wsa_error);
-        gpr_log(GPR_ERROR, "ReadFile overlapped error: %s", utf8_message);
-        gpr_free(utf8_message);
-      }
-      success = 0;
+      char *utf8_message = gpr_format_message(info->wsa_error);
+      error = GRPC_ERROR_CREATE(utf8_message);
+      gpr_free(utf8_message);
       gpr_slice_unref(tcp->read_slice);
     } else {
       if (info->bytes_transfered != 0 && !tcp->shutting_down) {
         sub = gpr_slice_sub_no_ref(tcp->read_slice, 0, info->bytes_transfered);
         gpr_slice_buffer_add(tcp->read_slices, sub);
-        success = 1;
       } else {
         gpr_slice_unref(tcp->read_slice);
-        success = 0;
+        error = GRPC_ERROR_CREATE("End of TCP stream");
       }
     }
   }
 
   tcp->read_cb = NULL;
   TCP_UNREF(tcp, "read");
-  if (cb) {
-    cb->cb(exec_ctx, cb->cb_arg, success);
-  }
+  grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
 }
 
 static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
@@ -194,7 +198,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   WSABUF buffer;
 
   if (tcp->shutting_down) {
-    grpc_exec_ctx_enqueue(exec_ctx, cb, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb,
+                        GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL);
     return;
   }
 
@@ -218,7 +223,7 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   /* Did we get data immediately ? Yay. */
   if (info->wsa_error != WSAEWOULDBLOCK) {
     info->bytes_transfered = bytes_read;
-    grpc_exec_ctx_enqueue(exec_ctx, &tcp->on_read, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &tcp->on_read, GRPC_ERROR_NONE, NULL);
     return;
   }
 
@@ -231,7 +236,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     int wsa_error = WSAGetLastError();
     if (wsa_error != WSA_IO_PENDING) {
       info->wsa_error = wsa_error;
-      grpc_exec_ctx_enqueue(exec_ctx, &tcp->on_read, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, &tcp->on_read,
+                          GRPC_WSA_ERROR(info->wsa_error, "WSARecv"), NULL);
       return;
     }
   }
@@ -240,32 +246,29 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 }
 
 /* Asynchronous callback from the IOCP, or the background thread. */
-static void on_write(grpc_exec_ctx *exec_ctx, void *tcpp, bool success) {
+static void on_write(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) {
   grpc_tcp *tcp = (grpc_tcp *)tcpp;
   grpc_winsocket *handle = tcp->socket;
   grpc_winsocket_callback_info *info = &handle->write_info;
   grpc_closure *cb;
 
+  GRPC_ERROR_REF(error);
+
   gpr_mu_lock(&tcp->mu);
   cb = tcp->write_cb;
   tcp->write_cb = NULL;
   gpr_mu_unlock(&tcp->mu);
 
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     if (info->wsa_error != 0) {
-      if (info->wsa_error != WSAECONNRESET) {
-        char *utf8_message = gpr_format_message(info->wsa_error);
-        gpr_log(GPR_ERROR, "WSASend overlapped error: %s", utf8_message);
-        gpr_free(utf8_message);
-      }
-      success = 0;
+      error = GRPC_WSA_ERROR(info->wsa_error, "WSASend");
     } else {
       GPR_ASSERT(info->bytes_transfered == tcp->write_slices->length);
     }
   }
 
   TCP_UNREF(tcp, "write");
-  cb->cb(exec_ctx, cb->cb_arg, success);
+  grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
 }
 
 /* Initiates a write. */
@@ -283,7 +286,8 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   size_t len;
 
   if (tcp->shutting_down) {
-    grpc_exec_ctx_enqueue(exec_ctx, cb, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, cb,
+                        GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL);
     return;
   }
 
@@ -311,19 +315,10 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
      connection that has its send queue filled up. But if we don't, then we can
      avoid doing an async write operation at all. */
   if (info->wsa_error != WSAEWOULDBLOCK) {
-    bool ok = false;
-    if (status == 0) {
-      ok = true;
-      GPR_ASSERT(bytes_sent == tcp->write_slices->length);
-    } else {
-      if (info->wsa_error != WSAECONNRESET) {
-        char *utf8_message = gpr_format_message(info->wsa_error);
-        gpr_log(GPR_ERROR, "WSASend error: %s", utf8_message);
-        gpr_free(utf8_message);
-      }
-    }
-    if (allocated) gpr_free(allocated);
-    grpc_exec_ctx_enqueue(exec_ctx, cb, ok, NULL);
+    grpc_error *error = status == 0
+                            ? GRPC_ERROR_NONE
+                            : GRPC_WSA_ERROR(info->wsa_error, "WSASend");
+    grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
     return;
   }
 
@@ -340,7 +335,8 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     int wsa_error = WSAGetLastError();
     if (wsa_error != WSA_IO_PENDING) {
       TCP_UNREF(tcp, "write");
-      grpc_exec_ctx_enqueue(exec_ctx, cb, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, cb, GRPC_WSA_ERROR(wsa_error, "WSASend"),
+                          NULL);
       return;
     }
   }
@@ -383,6 +379,7 @@ static void win_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
 }
 
 static void win_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
+  grpc_network_status_unregister_endpoint(ep);
   grpc_tcp *tcp = (grpc_tcp *)ep;
   TCP_UNREF(tcp, "destroy");
 }
@@ -406,6 +403,9 @@ grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) {
   grpc_closure_init(&tcp->on_read, on_read, tcp);
   grpc_closure_init(&tcp->on_write, on_write, tcp);
   tcp->peer_string = gpr_strdup(peer_string);
+  /* Tell network status tracking code about the new endpoint */
+  grpc_network_status_register_endpoint(&tcp->base);
+
   return &tcp->base;
 }
 
diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h
index a2f58eddd5..86d777235e 100644
--- a/src/core/lib/iomgr/tcp_windows.h
+++ b/src/core/lib/iomgr/tcp_windows.h
@@ -52,6 +52,6 @@
  */
 grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string);
 
-int grpc_tcp_prepare_socket(SOCKET sock);
+grpc_error *grpc_tcp_prepare_socket(SOCKET sock);
 
 #endif /* GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H */
diff --git a/src/core/lib/iomgr/timer.c b/src/core/lib/iomgr/timer.c
index acb5b26c87..9975fa1671 100644
--- a/src/core/lib/iomgr/timer.c
+++ b/src/core/lib/iomgr/timer.c
@@ -73,7 +73,7 @@ static shard_type *g_shard_queue[NUM_SHARDS];
 static bool g_initialized = false;
 
 static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now,
-                                   gpr_timespec *next, int success);
+                                   gpr_timespec *next, grpc_error *error);
 
 static gpr_timespec compute_min_deadline(shard_type *shard) {
   return grpc_timer_heap_is_empty(&shard->heap)
@@ -105,7 +105,8 @@ void grpc_timer_list_init(gpr_timespec now) {
 
 void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {
   int i;
-  run_some_expired_timers(exec_ctx, gpr_inf_future(g_clock_type), NULL, 0);
+  run_some_expired_timers(exec_ctx, gpr_inf_future(g_clock_type), NULL,
+                          GRPC_ERROR_CREATE("Timer list shutdown"));
   for (i = 0; i < NUM_SHARDS; i++) {
     shard_type *shard = &g_shards[i];
     gpr_mu_destroy(&shard->mu);
@@ -185,13 +186,16 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 
   if (!g_initialized) {
     timer->triggered = 1;
-    grpc_exec_ctx_enqueue(exec_ctx, &timer->closure, false, NULL);
+    grpc_exec_ctx_sched(
+        exec_ctx, &timer->closure,
+        GRPC_ERROR_CREATE("Attempt to create timer before initialization"),
+        NULL);
     return;
   }
 
   if (gpr_time_cmp(deadline, now) <= 0) {
     timer->triggered = 1;
-    grpc_exec_ctx_enqueue(exec_ctx, &timer->closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_NONE, NULL);
     return;
   }
 
@@ -235,10 +239,15 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 }
 
 void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
+  if (!g_initialized) {
+    /* must have already been cancelled, also the shard mutex is invalid */
+    return;
+  }
+
   shard_type *shard = &g_shards[shard_idx(timer)];
   gpr_mu_lock(&shard->mu);
   if (!timer->triggered) {
-    grpc_exec_ctx_enqueue(exec_ctx, &timer->closure, false, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_CANCELLED, NULL);
     timer->triggered = 1;
     if (timer->heap_index == INVALID_HEAP_INDEX) {
       list_remove(timer);
@@ -278,8 +287,8 @@ static int refill_queue(shard_type *shard, gpr_timespec now) {
   return !grpc_timer_heap_is_empty(&shard->heap);
 }
 
-/* This pops the next non-cancelled timer with deadline <= now from the queue,
-   or returns NULL if there isn't one.
+/* This pops the next non-cancelled timer with deadline <= now from the
+   queue, or returns NULL if there isn't one.
    REQUIRES: shard->mu locked */
 static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) {
   grpc_timer *timer;
@@ -299,12 +308,12 @@ static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) {
 /* REQUIRES: shard->mu unlocked */
 static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard,
                          gpr_timespec now, gpr_timespec *new_min_deadline,
-                         int success) {
+                         grpc_error *error) {
   size_t n = 0;
   grpc_timer *timer;
   gpr_mu_lock(&shard->mu);
   while ((timer = pop_one(shard, now))) {
-    grpc_exec_ctx_enqueue(exec_ctx, &timer->closure, success, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_REF(error), NULL);
     n++;
   }
   *new_min_deadline = compute_min_deadline(shard);
@@ -313,7 +322,7 @@ static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard,
 }
 
 static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now,
-                                   gpr_timespec *next, int success) {
+                                   gpr_timespec *next, grpc_error *error) {
   size_t n = 0;
 
   /* TODO(ctiller): verify that there are any timers (atomically) here */
@@ -327,8 +336,8 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now,
       /* For efficiency, we pop as many available timers as we can from the
          shard.  This may violate perfect timer deadline ordering, but that
          shouldn't be a big deal because we don't make ordering guarantees. */
-      n += pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline,
-                      success);
+      n +=
+          pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline, error);
 
       /* An grpc_timer_init() on the shard could intervene here, adding a new
          timer that is earlier than new_min_deadline.  However,
@@ -359,6 +368,8 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now,
         *next, gpr_time_add(now, gpr_time_from_millis(1, GPR_TIMESPAN)));
   }
 
+  GRPC_ERROR_UNREF(error);
+
   return (int)n;
 }
 
@@ -367,5 +378,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
   GPR_ASSERT(now.clock_type == g_clock_type);
   return run_some_expired_timers(
       exec_ctx, now, next,
-      gpr_time_cmp(now, gpr_inf_future(now.clock_type)) != 0);
+      gpr_time_cmp(now, gpr_inf_future(now.clock_type)) != 0
+          ? GRPC_ERROR_NONE
+          : GRPC_ERROR_CREATE("Shutting down timer system"));
 }
diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c
index df6cf956d9..1ebccf2ee2 100644
--- a/src/core/lib/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@@ -81,6 +81,7 @@ typedef struct {
   grpc_closure read_closure;
   grpc_closure destroyed_closure;
   grpc_udp_server_read_cb read_cb;
+  grpc_udp_server_orphan_cb orphan_cb;
 } server_port;
 
 /* the overall server */
@@ -168,6 +169,10 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
       server_port *sp = &s->ports[i];
       sp->destroyed_closure.cb = destroyed_port;
       sp->destroyed_closure.cb_arg = s;
+
+      GPR_ASSERT(sp->orphan_cb);
+      sp->orphan_cb(sp->emfd);
+
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
                      "udp_listener_shutdown");
     }
@@ -205,6 +210,8 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
                           size_t addr_len) {
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
+  /* Set send/receive socket buffers to 1 MB */
+  int buffer_size_bytes = 1024 * 1024;
 
   if (fd < 0) {
     goto error;
@@ -234,6 +241,18 @@ static int prepare_socket(int fd, const struct sockaddr *addr,
     goto error;
   }
 
+  if (!grpc_set_socket_sndbuf(fd, buffer_size_bytes)) {
+    gpr_log(GPR_ERROR, "Failed to set send buffer size to %d bytes",
+            buffer_size_bytes);
+    goto error;
+  }
+
+  if (!grpc_set_socket_rcvbuf(fd, buffer_size_bytes)) {
+    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %d bytes",
+            buffer_size_bytes);
+    goto error;
+  }
+
   return grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
 
 error:
@@ -268,7 +287,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
 
 static int add_socket_to_server(grpc_udp_server *s, int fd,
                                 const struct sockaddr *addr, size_t addr_len,
-                                grpc_udp_server_read_cb read_cb) {
+                                grpc_udp_server_read_cb read_cb,
+                                grpc_udp_server_orphan_cb orphan_cb) {
   server_port *sp;
   int port;
   char *addr_str;
@@ -292,6 +312,7 @@ static int add_socket_to_server(grpc_udp_server *s, int fd,
     memcpy(sp->addr.untyped, addr, addr_len);
     sp->addr_len = addr_len;
     sp->read_cb = read_cb;
+    sp->orphan_cb = orphan_cb;
     GPR_ASSERT(sp->emfd);
     gpr_mu_unlock(&s->mu);
     gpr_free(name);
@@ -301,7 +322,8 @@ static int add_socket_to_server(grpc_udp_server *s, int fd,
 }
 
 int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr,
-                             size_t addr_len, grpc_udp_server_read_cb read_cb) {
+                             size_t addr_len, grpc_udp_server_read_cb read_cb,
+                             grpc_udp_server_orphan_cb orphan_cb) {
   int allocated_port1 = -1;
   int allocated_port2 = -1;
   unsigned i;
@@ -348,7 +370,8 @@ int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr,
     addr = (struct sockaddr *)&wild6;
     addr_len = sizeof(wild6);
     fd = grpc_create_dualstack_socket(addr, SOCK_DGRAM, IPPROTO_UDP, &dsmode);
-    allocated_port1 = add_socket_to_server(s, fd, addr, addr_len, read_cb);
+    allocated_port1 =
+        add_socket_to_server(s, fd, addr, addr_len, read_cb, orphan_cb);
     if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) {
       goto done;
     }
@@ -370,7 +393,8 @@ int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr,
     addr = (struct sockaddr *)&addr4_copy;
     addr_len = sizeof(addr4_copy);
   }
-  allocated_port2 = add_socket_to_server(s, fd, addr, addr_len, read_cb);
+  allocated_port2 =
+      add_socket_to_server(s, fd, addr, addr_len, read_cb, orphan_cb);
 
 done:
   gpr_free(allocated_addr);
diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h
index d8cf957a22..33c5ce11cd 100644
--- a/src/core/lib/iomgr/udp_server.h
+++ b/src/core/lib/iomgr/udp_server.h
@@ -48,6 +48,9 @@ typedef struct grpc_udp_server grpc_udp_server;
 typedef void (*grpc_udp_server_read_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd,
                                         struct grpc_server *server);
 
+/* Called when the grpc_fd is about to be orphaned (and the FD closed). */
+typedef void (*grpc_udp_server_orphan_cb)(grpc_fd *emfd);
+
 /* Create a server, initially not bound to any ports */
 grpc_udp_server *grpc_udp_server_create(void);
 
@@ -69,7 +72,8 @@ int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index);
 /* TODO(ctiller): deprecate this, and make grpc_udp_server_add_ports to handle
                   all of the multiple socket port matching logic in one place */
 int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr,
-                             size_t addr_len, grpc_udp_server_read_cb read_cb);
+                             size_t addr_len, grpc_udp_server_read_cb read_cb,
+                             grpc_udp_server_orphan_cb orphan_cb);
 
 void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *server,
                              grpc_closure *on_done);
diff --git a/src/core/lib/iomgr/unix_sockets_posix.c b/src/core/lib/iomgr/unix_sockets_posix.c
index 5767c852df..0e7670e5a5 100644
--- a/src/core/lib/iomgr/unix_sockets_posix.c
+++ b/src/core/lib/iomgr/unix_sockets_posix.c
@@ -47,17 +47,18 @@ void grpc_create_socketpair_if_unix(int sv[2]) {
   GPR_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
 }
 
-grpc_resolved_addresses *grpc_resolve_unix_domain_address(const char *name) {
+grpc_error *grpc_resolve_unix_domain_address(const char *name,
+                                             grpc_resolved_addresses **addrs) {
   struct sockaddr_un *un;
 
-  grpc_resolved_addresses *addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
-  addrs->naddrs = 1;
-  addrs->addrs = gpr_malloc(sizeof(grpc_resolved_address));
-  un = (struct sockaddr_un *)addrs->addrs->addr;
+  *addrs = gpr_malloc(sizeof(grpc_resolved_addresses));
+  (*addrs)->naddrs = 1;
+  (*addrs)->addrs = gpr_malloc(sizeof(grpc_resolved_address));
+  un = (struct sockaddr_un *)(*addrs)->addrs->addr;
   un->sun_family = AF_UNIX;
   strcpy(un->sun_path, name);
-  addrs->addrs->len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
-  return addrs;
+  (*addrs)->addrs->len = strlen(un->sun_path) + sizeof(un->sun_family) + 1;
+  return GRPC_ERROR_NONE;
 }
 
 int grpc_is_unix_socket(const struct sockaddr *addr) {
diff --git a/src/core/lib/iomgr/unix_sockets_posix.h b/src/core/lib/iomgr/unix_sockets_posix.h
index 6758c498e5..db0516d945 100644
--- a/src/core/lib/iomgr/unix_sockets_posix.h
+++ b/src/core/lib/iomgr/unix_sockets_posix.h
@@ -43,7 +43,8 @@
 
 void grpc_create_socketpair_if_unix(int sv[2]);
 
-grpc_resolved_addresses *grpc_resolve_unix_domain_address(const char *name);
+grpc_error *grpc_resolve_unix_domain_address(
+    const char *name, grpc_resolved_addresses **addresses);
 
 int grpc_is_unix_socket(const struct sockaddr *addr);
 
diff --git a/src/core/lib/iomgr/unix_sockets_posix_noop.c b/src/core/lib/iomgr/unix_sockets_posix_noop.c
index d30952789f..56b47c3daf 100644
--- a/src/core/lib/iomgr/unix_sockets_posix_noop.c
+++ b/src/core/lib/iomgr/unix_sockets_posix_noop.c
@@ -44,8 +44,10 @@ void grpc_create_socketpair_if_unix(int sv[2]) {
   GPR_ASSERT(0);
 }
 
-grpc_resolved_addresses *grpc_resolve_unix_domain_address(const char *name) {
-  return NULL;
+grpc_error *grpc_resolve_unix_domain_address(
+    const char *name, grpc_resolved_addresses **addresses) {
+  *addresses = NULL;
+  return GRPC_ERROR_CREATE("Unix domain sockets are not supported on Windows");
 }
 
 int grpc_is_unix_socket(const struct sockaddr *addr) { return false; }
diff --git a/src/core/lib/iomgr/wakeup_fd_eventfd.c b/src/core/lib/iomgr/wakeup_fd_eventfd.c
index 8a772add13..95f6102330 100644
--- a/src/core/lib/iomgr/wakeup_fd_eventfd.c
+++ b/src/core/lib/iomgr/wakeup_fd_eventfd.c
@@ -44,29 +44,39 @@
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
 
-static void eventfd_create(grpc_wakeup_fd* fd_info) {
+static grpc_error* eventfd_create(grpc_wakeup_fd* fd_info) {
   int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
-  /* TODO(klempner): Handle failure more gracefully */
-  GPR_ASSERT(efd >= 0);
+  if (efd < 0) {
+    return GRPC_OS_ERROR(errno, "eventfd");
+  }
   fd_info->read_fd = efd;
   fd_info->write_fd = -1;
+  return GRPC_ERROR_NONE;
 }
 
-static void eventfd_consume(grpc_wakeup_fd* fd_info) {
+static grpc_error* eventfd_consume(grpc_wakeup_fd* fd_info) {
   eventfd_t value;
   int err;
   do {
     err = eventfd_read(fd_info->read_fd, &value);
   } while (err < 0 && errno == EINTR);
+  if (err < 0 && errno != EAGAIN) {
+    return GRPC_OS_ERROR(errno, "eventfd_read");
+  }
+  return GRPC_ERROR_NONE;
 }
 
-static void eventfd_wakeup(grpc_wakeup_fd* fd_info) {
+static grpc_error* eventfd_wakeup(grpc_wakeup_fd* fd_info) {
   int err;
   GPR_TIMER_BEGIN("eventfd_wakeup", 0);
   do {
     err = eventfd_write(fd_info->read_fd, 1);
   } while (err < 0 && errno == EINTR);
+  if (err < 0) {
+    return GRPC_OS_ERROR(errno, "eventfd_write");
+  }
   GPR_TIMER_END("eventfd_wakeup", 0);
+  return GRPC_ERROR_NONE;
 }
 
 static void eventfd_destroy(grpc_wakeup_fd* fd_info) {
@@ -74,8 +84,10 @@ static void eventfd_destroy(grpc_wakeup_fd* fd_info) {
 }
 
 static int eventfd_check_availability(void) {
-  /* TODO(klempner): Actually check if eventfd is available */
-  return 1;
+  const int efd = eventfd(0, 0);
+  const int is_available = efd >= 0;
+  if (is_available) close(efd);
+  return is_available;
 }
 
 const grpc_wakeup_fd_vtable grpc_specialized_wakeup_fd_vtable = {
diff --git a/src/core/lib/iomgr/wakeup_fd_pipe.c b/src/core/lib/iomgr/wakeup_fd_pipe.c
index e9b9a0119f..4e5dbdcb73 100644
--- a/src/core/lib/iomgr/wakeup_fd_pipe.c
+++ b/src/core/lib/iomgr/wakeup_fd_pipe.c
@@ -45,7 +45,7 @@
 
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 
-static void pipe_init(grpc_wakeup_fd* fd_info) {
+static grpc_error* pipe_init(grpc_wakeup_fd* fd_info) {
   int pipefd[2];
   /* TODO(klempner): Make this nonfatal */
   int r = pipe(pipefd);
@@ -53,36 +53,40 @@ static void pipe_init(grpc_wakeup_fd* fd_info) {
     gpr_log(GPR_ERROR, "pipe creation failed (%d): %s", errno, strerror(errno));
     abort();
   }
-  GPR_ASSERT(grpc_set_socket_nonblocking(pipefd[0], 1));
-  GPR_ASSERT(grpc_set_socket_nonblocking(pipefd[1], 1));
+  grpc_error* err;
+  err = grpc_set_socket_nonblocking(pipefd[0], 1);
+  if (err != GRPC_ERROR_NONE) return err;
+  err = grpc_set_socket_nonblocking(pipefd[1], 1);
+  if (err != GRPC_ERROR_NONE) return err;
   fd_info->read_fd = pipefd[0];
   fd_info->write_fd = pipefd[1];
+  return GRPC_ERROR_NONE;
 }
 
-static void pipe_consume(grpc_wakeup_fd* fd_info) {
+static grpc_error* pipe_consume(grpc_wakeup_fd* fd_info) {
   char buf[128];
   ssize_t r;
 
   for (;;) {
     r = read(fd_info->read_fd, buf, sizeof(buf));
     if (r > 0) continue;
-    if (r == 0) return;
+    if (r == 0) return GRPC_ERROR_NONE;
     switch (errno) {
       case EAGAIN:
-        return;
+        return GRPC_ERROR_NONE;
       case EINTR:
         continue;
       default:
-        gpr_log(GPR_ERROR, "error reading pipe: %s", strerror(errno));
-        return;
+        return GRPC_OS_ERROR(errno, "read");
     }
   }
 }
 
-static void pipe_wakeup(grpc_wakeup_fd* fd_info) {
+static grpc_error* pipe_wakeup(grpc_wakeup_fd* fd_info) {
   char c = 0;
   while (write(fd_info->write_fd, &c, 1) != 1 && errno == EINTR)
     ;
+  return GRPC_ERROR_NONE;
 }
 
 static void pipe_destroy(grpc_wakeup_fd* fd_info) {
diff --git a/src/core/lib/iomgr/wakeup_fd_posix.c b/src/core/lib/iomgr/wakeup_fd_posix.c
index 525369c356..046208abc8 100644
--- a/src/core/lib/iomgr/wakeup_fd_posix.c
+++ b/src/core/lib/iomgr/wakeup_fd_posix.c
@@ -53,16 +53,16 @@ void grpc_wakeup_fd_global_init(void) {
 
 void grpc_wakeup_fd_global_destroy(void) { wakeup_fd_vtable = NULL; }
 
-void grpc_wakeup_fd_init(grpc_wakeup_fd *fd_info) {
-  wakeup_fd_vtable->init(fd_info);
+grpc_error *grpc_wakeup_fd_init(grpc_wakeup_fd *fd_info) {
+  return wakeup_fd_vtable->init(fd_info);
 }
 
-void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd *fd_info) {
-  wakeup_fd_vtable->consume(fd_info);
+grpc_error *grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd *fd_info) {
+  return wakeup_fd_vtable->consume(fd_info);
 }
 
-void grpc_wakeup_fd_wakeup(grpc_wakeup_fd *fd_info) {
-  wakeup_fd_vtable->wakeup(fd_info);
+grpc_error *grpc_wakeup_fd_wakeup(grpc_wakeup_fd *fd_info) {
+  return wakeup_fd_vtable->wakeup(fd_info);
 }
 
 void grpc_wakeup_fd_destroy(grpc_wakeup_fd *fd_info) {
diff --git a/src/core/lib/iomgr/wakeup_fd_posix.h b/src/core/lib/iomgr/wakeup_fd_posix.h
index 6b069c1837..e269f242d8 100644
--- a/src/core/lib/iomgr/wakeup_fd_posix.h
+++ b/src/core/lib/iomgr/wakeup_fd_posix.h
@@ -62,6 +62,8 @@
 #ifndef GRPC_CORE_LIB_IOMGR_WAKEUP_FD_POSIX_H
 #define GRPC_CORE_LIB_IOMGR_WAKEUP_FD_POSIX_H
 
+#include "src/core/lib/iomgr/error.h"
+
 void grpc_wakeup_fd_global_init(void);
 void grpc_wakeup_fd_global_destroy(void);
 
@@ -72,9 +74,9 @@ void grpc_wakeup_fd_global_init_force_fallback(void);
 typedef struct grpc_wakeup_fd grpc_wakeup_fd;
 
 typedef struct grpc_wakeup_fd_vtable {
-  void (*init)(grpc_wakeup_fd* fd_info);
-  void (*consume)(grpc_wakeup_fd* fd_info);
-  void (*wakeup)(grpc_wakeup_fd* fd_info);
+  grpc_error* (*init)(grpc_wakeup_fd* fd_info);
+  grpc_error* (*consume)(grpc_wakeup_fd* fd_info);
+  grpc_error* (*wakeup)(grpc_wakeup_fd* fd_info);
   void (*destroy)(grpc_wakeup_fd* fd_info);
   /* Must be called before calling any other functions */
   int (*check_availability)(void);
@@ -89,9 +91,10 @@ extern int grpc_allow_specialized_wakeup_fd;
 
 #define GRPC_WAKEUP_FD_GET_READ_FD(fd_info) ((fd_info)->read_fd)
 
-void grpc_wakeup_fd_init(grpc_wakeup_fd* fd_info);
-void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd* fd_info);
-void grpc_wakeup_fd_wakeup(grpc_wakeup_fd* fd_info);
+grpc_error* grpc_wakeup_fd_init(grpc_wakeup_fd* fd_info) GRPC_MUST_USE_RESULT;
+grpc_error* grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd* fd_info)
+    GRPC_MUST_USE_RESULT;
+grpc_error* grpc_wakeup_fd_wakeup(grpc_wakeup_fd* fd_info) GRPC_MUST_USE_RESULT;
 void grpc_wakeup_fd_destroy(grpc_wakeup_fd* fd_info);
 
 /* Defined in some specialized implementation's .c file, or by
diff --git a/src/core/lib/iomgr/workqueue.h b/src/core/lib/iomgr/workqueue.h
index 3e2b223670..5cc40eea50 100644
--- a/src/core/lib/iomgr/workqueue.h
+++ b/src/core/lib/iomgr/workqueue.h
@@ -43,14 +43,15 @@
 #include "src/core/lib/iomgr/workqueue_posix.h"
 #endif
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 #include "src/core/lib/iomgr/workqueue_windows.h"
 #endif
 
 /* grpc_workqueue is forward declared in exec_ctx.h */
 
 /** Create a work queue */
-grpc_workqueue *grpc_workqueue_create(grpc_exec_ctx *exec_ctx);
+grpc_error *grpc_workqueue_create(grpc_exec_ctx *exec_ctx,
+                                  grpc_workqueue **workqueue);
 
 void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue);
 
@@ -77,7 +78,7 @@ void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx,
                                    grpc_pollset *pollset);
 
 /** Add a work item to a workqueue */
-void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure,
-                         int success);
+void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
+                            grpc_closure *closure, grpc_error *error);
 
 #endif /* GRPC_CORE_LIB_IOMGR_WORKQUEUE_H */
diff --git a/src/core/lib/iomgr/workqueue_posix.c b/src/core/lib/iomgr/workqueue_posix.c
index 80e7a0b206..45e0f6063b 100644
--- a/src/core/lib/iomgr/workqueue_posix.c
+++ b/src/core/lib/iomgr/workqueue_posix.c
@@ -45,22 +45,27 @@
 
 #include "src/core/lib/iomgr/ev_posix.h"
 
-static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success);
+static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 
-grpc_workqueue *grpc_workqueue_create(grpc_exec_ctx *exec_ctx) {
+grpc_error *grpc_workqueue_create(grpc_exec_ctx *exec_ctx,
+                                  grpc_workqueue **workqueue) {
   char name[32];
-  grpc_workqueue *workqueue = gpr_malloc(sizeof(grpc_workqueue));
-  gpr_ref_init(&workqueue->refs, 1);
-  gpr_mu_init(&workqueue->mu);
-  workqueue->closure_list.head = workqueue->closure_list.tail = NULL;
-  grpc_wakeup_fd_init(&workqueue->wakeup_fd);
-  sprintf(name, "workqueue:%p", (void *)workqueue);
-  workqueue->wakeup_read_fd =
-      grpc_fd_create(GRPC_WAKEUP_FD_GET_READ_FD(&workqueue->wakeup_fd), name);
-  grpc_closure_init(&workqueue->read_closure, on_readable, workqueue);
-  grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd,
-                         &workqueue->read_closure);
-  return workqueue;
+  *workqueue = gpr_malloc(sizeof(grpc_workqueue));
+  gpr_ref_init(&(*workqueue)->refs, 1);
+  gpr_mu_init(&(*workqueue)->mu);
+  (*workqueue)->closure_list.head = (*workqueue)->closure_list.tail = NULL;
+  grpc_error *err = grpc_wakeup_fd_init(&(*workqueue)->wakeup_fd);
+  if (err != GRPC_ERROR_NONE) {
+    gpr_free(*workqueue);
+    return err;
+  }
+  sprintf(name, "workqueue:%p", (void *)(*workqueue));
+  (*workqueue)->wakeup_read_fd = grpc_fd_create(
+      GRPC_WAKEUP_FD_GET_READ_FD(&(*workqueue)->wakeup_fd), name);
+  grpc_closure_init(&(*workqueue)->read_closure, on_readable, *workqueue);
+  grpc_fd_notify_on_read(exec_ctx, (*workqueue)->wakeup_read_fd,
+                         &(*workqueue)->read_closure);
+  return GRPC_ERROR_NONE;
 }
 
 static void workqueue_destroy(grpc_exec_ctx *exec_ctx,
@@ -103,17 +108,14 @@ void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx,
 
 void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
   gpr_mu_lock(&workqueue->mu);
-  if (grpc_closure_list_empty(workqueue->closure_list)) {
-    grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
-  }
   grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
   gpr_mu_unlock(&workqueue->mu);
 }
 
-static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_workqueue *workqueue = arg;
 
-  if (!success) {
+  if (error != GRPC_ERROR_NONE) {
     gpr_mu_destroy(&workqueue->mu);
     /* HACK: let wakeup_fd code know that we stole the fd */
     workqueue->wakeup_fd.read_fd = 0;
@@ -123,20 +125,32 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   } else {
     gpr_mu_lock(&workqueue->mu);
     grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
-    grpc_wakeup_fd_consume_wakeup(&workqueue->wakeup_fd);
+    error = grpc_wakeup_fd_consume_wakeup(&workqueue->wakeup_fd);
     gpr_mu_unlock(&workqueue->mu);
-    grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd,
-                           &workqueue->read_closure);
+    if (error == GRPC_ERROR_NONE) {
+      grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd,
+                             &workqueue->read_closure);
+    } else {
+      /* recurse to get error handling */
+      on_readable(exec_ctx, arg, error);
+    }
   }
 }
 
-void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure,
-                         int success) {
+void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
+                            grpc_closure *closure, grpc_error *error) {
+  grpc_error *push_error = GRPC_ERROR_NONE;
   gpr_mu_lock(&workqueue->mu);
   if (grpc_closure_list_empty(workqueue->closure_list)) {
-    grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
+    push_error = grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
+  }
+  grpc_closure_list_append(&workqueue->closure_list, closure, error);
+  if (push_error != GRPC_ERROR_NONE) {
+    const char *msg = grpc_error_string(push_error);
+    gpr_log(GPR_ERROR, "Failed to push to workqueue: %s", msg);
+    grpc_error_free_string(msg);
+    grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
   }
-  grpc_closure_list_add(&workqueue->closure_list, closure, success);
   gpr_mu_unlock(&workqueue->mu);
 }
 
diff --git a/src/core/lib/iomgr/workqueue_windows.c b/src/core/lib/iomgr/workqueue_windows.c
index c3c0446a57..275f040b1c 100644
--- a/src/core/lib/iomgr/workqueue_windows.c
+++ b/src/core/lib/iomgr/workqueue_windows.c
@@ -33,8 +33,8 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include "src/core/lib/iomgr/workqueue.h"
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c
index 50082cd7ee..51813d0461 100644
--- a/src/core/lib/profiling/basic_timers.c
+++ b/src/core/lib/profiling/basic_timers.c
@@ -141,11 +141,11 @@ static void write_log(gpr_timer_log *log) {
       entry->tm = gpr_time_0(entry->tm.clock_type);
     }
     fprintf(output_file,
-            "{\"t\": %lld.%09d, \"thd\": \"%d\", \"type\": \"%c\", \"tag\": "
+            "{\"t\": %" PRId64
+            ".%09d, \"thd\": \"%d\", \"type\": \"%c\", \"tag\": "
             "\"%s\", \"file\": \"%s\", \"line\": %d, \"imp\": %d}\n",
-            (long long)entry->tm.tv_sec, (int)entry->tm.tv_nsec, entry->thd,
-            entry->type, entry->tagstr, entry->file, entry->line,
-            entry->important);
+            entry->tm.tv_sec, entry->tm.tv_nsec, entry->thd, entry->type,
+            entry->tagstr, entry->file, entry->line, entry->important);
   }
 }
 
diff --git a/src/core/lib/security/security_context.c b/src/core/lib/security/context/security_context.c
index 343e0b5b8b..127b13ee50 100644
--- a/src/core/lib/security/security_context.c
+++ b/src/core/lib/security/context/security_context.c
@@ -33,7 +33,7 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
diff --git a/src/core/lib/security/security_context.h b/src/core/lib/security/context/security_context.h
index 81161ec47d..ef0c06b1fb 100644
--- a/src/core/lib/security/security_context.h
+++ b/src/core/lib/security/context/security_context.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
+#ifndef GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
+#define GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
 
 #include "src/core/lib/iomgr/pollset.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 /* --- grpc_auth_context ---
 
@@ -111,4 +111,4 @@ grpc_auth_context *grpc_auth_context_from_arg(const grpc_arg *arg);
 grpc_auth_context *grpc_find_auth_context_in_args(
     const grpc_channel_args *args);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H */
diff --git a/src/core/lib/security/credentials.c b/src/core/lib/security/credentials.c
deleted file mode 100644
index fd5ad3589b..0000000000
--- a/src/core/lib/security/credentials.c
+++ /dev/null
@@ -1,1296 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/lib/security/credentials.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/http_client_filter.h"
-#include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/http/parser.h"
-#include "src/core/lib/iomgr/executor.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/surface/api_trace.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/time.h>
-
-/* -- Common. -- */
-
-struct grpc_credentials_metadata_request {
-  grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
-  void *user_data;
-};
-
-static grpc_credentials_metadata_request *
-grpc_credentials_metadata_request_create(grpc_call_credentials *creds,
-                                         grpc_credentials_metadata_cb cb,
-                                         void *user_data) {
-  grpc_credentials_metadata_request *r =
-      gpr_malloc(sizeof(grpc_credentials_metadata_request));
-  r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
-  return r;
-}
-
-static void grpc_credentials_metadata_request_destroy(
-    grpc_credentials_metadata_request *r) {
-  grpc_call_credentials_unref(r->creds);
-  gpr_free(r);
-}
-
-grpc_channel_credentials *grpc_channel_credentials_ref(
-    grpc_channel_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
-  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
-  grpc_channel_credentials_unref(creds);
-}
-
-grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_call_credentials_unref(grpc_call_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_call_credentials_release(grpc_call_credentials *creds) {
-  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
-  grpc_call_credentials_unref(creds);
-}
-
-void grpc_call_credentials_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-    }
-    return;
-  }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pollset, context, cb,
-                                      user_data);
-}
-
-grpc_security_status grpc_channel_credentials_create_security_connector(
-    grpc_channel_credentials *channel_creds, const char *target,
-    const grpc_channel_args *args, grpc_channel_security_connector **sc,
-    grpc_channel_args **new_args) {
-  *new_args = NULL;
-  if (channel_creds == NULL) {
-    return GRPC_SECURITY_ERROR;
-  }
-  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
-  return channel_creds->vtable->create_security_connector(
-      channel_creds, NULL, target, args, sc, new_args);
-}
-
-grpc_server_credentials *grpc_server_credentials_ref(
-    grpc_server_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_server_credentials_unref(grpc_server_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-      creds->processor.destroy(creds->processor.state);
-    }
-    gpr_free(creds);
-  }
-}
-
-void grpc_server_credentials_release(grpc_server_credentials *creds) {
-  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
-  grpc_server_credentials_unref(creds);
-}
-
-grpc_security_status grpc_server_credentials_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
-    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
-    return GRPC_SECURITY_ERROR;
-  }
-  return creds->vtable->create_security_connector(creds, sc);
-}
-
-void grpc_server_credentials_set_auth_metadata_processor(
-    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
-  GRPC_API_TRACE(
-      "grpc_server_credentials_set_auth_metadata_processor("
-      "creds=%p, "
-      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
-      3, (creds, (void *)(intptr_t)processor.process, processor.state));
-  if (creds == NULL) return;
-  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-    creds->processor.destroy(creds->processor.state);
-  }
-  creds->processor = processor;
-}
-
-static void server_credentials_pointer_arg_destroy(void *p) {
-  grpc_server_credentials_unref(p);
-}
-
-static void *server_credentials_pointer_arg_copy(void *p) {
-  return grpc_server_credentials_ref(p);
-}
-
-static int server_credentials_pointer_cmp(void *a, void *b) {
-  return GPR_ICMP(a, b);
-}
-
-static const grpc_arg_pointer_vtable cred_ptr_vtable = {
-    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
-    server_credentials_pointer_cmp};
-
-grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
-  grpc_arg arg;
-  memset(&arg, 0, sizeof(grpc_arg));
-  arg.type = GRPC_ARG_POINTER;
-  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
-  arg.value.pointer.p = p;
-  arg.value.pointer.vtable = &cred_ptr_vtable;
-  return arg;
-}
-
-grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
-  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
-  if (arg->type != GRPC_ARG_POINTER) {
-    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
-            GRPC_SERVER_CREDENTIALS_ARG);
-    return NULL;
-  }
-  return arg->value.pointer.p;
-}
-
-grpc_server_credentials *grpc_find_server_credentials_in_args(
-    const grpc_channel_args *args) {
-  size_t i;
-  if (args == NULL) return NULL;
-  for (i = 0; i < args->num_args; i++) {
-    grpc_server_credentials *p =
-        grpc_server_credentials_from_arg(&args->args[i]);
-    if (p != NULL) return p;
-  }
-  return NULL;
-}
-
-/* -- Ssl credentials. -- */
-
-static void ssl_destruct(grpc_channel_credentials *creds) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
-  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
-}
-
-static void ssl_server_destruct(grpc_server_credentials *creds) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
-    if (c->config.pem_private_keys[i] != NULL) {
-      gpr_free(c->config.pem_private_keys[i]);
-    }
-    if (c->config.pem_cert_chains[i] != NULL) {
-      gpr_free(c->config.pem_cert_chains[i]);
-    }
-  }
-  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
-  if (c->config.pem_private_keys_sizes != NULL) {
-    gpr_free(c->config.pem_private_keys_sizes);
-  }
-  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
-  if (c->config.pem_cert_chains_sizes != NULL) {
-    gpr_free(c->config.pem_cert_chains_sizes);
-  }
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-}
-
-static grpc_security_status ssl_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_OK;
-  size_t i = 0;
-  const char *overridden_target_name = NULL;
-  grpc_arg new_arg;
-
-  for (i = 0; args && i < args->num_args; i++) {
-    grpc_arg *arg = &args->args[i];
-    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
-        arg->type == GRPC_ARG_STRING) {
-      overridden_target_name = arg->value.string;
-      break;
-    }
-  }
-  status = grpc_ssl_channel_security_connector_create(
-      call_creds, &c->config, target, overridden_target_name, sc);
-  if (status != GRPC_SECURITY_OK) {
-    return status;
-  }
-  new_arg.type = GRPC_ARG_STRING;
-  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
-  new_arg.value.string = "https";
-  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
-  return status;
-}
-
-static grpc_security_status ssl_server_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  return grpc_ssl_server_security_connector_create(&c->config, sc);
-}
-
-static grpc_channel_credentials_vtable ssl_vtable = {
-    ssl_destruct, ssl_create_security_connector};
-
-static grpc_server_credentials_vtable ssl_server_vtable = {
-    ssl_server_destruct, ssl_server_create_security_connector};
-
-static void ssl_copy_key_material(const char *input, unsigned char **output,
-                                  size_t *output_size) {
-  *output_size = strlen(input);
-  *output = gpr_malloc(*output_size);
-  memcpy(*output, input, *output_size);
-}
-
-static void ssl_build_config(const char *pem_root_certs,
-                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-                             grpc_ssl_config *config) {
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (pem_key_cert_pair != NULL) {
-    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pair->private_key,
-                          &config->pem_private_key,
-                          &config->pem_private_key_size);
-    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
-                          &config->pem_cert_chain,
-                          &config->pem_cert_chain_size);
-  }
-}
-
-static void ssl_build_server_config(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    grpc_ssl_server_config *config) {
-  size_t i;
-  config->client_certificate_request = client_certificate_request;
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (num_key_cert_pairs > 0) {
-    GPR_ASSERT(pem_key_cert_pairs != NULL);
-    config->pem_private_keys =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_cert_chains =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_private_keys_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-    config->pem_cert_chains_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-  }
-  config->num_key_cert_pairs = num_key_cert_pairs;
-  for (i = 0; i < num_key_cert_pairs; i++) {
-    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
-                          &config->pem_private_keys[i],
-                          &config->pem_private_keys_sizes[i]);
-    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
-                          &config->pem_cert_chains[i],
-                          &config->pem_cert_chains_sizes[i]);
-  }
-}
-
-grpc_channel_credentials *grpc_ssl_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-    void *reserved) {
-  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_credentials_create(pem_root_certs=%s, "
-      "pem_key_cert_pair=%p, "
-      "reserved=%p)",
-      3, (pem_root_certs, pem_key_cert_pair, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  c->base.vtable = &ssl_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
-  return &c->base;
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
-  return grpc_ssl_server_credentials_create_ex(
-      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
-      force_client_auth
-          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
-          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
-      reserved);
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    void *reserved) {
-  grpc_ssl_server_credentials *c =
-      gpr_malloc(sizeof(grpc_ssl_server_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_server_credentials_create_ex("
-      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
-      "client_certificate_request=%d, reserved=%p)",
-      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
-          client_certificate_request, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_server_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &ssl_server_vtable;
-  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
-                          num_key_cert_pairs, client_certificate_request,
-                          &c->config);
-  return &c->base;
-}
-
-/* -- Jwt credentials -- */
-
-static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
-  if (c->cached.service_url != NULL) {
-    gpr_free(c->cached.service_url);
-    c->cached.service_url = NULL;
-  }
-  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-}
-
-static void jwt_destruct(grpc_call_credentials *creds) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  grpc_auth_json_key_destruct(&c->key);
-  jwt_reset_cache(c);
-  gpr_mu_destroy(&c->cache_mu);
-}
-
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pollset *pollset,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-
-  /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
-  {
-    gpr_mu_lock(&c->cache_mu);
-    if (c->cached.service_url != NULL &&
-        strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
-        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
-                                   gpr_now(GPR_CLOCK_REALTIME)),
-                      refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md == NULL) {
-    char *jwt = NULL;
-    /* Generate a new jwt. */
-    gpr_mu_lock(&c->cache_mu);
-    jwt_reset_cache(c);
-    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
-                                   c->jwt_lifetime, NULL);
-    if (jwt != NULL) {
-      char *md_value;
-      gpr_asprintf(&md_value, "Bearer %s", jwt);
-      gpr_free(jwt);
-      c->cached.jwt_expiration =
-          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
-      c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
-      gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(jwt_md);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  }
-}
-
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
-
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime) {
-  grpc_service_account_jwt_access_credentials *c;
-  if (!grpc_auth_json_key_is_valid(&key)) {
-    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
-  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &jwt_vtable;
-  c->key = key;
-  c->jwt_lifetime = token_lifetime;
-  gpr_mu_init(&c->cache_mu);
-  jwt_reset_cache(c);
-  return &c->base;
-}
-
-grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
-    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_service_account_jwt_access_credentials_create("
-      "json_key=%s, "
-      "token_lifetime="
-      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
-      "reserved=%p)",
-      5,
-      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
-       (int)token_lifetime.clock_type, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
-}
-
-/* -- Oauth2TokenFetcher credentials -- */
-
-static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-  gpr_mu_destroy(&c->mu);
-  grpc_httpcli_context_destroy(&c->httpcli_context);
-}
-
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const grpc_http_response *response, grpc_credentials_md_store **token_md,
-    gpr_timespec *token_lifetime) {
-  char *null_terminated_body = NULL;
-  char *new_access_token = NULL;
-  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
-  grpc_json *json = NULL;
-
-  if (response == NULL) {
-    gpr_log(GPR_ERROR, "Received NULL response.");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  }
-
-  if (response->body_length > 0) {
-    null_terminated_body = gpr_malloc(response->body_length + 1);
-    null_terminated_body[response->body_length] = '\0';
-    memcpy(null_terminated_body, response->body, response->body_length);
-  }
-
-  if (response->status != 200) {
-    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
-            response->status,
-            null_terminated_body != NULL ? null_terminated_body : "");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  } else {
-    grpc_json *access_token = NULL;
-    grpc_json *token_type = NULL;
-    grpc_json *expires_in = NULL;
-    grpc_json *ptr;
-    json = grpc_json_parse_string(null_terminated_body);
-    if (json == NULL) {
-      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (json->type != GRPC_JSON_OBJECT) {
-      gpr_log(GPR_ERROR, "Response should be a JSON object");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    for (ptr = json->child; ptr; ptr = ptr->next) {
-      if (strcmp(ptr->key, "access_token") == 0) {
-        access_token = ptr;
-      } else if (strcmp(ptr->key, "token_type") == 0) {
-        token_type = ptr;
-      } else if (strcmp(ptr->key, "expires_in") == 0) {
-        expires_in = ptr;
-      }
-    }
-    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
-      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
-                 access_token->value);
-    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
-    token_lifetime->tv_nsec = 0;
-    token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
-    status = GRPC_CREDENTIALS_OK;
-  }
-
-end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(*token_md);
-    *token_md = NULL;
-  }
-  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
-  if (new_access_token != NULL) gpr_free(new_access_token);
-  if (json != NULL) grpc_json_destroy(json);
-  return status;
-}
-
-static void on_oauth2_token_fetcher_http_response(
-    grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_http_response *response) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)r->creds;
-  gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, status);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status);
-  }
-  gpr_mu_unlock(&c->mu);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void oauth2_token_fetcher_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
-    gpr_mu_unlock(&c->mu);
-  }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(cached_access_token_md);
-  } else {
-    c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pollset, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
-  }
-}
-
-static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
-                                      grpc_fetch_oauth2_func fetch_func) {
-  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  gpr_ref_init(&c->base.refcount, 1);
-  gpr_mu_init(&c->mu);
-  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-  c->fetch_func = fetch_func;
-  grpc_httpcli_context_init(&c->httpcli_context);
-}
-
-/* -- GoogleComputeEngine credentials. -- */
-
-static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void compute_engine_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_http_header header = {"Metadata-Flavor", "Google"};
-  grpc_httpcli_request request;
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
-  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  grpc_httpcli_get(exec_ctx, httpcli_context, pollset, &request, deadline,
-                   response_cb, metadata_req);
-}
-
-grpc_call_credentials *grpc_google_compute_engine_credentials_create(
-    void *reserved) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
-  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
-  c->base.vtable = &compute_engine_vtable;
-  return &c->base;
-}
-
-/* -- GoogleRefreshToken credentials. -- */
-
-static void refresh_token_destruct(grpc_call_credentials *creds) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)creds;
-  grpc_auth_refresh_token_destruct(&c->refresh_token);
-  oauth2_token_fetcher_destruct(&c->base.base);
-}
-
-static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void refresh_token_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)metadata_req->creds;
-  grpc_http_header header = {"Content-Type",
-                             "application/x-www-form-urlencoded"};
-  grpc_httpcli_request request;
-  char *body = NULL;
-  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
-               c->refresh_token.client_id, c->refresh_token.client_secret,
-               c->refresh_token.refresh_token);
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
-  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  request.handshaker = &grpc_httpcli_ssl;
-  grpc_httpcli_post(exec_ctx, httpcli_context, pollset, &request, body,
-                    strlen(body), deadline, response_cb, metadata_req);
-  gpr_free(body);
-}
-
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token refresh_token) {
-  grpc_google_refresh_token_credentials *c;
-  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
-    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
-  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
-  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
-  c->base.base.vtable = &refresh_token_vtable;
-  c->refresh_token = refresh_token;
-  return &c->base.base;
-}
-
-grpc_call_credentials *grpc_google_refresh_token_credentials_create(
-    const char *json_refresh_token, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
-      "reserved=%p)",
-      2, (json_refresh_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
-      grpc_auth_refresh_token_create_from_string(json_refresh_token));
-}
-
-/* -- Metadata-only credentials. -- */
-
-static void md_only_test_destruct(grpc_call_credentials *creds) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(c->md_store);
-}
-
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, bool success) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void md_only_test_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
-  if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    grpc_executor_enqueue(
-        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
-
-grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
-  grpc_md_only_test_credentials *c =
-      gpr_malloc(sizeof(grpc_md_only_test_credentials));
-  memset(c, 0, sizeof(grpc_md_only_test_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &md_only_test_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
-  c->is_async = is_async;
-  return &c->base;
-}
-
-/* -- Oauth2 Access Token credentials. -- */
-
-static void access_token_destruct(grpc_call_credentials *creds) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-}
-
-static void access_token_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
-
-grpc_call_credentials *grpc_access_token_credentials_create(
-    const char *access_token, void *reserved) {
-  grpc_access_token_credentials *c =
-      gpr_malloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
-  GRPC_API_TRACE(
-      "grpc_access_token_credentials_create(access_token=%s, "
-      "reserved=%p)",
-      2, (access_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_access_token_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &access_token_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
-  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
-  gpr_free(token_md_value);
-  return &c->base;
-}
-
-/* -- Fake transport security credentials. -- */
-
-static grpc_security_status fake_transport_security_create_security_connector(
-    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  *sc = grpc_fake_channel_security_connector_create(call_creds);
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_security_status
-fake_transport_security_server_create_security_connector(
-    grpc_server_credentials *c, grpc_server_security_connector **sc) {
-  *sc = grpc_fake_server_security_connector_create();
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_channel_credentials_vtable
-    fake_transport_security_credentials_vtable = {
-        NULL, fake_transport_security_create_security_connector};
-
-static grpc_server_credentials_vtable
-    fake_transport_security_server_credentials_vtable = {
-        NULL, fake_transport_security_server_create_security_connector};
-
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
-    void) {
-  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
-  memset(c, 0, sizeof(grpc_channel_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  c->vtable = &fake_transport_security_credentials_vtable;
-  gpr_ref_init(&c->refcount, 1);
-  return c;
-}
-
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void) {
-  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
-  memset(c, 0, sizeof(grpc_server_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  gpr_ref_init(&c->refcount, 1);
-  c->vtable = &fake_transport_security_server_credentials_vtable;
-  return c;
-}
-
-/* -- Composite call credentials. -- */
-
-typedef struct {
-  grpc_composite_call_credentials *composite_creds;
-  size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
-  grpc_pollset *pollset;
-  grpc_credentials_metadata_cb cb;
-} grpc_composite_call_credentials_metadata_context;
-
-static void composite_call_destruct(grpc_call_credentials *creds) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
-    grpc_call_credentials_unref(c->inner.creds_array[i]);
-  }
-  gpr_free(c->inner.creds_array);
-}
-
-static void composite_call_md_context_destroy(
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status) {
-  grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
-    }
-  }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(
-        exec_ctx, inner_creds, ctx->pollset, ctx->auth_md_context,
-        composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
-  composite_call_md_context_destroy(ctx);
-}
-
-static void composite_call_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context auth_md_context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  grpc_composite_call_credentials_metadata_context *ctx;
-
-  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
-  ctx->composite_creds = c;
-  ctx->pollset = pollset;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], pollset,
-      auth_md_context, composite_call_metadata_cb, ctx);
-}
-
-static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
-
-static grpc_call_credentials_array get_creds_array(
-    grpc_call_credentials **creds_addr) {
-  grpc_call_credentials_array result;
-  grpc_call_credentials *creds = *creds_addr;
-  result.creds_array = creds_addr;
-  result.num_creds = 1;
-  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    result = *grpc_composite_call_credentials_get_credentials(creds);
-  }
-  return result;
-}
-
-grpc_call_credentials *grpc_composite_call_credentials_create(
-    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
-    void *reserved) {
-  size_t i;
-  size_t creds_array_byte_size;
-  grpc_call_credentials_array creds1_array;
-  grpc_call_credentials_array creds2_array;
-  grpc_composite_call_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
-      "reserved=%p)",
-      3, (creds1, creds2, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(creds1 != NULL);
-  GPR_ASSERT(creds2 != NULL);
-  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
-  memset(c, 0, sizeof(grpc_composite_call_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
-  c->base.vtable = &composite_call_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  creds1_array = get_creds_array(&creds1);
-  creds2_array = get_creds_array(&creds2);
-  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
-  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
-  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
-  memset(c->inner.creds_array, 0, creds_array_byte_size);
-  for (i = 0; i < creds1_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
-    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
-  }
-  for (i = 0; i < creds2_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
-    c->inner.creds_array[i + creds1_array.num_creds] =
-        grpc_call_credentials_ref(cur_creds);
-  }
-  return &c->base;
-}
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
-  const grpc_composite_call_credentials *c =
-      (const grpc_composite_call_credentials *)creds;
-  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
-  return &c->inner;
-}
-
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds) {
-  size_t i;
-  if (strcmp(creds->type, type) == 0) {
-    if (composite_creds != NULL) *composite_creds = NULL;
-    return creds;
-  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    const grpc_call_credentials_array *inner_creds_array =
-        grpc_composite_call_credentials_get_credentials(creds);
-    for (i = 0; i < inner_creds_array->num_creds; i++) {
-      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
-        if (composite_creds != NULL) *composite_creds = creds;
-        return inner_creds_array->creds_array[i];
-      }
-    }
-  }
-  return NULL;
-}
-
-/* -- IAM credentials. -- */
-
-static void iam_destruct(grpc_call_credentials *creds) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(c->iam_md);
-}
-
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pollset *pollset,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
-
-grpc_call_credentials *grpc_google_iam_credentials_create(
-    const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
-      "reserved=%p)",
-      3, (token, authority_selector, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(token != NULL);
-  GPR_ASSERT(authority_selector != NULL);
-  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
-  memset(c, 0, sizeof(grpc_google_iam_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
-  c->base.vtable = &iam_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
-  return &c->base;
-}
-
-/* -- Plugin credentials. -- */
-
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
-static void plugin_destruct(grpc_call_credentials *creds) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
-    c->plugin.destroy(c->plugin.state);
-  }
-}
-
-static void plugin_md_request_metadata_ready(void *request,
-                                             const grpc_metadata *md,
-                                             size_t num_md,
-                                             grpc_status_code status,
-                                             const char *error_details) {
-  /* called from application code */
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  } else {
-    size_t i;
-    grpc_credentials_md *md_array = NULL;
-    if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
-        md_array[i].value =
-            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
-      }
-    }
-    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
-    if (md_array != NULL) {
-      for (i = 0; i < num_md; i++) {
-        gpr_slice_unref(md_array[i].key);
-        gpr_slice_unref(md_array[i].value);
-      }
-      gpr_free(md_array);
-    }
-  }
-  gpr_free(r);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_credentials *creds,
-                                        grpc_pollset *pollset,
-                                        grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
-    memset(request, 0, sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
-    c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
-
-grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
-    grpc_metadata_credentials_plugin plugin, void *reserved) {
-  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
-  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(*c));
-  c->base.type = plugin.type;
-  c->base.vtable = &plugin_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->plugin = plugin;
-  return &c->base;
-}
-
-/* -- Composite channel credentials. -- */
-
-static void composite_channel_destruct(grpc_channel_credentials *creds) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_channel_credentials_unref(c->inner_creds);
-  grpc_call_credentials_unref(c->call_creds);
-}
-
-static grpc_security_status composite_channel_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_ERROR;
-
-  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
-             c->inner_creds->vtable != NULL &&
-             c->inner_creds->vtable->create_security_connector != NULL);
-  /* If we are passed a call_creds, create a call composite to pass it
-     downstream. */
-  if (call_creds != NULL) {
-    grpc_call_credentials *composite_call_creds =
-        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, composite_call_creds, target, args, sc, new_args);
-    grpc_call_credentials_unref(composite_call_creds);
-  } else {
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, c->call_creds, target, args, sc, new_args);
-  }
-  return status;
-}
-
-static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
-    composite_channel_destruct, composite_channel_create_security_connector};
-
-grpc_channel_credentials *grpc_composite_channel_credentials_create(
-    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
-    void *reserved) {
-  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
-  memset(c, 0, sizeof(*c));
-  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
-  GRPC_API_TRACE(
-      "grpc_composite_channel_credentials_create(channel_creds=%p, "
-      "call_creds=%p, reserved=%p)",
-      3, (channel_creds, call_creds, reserved));
-  c->base.type = channel_creds->type;
-  c->base.vtable = &composite_channel_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
-  c->call_creds = grpc_call_credentials_ref(call_creds);
-  return &c->base;
-}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
new file mode 100644
index 0000000000..850e41e646
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -0,0 +1,264 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/composite/composite_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Composite call credentials. -- */
+
+typedef struct {
+  grpc_composite_call_credentials *composite_creds;
+  size_t creds_index;
+  grpc_credentials_md_store *md_elems;
+  grpc_auth_metadata_context auth_md_context;
+  void *user_data;
+  grpc_polling_entity *pollent;
+  grpc_credentials_metadata_cb cb;
+} grpc_composite_call_credentials_metadata_context;
+
+static void composite_call_destruct(grpc_call_credentials *creds) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->inner.num_creds; i++) {
+    grpc_call_credentials_unref(c->inner.creds_array[i]);
+  }
+  gpr_free(c->inner.creds_array);
+}
+
+static void composite_call_md_context_destroy(
+    grpc_composite_call_credentials_metadata_context *ctx) {
+  grpc_credentials_md_store_unref(ctx->md_elems);
+  gpr_free(ctx);
+}
+
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
+                                       grpc_credentials_md *md_elems,
+                                       size_t num_md,
+                                       grpc_credentials_status status,
+                                       const char *error_details) {
+  grpc_composite_call_credentials_metadata_context *ctx =
+      (grpc_composite_call_credentials_metadata_context *)user_data;
+  if (status != GRPC_CREDENTIALS_OK) {
+    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status, error_details);
+    return;
+  }
+
+  /* Copy the metadata in the context. */
+  if (num_md > 0) {
+    size_t i;
+    for (i = 0; i < num_md; i++) {
+      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
+                                    md_elems[i].value);
+    }
+  }
+
+  /* See if we need to get some more metadata. */
+  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    grpc_call_credentials_get_request_metadata(
+        exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+        composite_call_metadata_cb, ctx);
+    return;
+  }
+
+  /* We're done!. */
+  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
+          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK, NULL);
+  composite_call_md_context_destroy(ctx);
+}
+
+static void composite_call_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context auth_md_context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  grpc_composite_call_credentials_metadata_context *ctx;
+
+  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
+  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
+  ctx->auth_md_context = auth_md_context;
+  ctx->user_data = user_data;
+  ctx->cb = cb;
+  ctx->composite_creds = c;
+  ctx->pollent = pollent;
+  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
+  grpc_call_credentials_get_request_metadata(
+      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pollent,
+      auth_md_context, composite_call_metadata_cb, ctx);
+}
+
+static grpc_call_credentials_vtable composite_call_credentials_vtable = {
+    composite_call_destruct, composite_call_get_request_metadata};
+
+static grpc_call_credentials_array get_creds_array(
+    grpc_call_credentials **creds_addr) {
+  grpc_call_credentials_array result;
+  grpc_call_credentials *creds = *creds_addr;
+  result.creds_array = creds_addr;
+  result.num_creds = 1;
+  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    result = *grpc_composite_call_credentials_get_credentials(creds);
+  }
+  return result;
+}
+
+grpc_call_credentials *grpc_composite_call_credentials_create(
+    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
+    void *reserved) {
+  size_t i;
+  size_t creds_array_byte_size;
+  grpc_call_credentials_array creds1_array;
+  grpc_call_credentials_array creds2_array;
+  grpc_composite_call_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
+      "reserved=%p)",
+      3, (creds1, creds2, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(creds1 != NULL);
+  GPR_ASSERT(creds2 != NULL);
+  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
+  memset(c, 0, sizeof(grpc_composite_call_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
+  c->base.vtable = &composite_call_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  creds1_array = get_creds_array(&creds1);
+  creds2_array = get_creds_array(&creds2);
+  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
+  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
+  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
+  memset(c->inner.creds_array, 0, creds_array_byte_size);
+  for (i = 0; i < creds1_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
+    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
+  }
+  for (i = 0; i < creds2_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
+    c->inner.creds_array[i + creds1_array.num_creds] =
+        grpc_call_credentials_ref(cur_creds);
+  }
+  return &c->base;
+}
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
+  const grpc_composite_call_credentials *c =
+      (const grpc_composite_call_credentials *)creds;
+  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
+  return &c->inner;
+}
+
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds) {
+  size_t i;
+  if (strcmp(creds->type, type) == 0) {
+    if (composite_creds != NULL) *composite_creds = NULL;
+    return creds;
+  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    const grpc_call_credentials_array *inner_creds_array =
+        grpc_composite_call_credentials_get_credentials(creds);
+    for (i = 0; i < inner_creds_array->num_creds; i++) {
+      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
+        if (composite_creds != NULL) *composite_creds = creds;
+        return inner_creds_array->creds_array[i];
+      }
+    }
+  }
+  return NULL;
+}
+
+/* -- Composite channel credentials. -- */
+
+static void composite_channel_destruct(grpc_channel_credentials *creds) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_channel_credentials_unref(c->inner_creds);
+  grpc_call_credentials_unref(c->call_creds);
+}
+
+static grpc_security_status composite_channel_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_ERROR;
+
+  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
+             c->inner_creds->vtable != NULL &&
+             c->inner_creds->vtable->create_security_connector != NULL);
+  /* If we are passed a call_creds, create a call composite to pass it
+     downstream. */
+  if (call_creds != NULL) {
+    grpc_call_credentials *composite_call_creds =
+        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, composite_call_creds, target, args, sc, new_args);
+    grpc_call_credentials_unref(composite_call_creds);
+  } else {
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, c->call_creds, target, args, sc, new_args);
+  }
+  return status;
+}
+
+static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
+    composite_channel_destruct, composite_channel_create_security_connector};
+
+grpc_channel_credentials *grpc_composite_channel_credentials_create(
+    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
+    void *reserved) {
+  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
+  memset(c, 0, sizeof(*c));
+  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
+  GRPC_API_TRACE(
+      "grpc_composite_channel_credentials_create(channel_creds=%p, "
+      "call_creds=%p, reserved=%p)",
+      3, (channel_creds, call_creds, reserved));
+  c->base.type = channel_creds->type;
+  c->base.vtable = &composite_channel_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
+  c->call_creds = grpc_call_credentials_ref(call_creds);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h
new file mode 100644
index 0000000000..0d8966f464
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.h
@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials **creds_array;
+  size_t num_creds;
+} grpc_call_credentials_array;
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(
+    grpc_call_credentials *composite_creds);
+
+/* Returns creds if creds is of the specified type or the inner creds of the
+   specified type (if found), if the creds is of type COMPOSITE.
+   If composite_creds is not NULL, *composite_creds will point to creds if of
+   type COMPOSITE in case of success. */
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds);
+
+/* -- Channel composite credentials. -- */
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_channel_credentials *inner_creds;
+  grpc_call_credentials *call_creds;
+} grpc_composite_channel_credentials;
+
+/* -- Composite credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_call_credentials_array inner;
+} grpc_composite_call_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
new file mode 100644
index 0000000000..029a357261
--- /dev/null
+++ b/src/core/lib/security/credentials/credentials.c
@@ -0,0 +1,233 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+
+/* -- Common. -- */
+
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data) {
+  grpc_credentials_metadata_request *r =
+      gpr_malloc(sizeof(grpc_credentials_metadata_request));
+  memset(&r->response, 0, sizeof(r->response));
+  r->creds = grpc_call_credentials_ref(creds);
+  r->cb = cb;
+  r->user_data = user_data;
+  return r;
+}
+
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r) {
+  grpc_call_credentials_unref(r->creds);
+  grpc_http_response_destroy(&r->response);
+  gpr_free(r);
+}
+
+grpc_channel_credentials *grpc_channel_credentials_ref(
+    grpc_channel_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
+  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
+  grpc_channel_credentials_unref(creds);
+}
+
+grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_call_credentials_unref(grpc_call_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_call_credentials_release(grpc_call_credentials *creds) {
+  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
+  grpc_call_credentials_unref(creds);
+}
+
+void grpc_call_credentials_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
+    if (cb != NULL) {
+      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
+    }
+    return;
+  }
+  creds->vtable->get_request_metadata(exec_ctx, creds, pollent, context, cb,
+                                      user_data);
+}
+
+grpc_security_status grpc_channel_credentials_create_security_connector(
+    grpc_channel_credentials *channel_creds, const char *target,
+    const grpc_channel_args *args, grpc_channel_security_connector **sc,
+    grpc_channel_args **new_args) {
+  *new_args = NULL;
+  if (channel_creds == NULL) {
+    return GRPC_SECURITY_ERROR;
+  }
+  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
+  return channel_creds->vtable->create_security_connector(
+      channel_creds, NULL, target, args, sc, new_args);
+}
+
+grpc_server_credentials *grpc_server_credentials_ref(
+    grpc_server_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_server_credentials_unref(grpc_server_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+      creds->processor.destroy(creds->processor.state);
+    }
+    gpr_free(creds);
+  }
+}
+
+void grpc_server_credentials_release(grpc_server_credentials *creds) {
+  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
+  grpc_server_credentials_unref(creds);
+}
+
+grpc_security_status grpc_server_credentials_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
+    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
+    return GRPC_SECURITY_ERROR;
+  }
+  return creds->vtable->create_security_connector(creds, sc);
+}
+
+void grpc_server_credentials_set_auth_metadata_processor(
+    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
+  GRPC_API_TRACE(
+      "grpc_server_credentials_set_auth_metadata_processor("
+      "creds=%p, "
+      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
+      3, (creds, (void *)(intptr_t)processor.process, processor.state));
+  if (creds == NULL) return;
+  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+    creds->processor.destroy(creds->processor.state);
+  }
+  creds->processor = processor;
+}
+
+static void server_credentials_pointer_arg_destroy(void *p) {
+  grpc_server_credentials_unref(p);
+}
+
+static void *server_credentials_pointer_arg_copy(void *p) {
+  return grpc_server_credentials_ref(p);
+}
+
+static int server_credentials_pointer_cmp(void *a, void *b) {
+  return GPR_ICMP(a, b);
+}
+
+static const grpc_arg_pointer_vtable cred_ptr_vtable = {
+    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
+    server_credentials_pointer_cmp};
+
+grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
+  grpc_arg arg;
+  memset(&arg, 0, sizeof(grpc_arg));
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
+  arg.value.pointer.p = p;
+  arg.value.pointer.vtable = &cred_ptr_vtable;
+  return arg;
+}
+
+grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
+  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
+  if (arg->type != GRPC_ARG_POINTER) {
+    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
+            GRPC_SERVER_CREDENTIALS_ARG);
+    return NULL;
+  }
+  return arg->value.pointer.p;
+}
+
+grpc_server_credentials *grpc_find_server_credentials_in_args(
+    const grpc_channel_args *args) {
+  size_t i;
+  if (args == NULL) return NULL;
+  for (i = 0; i < args->num_args; i++) {
+    grpc_server_credentials *p =
+        grpc_server_credentials_from_arg(&args->args[i]);
+    if (p != NULL) return p;
+  }
+  return NULL;
+}
diff --git a/src/core/lib/security/credentials.h b/src/core/lib/security/credentials/credentials.h
index 0373ceaa3f..8e9d842ead 100644
--- a/src/core/lib/security/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
-#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
@@ -41,8 +41,8 @@
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
-#include "src/core/lib/security/json_token.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 struct grpc_http_response;
 
@@ -69,10 +69,6 @@ typedef enum {
   "x-goog-iam-authorization-token"
 #define GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY "x-goog-iam-authority-selector"
 
-#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
-#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
-  "application_default_credentials.json"
-
 #define GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS 60
 
 #define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata"
@@ -160,16 +156,16 @@ void grpc_credentials_md_store_unref(grpc_credentials_md_store *store);
 
 /* --- grpc_call_credentials. --- */
 
-typedef void (*grpc_credentials_metadata_cb)(grpc_exec_ctx *exec_ctx,
-                                             void *user_data,
-                                             grpc_credentials_md *md_elems,
-                                             size_t num_md,
-                                             grpc_credentials_status status);
+/* error_details must be NULL if status is GRPC_CREDENTIALS_OK. */
+typedef void (*grpc_credentials_metadata_cb)(
+    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
+    size_t num_md, grpc_credentials_status status, const char *error_details);
 
 typedef struct {
   void (*destruct)(grpc_call_credentials *c);
   void (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
-                               grpc_call_credentials *c, grpc_pollset *pollset,
+                               grpc_call_credentials *c,
+                               grpc_polling_entity *pollent,
                                grpc_auth_metadata_context context,
                                grpc_credentials_metadata_cb cb,
                                void *user_data);
@@ -185,51 +181,14 @@ grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds);
 void grpc_call_credentials_unref(grpc_call_credentials *creds);
 void grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data);
 
-typedef struct {
-  grpc_call_credentials **creds_array;
-  size_t num_creds;
-} grpc_call_credentials_array;
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(
-    grpc_call_credentials *composite_creds);
-
-/* Returns creds if creds is of the specified type or the inner creds of the
-   specified type (if found), if the creds is of type COMPOSITE.
-   If composite_creds is not NULL, *composite_creds will point to creds if of
-   type COMPOSITE in case of success. */
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds);
-
-/* Exposed for testing only. */
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
-
-void grpc_flush_cached_google_default_credentials(void);
-
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
     const char *md_key, const char *md_value, int is_async);
 
-/* Private constructor for jwt credentials from an already parsed json key.
-   Takes ownership of the key. */
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime);
-
-/* Private constructor for refresh token credentials from an already parsed
-   refresh token. Takes ownership of the refresh token. */
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token token);
-
 /* --- grpc_server_credentials. --- */
 
 typedef struct {
@@ -260,118 +219,20 @@ grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg);
 grpc_server_credentials *grpc_find_server_credentials_in_args(
     const grpc_channel_args *args);
 
-/* -- Fake transport security credentials. -- */
-
-/* Creates a fake transport security credentials object for testing. */
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
-/* Creates a fake server transport security credentials object for testing. */
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void);
-
-/* -- Ssl credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_ssl_config config;
-} grpc_ssl_credentials;
-
-typedef struct {
-  grpc_server_credentials base;
-  grpc_ssl_server_config config;
-} grpc_ssl_server_credentials;
-
-/* -- Channel composite credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_channel_credentials *inner_creds;
-  grpc_call_credentials *call_creds;
-} grpc_composite_channel_credentials;
-
-/* -- Jwt credentials -- */
+/* -- Credentials Metadata Request. -- */
 
 typedef struct {
-  grpc_call_credentials base;
-
-  /* Have a simple cache for now with just 1 entry. We could have a map based on
-     the service_url for a more sophisticated one. */
-  gpr_mu cache_mu;
-  struct {
-    grpc_credentials_md_store *jwt_md;
-    char *service_url;
-    gpr_timespec jwt_expiration;
-  } cached;
-
-  grpc_auth_json_key key;
-  gpr_timespec jwt_lifetime;
-} grpc_service_account_jwt_access_credentials;
-
-/* -- Oauth2TokenFetcher credentials --
-
-   This object is a base for credentials that need to acquire an oauth2 token
-   from an http service. */
-
-typedef struct grpc_credentials_metadata_request
-    grpc_credentials_metadata_request;
+  grpc_call_credentials *creds;
+  grpc_credentials_metadata_cb cb;
+  grpc_http_response response;
+  void *user_data;
+} grpc_credentials_metadata_request;
 
-typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
-                                       grpc_credentials_metadata_request *req,
-                                       grpc_httpcli_context *http_context,
-                                       grpc_pollset *pollset,
-                                       grpc_httpcli_response_cb response_cb,
-                                       gpr_timespec deadline);
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data);
 
-typedef struct {
-  grpc_call_credentials base;
-  gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
-  gpr_timespec token_expiration;
-  grpc_httpcli_context httpcli_context;
-  grpc_fetch_oauth2_func fetch_func;
-} grpc_oauth2_token_fetcher_credentials;
-
-/* -- GoogleRefreshToken credentials. -- */
-
-typedef struct {
-  grpc_oauth2_token_fetcher_credentials base;
-  grpc_auth_refresh_token refresh_token;
-} grpc_google_refresh_token_credentials;
-
-/* -- Oauth2 Access Token credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
-} grpc_access_token_credentials;
-
-/* --  Metadata-only Test credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
-} grpc_md_only_test_credentials;
-
-/* -- GoogleIAM credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
-} grpc_google_iam_credentials;
-
-/* -- Composite credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_call_credentials_array inner;
-} grpc_composite_call_credentials;
-
-/* -- Plugin credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
-} grpc_plugin_credentials;
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r);
 
-#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
index bd00194278..6a352aab3a 100644
--- a/src/core/lib/security/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <grpc/support/alloc.h>
 
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
new file mode 100644
index 0000000000..51cafd986f
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -0,0 +1,139 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/executor.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Fake transport security credentials. -- */
+
+static grpc_security_status fake_transport_security_create_security_connector(
+    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  *sc = grpc_fake_channel_security_connector_create(call_creds);
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_security_status
+fake_transport_security_server_create_security_connector(
+    grpc_server_credentials *c, grpc_server_security_connector **sc) {
+  *sc = grpc_fake_server_security_connector_create();
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_channel_credentials_vtable
+    fake_transport_security_credentials_vtable = {
+        NULL, fake_transport_security_create_security_connector};
+
+static grpc_server_credentials_vtable
+    fake_transport_security_server_credentials_vtable = {
+        NULL, fake_transport_security_server_create_security_connector};
+
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
+    void) {
+  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
+  memset(c, 0, sizeof(grpc_channel_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  c->vtable = &fake_transport_security_credentials_vtable;
+  gpr_ref_init(&c->refcount, 1);
+  return c;
+}
+
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void) {
+  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
+  memset(c, 0, sizeof(grpc_server_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  gpr_ref_init(&c->refcount, 1);
+  c->vtable = &fake_transport_security_server_credentials_vtable;
+  return c;
+}
+
+/* -- Metadata-only test credentials. -- */
+
+static void md_only_test_destruct(grpc_call_credentials *creds) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+  grpc_credentials_md_store_unref(c->md_store);
+}
+
+static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
+                                          void *user_data, grpc_error *error) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
+  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
+        GRPC_CREDENTIALS_OK, NULL);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void md_only_test_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+
+  if (c->is_async) {
+    grpc_credentials_metadata_request *cb_arg =
+        grpc_credentials_metadata_request_create(creds, cb, user_data);
+    grpc_executor_push(
+        grpc_closure_create(on_simulated_token_fetch_done, cb_arg),
+        GRPC_ERROR_NONE);
+  } else {
+    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK, NULL);
+  }
+}
+
+static grpc_call_credentials_vtable md_only_test_vtable = {
+    md_only_test_destruct, md_only_test_get_request_metadata};
+
+grpc_call_credentials *grpc_md_only_test_credentials_create(
+    const char *md_key, const char *md_value, int is_async) {
+  grpc_md_only_test_credentials *c =
+      gpr_malloc(sizeof(grpc_md_only_test_credentials));
+  memset(c, 0, sizeof(grpc_md_only_test_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &md_only_test_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->md_store = grpc_credentials_md_store_create(1);
+  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->is_async = is_async;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
new file mode 100644
index 0000000000..9cf38084a3
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+/* -- Fake transport security credentials. -- */
+
+/* Creates a fake transport security credentials object for testing. */
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
+
+/* Creates a fake server transport security credentials object for testing. */
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void);
+
+/* --  Metadata-only Test credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *md_store;
+  int is_async;
+} grpc_md_only_test_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_posix.c b/src/core/lib/security/credentials/google_default/credentials_posix.c
index a07de182a0..42c9d7f997 100644
--- a/src/core/lib/security/credentials_posix.c
+++ b/src/core/lib/security/credentials/google_default/credentials_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/credentials_win32.c b/src/core/lib/security/credentials/google_default/credentials_windows.c
index d29847af38..208b8fd9ad 100644
--- a/src/core/lib/security/credentials_win32.c
+++ b/src/core/lib/security/credentials/google_default/credentials_windows.c
@@ -33,9 +33,9 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -58,4 +58,4 @@ char *grpc_get_well_known_google_credentials_file_path_impl(void) {
   return result;
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/security/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c
index 236f1d7fa7..312a3d4f90 100644
--- a/src/core/lib/security/google_default_credentials.c
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <string.h>
 
@@ -41,8 +41,12 @@
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/load_file.h"
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/support/env.h"
-#include "src/core/lib/support/load_file.h"
+#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 
 /* -- Constants. -- */
@@ -60,21 +64,23 @@ static gpr_once g_once = GPR_ONCE_INIT;
 static void init_default_credentials(void) { gpr_mu_init(&g_state_mu); }
 
 typedef struct {
-  grpc_pollset *pollset;
+  grpc_polling_entity pollent;
   int is_done;
   int success;
+  grpc_http_response response;
 } compute_engine_detector;
 
-static void on_compute_engine_detection_http_response(
-    grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_http_response *response) {
+static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx,
+                                                      void *user_data,
+                                                      grpc_error *error) {
   compute_engine_detector *detector = (compute_engine_detector *)user_data;
-  if (response != NULL && response->status == 200 && response->hdr_count > 0) {
+  if (error == GRPC_ERROR_NONE && detector->response.status == 200 &&
+      detector->response.hdr_count > 0) {
     /* Internet providers can return a generic response to all requests, so
        it is necessary to check that metadata header is present also. */
     size_t i;
-    for (i = 0; i < response->hdr_count; i++) {
-      grpc_http_header *header = &response->hdrs[i];
+    for (i = 0; i < detector->response.hdr_count; i++) {
+      grpc_http_header *header = &detector->response.hdrs[i];
       if (strcmp(header->key, "Metadata-Flavor") == 0 &&
           strcmp(header->value, "Google") == 0) {
         detector->success = 1;
@@ -84,11 +90,13 @@ static void on_compute_engine_detection_http_response(
   }
   gpr_mu_lock(g_polling_mu);
   detector->is_done = 1;
-  grpc_pollset_kick(detector->pollset, NULL);
+  GRPC_LOG_IF_ERROR(
+      "Pollset kick",
+      grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL));
   gpr_mu_unlock(g_polling_mu);
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool s) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *e) {
   grpc_pollset_destroy(p);
 }
 
@@ -103,11 +111,13 @@ static int is_stack_running_on_compute_engine(void) {
      on compute engine. */
   gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN);
 
-  detector.pollset = gpr_malloc(grpc_pollset_size());
-  grpc_pollset_init(detector.pollset, &g_polling_mu);
+  grpc_pollset *pollset = gpr_malloc(grpc_pollset_size());
+  grpc_pollset_init(pollset, &g_polling_mu);
+  detector.pollent = grpc_polling_entity_create_from_pollset(pollset);
   detector.is_done = 0;
   detector.success = 0;
 
+  memset(&detector.response, 0, sizeof(detector.response));
   memset(&request, 0, sizeof(grpc_httpcli_request));
   request.host = GRPC_COMPUTE_ENGINE_DETECTION_HOST;
   request.http.path = "/";
@@ -115,48 +125,71 @@ static int is_stack_running_on_compute_engine(void) {
   grpc_httpcli_context_init(&context);
 
   grpc_httpcli_get(
-      &exec_ctx, &context, detector.pollset, &request,
+      &exec_ctx, &context, &detector.pollent, &request,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay),
-      on_compute_engine_detection_http_response, &detector);
+      grpc_closure_create(on_compute_engine_detection_http_response, &detector),
+      &detector.response);
 
-  grpc_exec_ctx_finish(&exec_ctx);
+  grpc_exec_ctx_flush(&exec_ctx);
 
   /* Block until we get the response. This is not ideal but this should only be
      called once for the lifetime of the process by the default credentials. */
   gpr_mu_lock(g_polling_mu);
   while (!detector.is_done) {
     grpc_pollset_worker *worker = NULL;
-    grpc_pollset_work(&exec_ctx, detector.pollset, &worker,
-                      gpr_now(GPR_CLOCK_MONOTONIC),
-                      gpr_inf_future(GPR_CLOCK_MONOTONIC));
+    if (!GRPC_LOG_IF_ERROR(
+            "pollset_work",
+            grpc_pollset_work(&exec_ctx,
+                              grpc_polling_entity_pollset(&detector.pollent),
+                              &worker, gpr_now(GPR_CLOCK_MONOTONIC),
+                              gpr_inf_future(GPR_CLOCK_MONOTONIC)))) {
+      detector.is_done = 1;
+      detector.success = 0;
+    }
   }
   gpr_mu_unlock(g_polling_mu);
 
   grpc_httpcli_context_destroy(&context);
-  grpc_closure_init(&destroy_closure, destroy_pollset, detector.pollset);
-  grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure);
+  grpc_closure_init(&destroy_closure, destroy_pollset,
+                    grpc_polling_entity_pollset(&detector.pollent));
+  grpc_pollset_shutdown(&exec_ctx,
+                        grpc_polling_entity_pollset(&detector.pollent),
+                        &destroy_closure);
   grpc_exec_ctx_finish(&exec_ctx);
   g_polling_mu = NULL;
 
-  gpr_free(detector.pollset);
+  gpr_free(grpc_polling_entity_pollset(&detector.pollent));
+  grpc_http_response_destroy(&detector.response);
 
   return detector.success;
 }
 
 /* Takes ownership of creds_path if not NULL. */
-static grpc_call_credentials *create_default_creds_from_path(char *creds_path) {
+static grpc_error *create_default_creds_from_path(
+    char *creds_path, grpc_call_credentials **creds) {
   grpc_json *json = NULL;
   grpc_auth_json_key key;
   grpc_auth_refresh_token token;
   grpc_call_credentials *result = NULL;
   gpr_slice creds_data = gpr_empty_slice();
-  int file_ok = 0;
-  if (creds_path == NULL) goto end;
-  creds_data = gpr_load_file(creds_path, 0, &file_ok);
-  if (!file_ok) goto end;
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (creds_path == NULL) {
+    error = GRPC_ERROR_CREATE("creds_path unset");
+    goto end;
+  }
+  error = grpc_load_file(creds_path, 0, &creds_data);
+  if (error != GRPC_ERROR_NONE) {
+    goto end;
+  }
   json = grpc_json_parse_string_with_len(
       (char *)GPR_SLICE_START_PTR(creds_data), GPR_SLICE_LENGTH(creds_data));
-  if (json == NULL) goto end;
+  if (json == NULL) {
+    char *dump = gpr_dump_slice(creds_data, GPR_DUMP_HEX | GPR_DUMP_ASCII);
+    error = grpc_error_set_str(GRPC_ERROR_CREATE("Failed to parse JSON"),
+                               GRPC_ERROR_STR_RAW_BYTES, dump);
+    gpr_free(dump);
+    goto end;
+  }
 
   /* First, try an auth json key. */
   key = grpc_auth_json_key_create_from_json(json);
@@ -164,6 +197,11 @@ static grpc_call_credentials *create_default_creds_from_path(char *creds_path) {
     result =
         grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
             key, grpc_max_auth_token_lifetime());
+    if (result == NULL) {
+      error = GRPC_ERROR_CREATE(
+          "grpc_service_account_jwt_access_credentials_create_from_auth_json_"
+          "key failed");
+    }
     goto end;
   }
 
@@ -172,19 +210,28 @@ static grpc_call_credentials *create_default_creds_from_path(char *creds_path) {
   if (grpc_auth_refresh_token_is_valid(&token)) {
     result =
         grpc_refresh_token_credentials_create_from_auth_refresh_token(token);
+    if (result == NULL) {
+      error = GRPC_ERROR_CREATE(
+          "grpc_refresh_token_credentials_create_from_auth_refresh_token "
+          "failed");
+    }
     goto end;
   }
 
 end:
+  GPR_ASSERT((result == NULL) + (error == GRPC_ERROR_NONE) == 1);
   if (creds_path != NULL) gpr_free(creds_path);
   gpr_slice_unref(creds_data);
   if (json != NULL) grpc_json_destroy(json);
-  return result;
+  *creds = result;
+  return error;
 }
 
 grpc_channel_credentials *grpc_google_default_credentials_create(void) {
   grpc_channel_credentials *result = NULL;
   grpc_call_credentials *call_creds = NULL;
+  grpc_error *error = GRPC_ERROR_CREATE("Failed to create Google credentials");
+  grpc_error *err;
 
   GRPC_API_TRACE("grpc_google_default_credentials_create(void)", 0, ());
 
@@ -198,14 +245,16 @@ grpc_channel_credentials *grpc_google_default_credentials_create(void) {
   }
 
   /* First, try the environment variable. */
-  call_creds = create_default_creds_from_path(
-      gpr_getenv(GRPC_GOOGLE_CREDENTIALS_ENV_VAR));
-  if (call_creds != NULL) goto end;
+  err = create_default_creds_from_path(
+      gpr_getenv(GRPC_GOOGLE_CREDENTIALS_ENV_VAR), &call_creds);
+  if (err == GRPC_ERROR_NONE) goto end;
+  error = grpc_error_add_child(error, err);
 
   /* Then the well-known file. */
-  call_creds = create_default_creds_from_path(
-      grpc_get_well_known_google_credentials_file_path());
-  if (call_creds != NULL) goto end;
+  err = create_default_creds_from_path(
+      grpc_get_well_known_google_credentials_file_path(), &call_creds);
+  if (err == GRPC_ERROR_NONE) goto end;
+  error = grpc_error_add_child(error, err);
 
   /* At last try to see if we're on compute engine (do the detection only once
      since it requires a network test). */
@@ -214,6 +263,10 @@ grpc_channel_credentials *grpc_google_default_credentials_create(void) {
     compute_engine_detection_done = 1;
     if (need_compute_engine_creds) {
       call_creds = grpc_google_compute_engine_credentials_create(NULL);
+      if (call_creds == NULL) {
+        error = grpc_error_add_child(
+            error, GRPC_ERROR_CREATE("Failed to get credentials from network"));
+      }
     }
   }
 
@@ -237,6 +290,11 @@ end:
     }
   }
   gpr_mu_unlock(&g_state_mu);
+  if (result == NULL) {
+    GRPC_LOG_IF_ERROR("grpc_google_default_credentials_create", error);
+  } else {
+    GRPC_ERROR_UNREF(error);
+  }
   return result;
 }
 
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.h b/src/core/lib/security/credentials/google_default/google_default_credentials.h
new file mode 100644
index 0000000000..fac4377e2c
--- /dev/null
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
+#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
+  "application_default_credentials.json"
+
+void grpc_flush_cached_google_default_credentials(void);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
new file mode 100644
index 0000000000..370a384d0e
--- /dev/null
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/iam/iam_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void iam_destruct(grpc_call_credentials *creds) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  grpc_credentials_md_store_unref(c->iam_md);
+}
+
+static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_polling_entity *pollent,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
+     GRPC_CREDENTIALS_OK, NULL);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
+                                                  iam_get_request_metadata};
+
+grpc_call_credentials *grpc_google_iam_credentials_create(
+    const char *token, const char *authority_selector, void *reserved) {
+  grpc_google_iam_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
+      "reserved=%p)",
+      3, (token, authority_selector, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(token != NULL);
+  GPR_ASSERT(authority_selector != NULL);
+  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
+  memset(c, 0, sizeof(grpc_google_iam_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
+  c->base.vtable = &iam_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->iam_md = grpc_credentials_md_store_create(2);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h
new file mode 100644
index 0000000000..af54faa586
--- /dev/null
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -0,0 +1,44 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *iam_md;
+} grpc_google_iam_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/json_token.c b/src/core/lib/security/credentials/jwt/json_token.c
index d5bc2c8d60..354c13133e 100644
--- a/src/core/lib/security/json_token.c
+++ b/src/core/lib/security/credentials/jwt/json_token.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
 
 #include <string.h>
 
@@ -39,7 +39,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
+#include "src/core/lib/security/util/json_util.h"
 #include "src/core/lib/support/string.h"
 
 #include <openssl/bio.h>
@@ -66,28 +67,6 @@ static grpc_jwt_encode_and_sign_override g_jwt_encode_and_sign_override = NULL;
 
 /* --- grpc_auth_json_key. --- */
 
-static const char *json_get_string_property(const grpc_json *json,
-                                            const char *prop_name) {
-  grpc_json *child;
-  for (child = json->child; child != NULL; child = child->next) {
-    if (strcmp(child->key, prop_name) == 0) break;
-  }
-  if (child == NULL || child->type != GRPC_JSON_STRING) {
-    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
-    return NULL;
-  }
-  return child->value;
-}
-
-static int set_json_key_string_property(const grpc_json *json,
-                                        const char *prop_name,
-                                        char **json_key_field) {
-  const char *prop_value = json_get_string_property(json, prop_name);
-  if (prop_value == NULL) return 0;
-  *json_key_field = gpr_strdup(prop_value);
-  return 1;
-}
-
 int grpc_auth_json_key_is_valid(const grpc_auth_json_key *json_key) {
   return (json_key != NULL) &&
          strcmp(json_key->type, GRPC_AUTH_JSON_TYPE_INVALID);
@@ -106,22 +85,22 @@ grpc_auth_json_key grpc_auth_json_key_create_from_json(const grpc_json *json) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "type");
+  prop_value = grpc_json_get_string_property(json, "type");
   if (prop_value == NULL ||
       strcmp(prop_value, GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT)) {
     goto end;
   }
   result.type = GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT;
 
-  if (!set_json_key_string_property(json, "private_key_id",
-                                    &result.private_key_id) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "client_email",
-                                    &result.client_email)) {
+  if (!grpc_copy_json_string_property(json, "private_key_id",
+                                      &result.private_key_id) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "client_email",
+                                      &result.client_email)) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "private_key");
+  prop_value = grpc_json_get_string_property(json, "private_key");
   if (prop_value == NULL) {
     goto end;
   }
@@ -339,73 +318,3 @@ void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func) {
   g_jwt_encode_and_sign_override = func;
 }
-
-/* --- grpc_auth_refresh_token --- */
-
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token) {
-  return (refresh_token != NULL) &&
-         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json) {
-  grpc_auth_refresh_token result;
-  const char *prop_value;
-  int success = 0;
-
-  memset(&result, 0, sizeof(grpc_auth_refresh_token));
-  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (json == NULL) {
-    gpr_log(GPR_ERROR, "Invalid json.");
-    goto end;
-  }
-
-  prop_value = json_get_string_property(json, "type");
-  if (prop_value == NULL ||
-      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
-    goto end;
-  }
-  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
-
-  if (!set_json_key_string_property(json, "client_secret",
-                                    &result.client_secret) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "refresh_token",
-                                    &result.refresh_token)) {
-    goto end;
-  }
-  success = 1;
-
-end:
-  if (!success) grpc_auth_refresh_token_destruct(&result);
-  return result;
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string) {
-  char *scratchpad = gpr_strdup(json_string);
-  grpc_json *json = grpc_json_parse_string(scratchpad);
-  grpc_auth_refresh_token result =
-      grpc_auth_refresh_token_create_from_json(json);
-  if (json != NULL) grpc_json_destroy(json);
-  gpr_free(scratchpad);
-  return result;
-}
-
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
-  if (refresh_token == NULL) return;
-  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (refresh_token->client_id != NULL) {
-    gpr_free(refresh_token->client_id);
-    refresh_token->client_id = NULL;
-  }
-  if (refresh_token->client_secret != NULL) {
-    gpr_free(refresh_token->client_secret);
-    refresh_token->client_secret = NULL;
-  }
-  if (refresh_token->refresh_token != NULL) {
-    gpr_free(refresh_token->refresh_token);
-    refresh_token->refresh_token = NULL;
-  }
-}
diff --git a/src/core/lib/security/json_token.h b/src/core/lib/security/credentials/jwt/json_token.h
index 123fa652fd..07fc5bf0e0 100644
--- a/src/core/lib/security/json_token.h
+++ b/src/core/lib/security/credentials/jwt/json_token.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
-#define GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
 
 #include <grpc/support/slice.h>
 #include <openssl/rsa.h>
@@ -43,10 +43,6 @@
 
 #define GRPC_JWT_OAUTH2_AUDIENCE "https://www.googleapis.com/oauth2/v3/token"
 
-#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
-#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
-#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
-
 /* --- auth_json_key parsing. --- */
 
 typedef struct {
@@ -89,30 +85,4 @@ typedef char *(*grpc_jwt_encode_and_sign_override)(
 void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func);
 
-/* --- auth_refresh_token parsing. --- */
-
-typedef struct {
-  const char *type;
-  char *client_id;
-  char *client_secret;
-  char *refresh_token;
-} grpc_auth_refresh_token;
-
-/* Returns 1 if the object is valid, 0 otherwise. */
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token);
-
-/* Creates a refresh token object from string. Returns an invalid object if a
-   parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string);
-
-/* Creates a refresh token object from parsed json. Returns an invalid object if
-   a parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json);
-
-/* Destructs the object. */
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
-
-#endif /* GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
new file mode 100644
index 0000000000..f87ba0ce8d
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -0,0 +1,161 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
+  if (c->cached.jwt_md != NULL) {
+    grpc_credentials_md_store_unref(c->cached.jwt_md);
+    c->cached.jwt_md = NULL;
+  }
+  if (c->cached.service_url != NULL) {
+    gpr_free(c->cached.service_url);
+    c->cached.service_url = NULL;
+  }
+  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+}
+
+static void jwt_destruct(grpc_call_credentials *creds) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  grpc_auth_json_key_destruct(&c->key);
+  jwt_reset_cache(c);
+  gpr_mu_destroy(&c->cache_mu);
+}
+
+static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_polling_entity *pollent,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+
+  /* See if we can return a cached jwt. */
+  grpc_credentials_md_store *jwt_md = NULL;
+  {
+    gpr_mu_lock(&c->cache_mu);
+    if (c->cached.service_url != NULL &&
+        strcmp(c->cached.service_url, context.service_url) == 0 &&
+        c->cached.jwt_md != NULL &&
+        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
+                                   gpr_now(GPR_CLOCK_REALTIME)),
+                      refresh_threshold) > 0)) {
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md == NULL) {
+    char *jwt = NULL;
+    /* Generate a new jwt. */
+    gpr_mu_lock(&c->cache_mu);
+    jwt_reset_cache(c);
+    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
+                                   c->jwt_lifetime, NULL);
+    if (jwt != NULL) {
+      char *md_value;
+      gpr_asprintf(&md_value, "Bearer %s", jwt);
+      gpr_free(jwt);
+      c->cached.jwt_expiration =
+          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
+      c->cached.service_url = gpr_strdup(context.service_url);
+      c->cached.jwt_md = grpc_credentials_md_store_create(1);
+      grpc_credentials_md_store_add_cstrings(
+          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      gpr_free(md_value);
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md != NULL) {
+    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
+       GRPC_CREDENTIALS_OK, NULL);
+    grpc_credentials_md_store_unref(jwt_md);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
+       "Could not generate JWT.");
+  }
+}
+
+static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
+                                                  jwt_get_request_metadata};
+
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime) {
+  grpc_service_account_jwt_access_credentials *c;
+  if (!grpc_auth_json_key_is_valid(&key)) {
+    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
+  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &jwt_vtable;
+  c->key = key;
+  c->jwt_lifetime = token_lifetime;
+  gpr_mu_init(&c->cache_mu);
+  jwt_reset_cache(c);
+  return &c->base;
+}
+
+grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
+    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_service_account_jwt_access_credentials_create("
+      "json_key=%s, "
+      "token_lifetime="
+      "gpr_timespec { tv_sec: %" PRId64
+      ", tv_nsec: %d, clock_type: %d }, "
+      "reserved=%p)",
+      5, (json_key, token_lifetime.tv_sec, token_lifetime.tv_nsec,
+          (int)token_lifetime.clock_type, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
+}
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
new file mode 100644
index 0000000000..d572606179
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
+
+typedef struct {
+  grpc_call_credentials base;
+
+  // Have a simple cache for now with just 1 entry. We could have a map based on
+  // the service_url for a more sophisticated one.
+  gpr_mu cache_mu;
+  struct {
+    grpc_credentials_md_store *jwt_md;
+    char *service_url;
+    gpr_timespec jwt_expiration;
+  } cached;
+
+  grpc_auth_json_key key;
+  gpr_timespec jwt_lifetime;
+} grpc_service_account_jwt_access_credentials;
+
+// Private constructor for jwt credentials from an already parsed json key.
+// Takes ownership of the key.
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H */
diff --git a/src/core/lib/security/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
index 0e012294de..73eb2e3258 100644
--- a/src/core/lib/security/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -31,19 +31,21 @@
  *
  */
 
-#include "src/core/lib/security/jwt_verifier.h"
+#include "src/core/lib/security/credentials/jwt/jwt_verifier.h"
 
 #include <limits.h>
 #include <string.h>
 
 #include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/security/util/b64.h"
 #include "src/core/lib/tsi/ssl_types.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
+#include <grpc/support/useful.h>
 #include <openssl/pem.h>
 
 /* --- Utils. --- */
@@ -319,9 +321,15 @@ grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
 
 /* --- verifier_cb_ctx object. --- */
 
+typedef enum {
+  HTTP_RESPONSE_OPENID = 0,
+  HTTP_RESPONSE_KEYS,
+  HTTP_RESPONSE_COUNT /* must be last */
+} http_response_index;
+
 typedef struct {
   grpc_jwt_verifier *verifier;
-  grpc_pollset *pollset;
+  grpc_polling_entity pollent;
   jose_header *header;
   grpc_jwt_claims *claims;
   char *audience;
@@ -329,6 +337,7 @@ typedef struct {
   gpr_slice signed_data;
   void *user_data;
   grpc_jwt_verification_done_cb user_cb;
+  grpc_http_response responses[HTTP_RESPONSE_COUNT];
 } verifier_cb_ctx;
 
 /* Takes ownership of the header, claims and signature. */
@@ -337,10 +346,11 @@ static verifier_cb_ctx *verifier_cb_ctx_create(
     grpc_jwt_claims *claims, const char *audience, gpr_slice signature,
     const char *signed_jwt, size_t signed_jwt_len, void *user_data,
     grpc_jwt_verification_done_cb cb) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   verifier_cb_ctx *ctx = gpr_malloc(sizeof(verifier_cb_ctx));
   memset(ctx, 0, sizeof(verifier_cb_ctx));
   ctx->verifier = verifier;
-  ctx->pollset = pollset;
+  ctx->pollent = grpc_polling_entity_create_from_pollset(pollset);
   ctx->header = header;
   ctx->audience = gpr_strdup(audience);
   ctx->claims = claims;
@@ -348,6 +358,7 @@ static verifier_cb_ctx *verifier_cb_ctx_create(
   ctx->signed_data = gpr_slice_from_copied_buffer(signed_jwt, signed_jwt_len);
   ctx->user_data = user_data;
   ctx->user_cb = cb;
+  grpc_exec_ctx_finish(&exec_ctx);
   return ctx;
 }
 
@@ -357,6 +368,9 @@ void verifier_cb_ctx_destroy(verifier_cb_ctx *ctx) {
   gpr_slice_unref(ctx->signature);
   gpr_slice_unref(ctx->signed_data);
   jose_header_destroy(ctx->header);
+  for (size_t i = 0; i < HTTP_RESPONSE_COUNT; i++) {
+    grpc_http_response_destroy(&ctx->responses[i]);
+  }
   /* TODO: see what to do with claims... */
   gpr_free(ctx);
 }
@@ -571,9 +585,9 @@ end:
 }
 
 static void on_keys_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
-                              const grpc_httpcli_response *response) {
-  grpc_json *json = json_from_http(response);
+                              grpc_error *error) {
   verifier_cb_ctx *ctx = (verifier_cb_ctx *)user_data;
+  grpc_json *json = json_from_http(&ctx->responses[HTTP_RESPONSE_KEYS]);
   EVP_PKEY *verification_key = NULL;
   grpc_jwt_verifier_status status = GRPC_JWT_VERIFIER_GENERIC_ERROR;
   grpc_jwt_claims *claims = NULL;
@@ -612,10 +626,11 @@ end:
 }
 
 static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       const grpc_httpcli_response *response) {
+                                       grpc_error *error) {
   const grpc_json *cur;
-  grpc_json *json = json_from_http(response);
   verifier_cb_ctx *ctx = (verifier_cb_ctx *)user_data;
+  const grpc_http_response *response = &ctx->responses[HTTP_RESPONSE_OPENID];
+  grpc_json *json = json_from_http(response);
   grpc_httpcli_request req;
   const char *jwks_uri;
 
@@ -641,10 +656,12 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
   } else {
     *(req.host + (req.http.path - jwks_uri)) = '\0';
   }
+
   grpc_httpcli_get(
-      exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req,
+      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
-      on_keys_retrieved, ctx);
+      grpc_closure_create(on_keys_retrieved, ctx),
+      &ctx->responses[HTTP_RESPONSE_KEYS]);
   grpc_json_destroy(json);
   gpr_free(req.host);
   return;
@@ -686,12 +703,13 @@ static void verifier_put_mapping(grpc_jwt_verifier *v, const char *email_domain,
 static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
                                     verifier_cb_ctx *ctx) {
   const char *at_sign;
-  grpc_httpcli_response_cb http_cb;
+  grpc_closure *http_cb;
   char *path_prefix = NULL;
   const char *iss;
   grpc_httpcli_request req;
   memset(&req, 0, sizeof(grpc_httpcli_request));
   req.handshaker = &grpc_httpcli_ssl;
+  http_response_index rsp_idx;
 
   GPR_ASSERT(ctx != NULL && ctx->header != NULL && ctx->claims != NULL);
   iss = ctx->claims->iss;
@@ -730,7 +748,8 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
       *(path_prefix++) = '\0';
       gpr_asprintf(&req.http.path, "/%s/%s", path_prefix, iss);
     }
-    http_cb = on_keys_retrieved;
+    http_cb = grpc_closure_create(on_keys_retrieved, ctx);
+    rsp_idx = HTTP_RESPONSE_KEYS;
   } else {
     req.host = gpr_strdup(strstr(iss, "https://") == iss ? iss + 8 : iss);
     path_prefix = strchr(req.host, '/');
@@ -741,13 +760,14 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
       gpr_asprintf(&req.http.path, "/%s%s", path_prefix,
                    GRPC_OPENID_CONFIG_URL_SUFFIX);
     }
-    http_cb = on_openid_config_retrieved;
+    http_cb = grpc_closure_create(on_openid_config_retrieved, ctx);
+    rsp_idx = HTTP_RESPONSE_OPENID;
   }
 
   grpc_httpcli_get(
-      exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req,
+      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
-      http_cb, ctx);
+      http_cb, &ctx->responses[rsp_idx]);
   gpr_free(req.host);
   gpr_free(req.http.path);
   return;
diff --git a/src/core/lib/security/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h
index 98a4f6b116..b0f6d1c240 100644
--- a/src/core/lib/security/jwt_verifier.h
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
-#define GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
 
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/json/json.h"
@@ -133,4 +133,4 @@ grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer);
 grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
                                                const char *audience);
 
-#endif /* GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H */
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
new file mode 100644
index 0000000000..c22ea5c468
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -0,0 +1,435 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/security/util/json_util.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+//
+// Auth Refresh Token.
+//
+
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token) {
+  return (refresh_token != NULL) &&
+         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json) {
+  grpc_auth_refresh_token result;
+  const char *prop_value;
+  int success = 0;
+
+  memset(&result, 0, sizeof(grpc_auth_refresh_token));
+  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (json == NULL) {
+    gpr_log(GPR_ERROR, "Invalid json.");
+    goto end;
+  }
+
+  prop_value = grpc_json_get_string_property(json, "type");
+  if (prop_value == NULL ||
+      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
+    goto end;
+  }
+  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
+
+  if (!grpc_copy_json_string_property(json, "client_secret",
+                                      &result.client_secret) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "refresh_token",
+                                      &result.refresh_token)) {
+    goto end;
+  }
+  success = 1;
+
+end:
+  if (!success) grpc_auth_refresh_token_destruct(&result);
+  return result;
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string) {
+  char *scratchpad = gpr_strdup(json_string);
+  grpc_json *json = grpc_json_parse_string(scratchpad);
+  grpc_auth_refresh_token result =
+      grpc_auth_refresh_token_create_from_json(json);
+  if (json != NULL) grpc_json_destroy(json);
+  gpr_free(scratchpad);
+  return result;
+}
+
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
+  if (refresh_token == NULL) return;
+  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (refresh_token->client_id != NULL) {
+    gpr_free(refresh_token->client_id);
+    refresh_token->client_id = NULL;
+  }
+  if (refresh_token->client_secret != NULL) {
+    gpr_free(refresh_token->client_secret);
+    refresh_token->client_secret = NULL;
+  }
+  if (refresh_token->refresh_token != NULL) {
+    gpr_free(refresh_token->refresh_token);
+    refresh_token->refresh_token = NULL;
+  }
+}
+
+//
+// Oauth2 Token Fetcher credentials.
+//
+
+static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+  gpr_mu_destroy(&c->mu);
+  grpc_httpcli_context_destroy(&c->httpcli_context);
+}
+
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const grpc_http_response *response, grpc_credentials_md_store **token_md,
+    gpr_timespec *token_lifetime) {
+  char *null_terminated_body = NULL;
+  char *new_access_token = NULL;
+  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
+  grpc_json *json = NULL;
+
+  if (response == NULL) {
+    gpr_log(GPR_ERROR, "Received NULL response.");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  }
+
+  if (response->body_length > 0) {
+    null_terminated_body = gpr_malloc(response->body_length + 1);
+    null_terminated_body[response->body_length] = '\0';
+    memcpy(null_terminated_body, response->body, response->body_length);
+  }
+
+  if (response->status != 200) {
+    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
+            response->status,
+            null_terminated_body != NULL ? null_terminated_body : "");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  } else {
+    grpc_json *access_token = NULL;
+    grpc_json *token_type = NULL;
+    grpc_json *expires_in = NULL;
+    grpc_json *ptr;
+    json = grpc_json_parse_string(null_terminated_body);
+    if (json == NULL) {
+      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (json->type != GRPC_JSON_OBJECT) {
+      gpr_log(GPR_ERROR, "Response should be a JSON object");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    for (ptr = json->child; ptr; ptr = ptr->next) {
+      if (strcmp(ptr->key, "access_token") == 0) {
+        access_token = ptr;
+      } else if (strcmp(ptr->key, "token_type") == 0) {
+        token_type = ptr;
+      } else if (strcmp(ptr->key, "expires_in") == 0) {
+        expires_in = ptr;
+      }
+    }
+    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
+      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
+                 access_token->value);
+    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
+    token_lifetime->tv_nsec = 0;
+    token_lifetime->clock_type = GPR_TIMESPAN;
+    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
+    *token_md = grpc_credentials_md_store_create(1);
+    grpc_credentials_md_store_add_cstrings(
+        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    status = GRPC_CREDENTIALS_OK;
+  }
+
+end:
+  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
+    grpc_credentials_md_store_unref(*token_md);
+    *token_md = NULL;
+  }
+  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
+  if (new_access_token != NULL) gpr_free(new_access_token);
+  if (json != NULL) grpc_json_destroy(json);
+  return status;
+}
+
+static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx,
+                                                  void *user_data,
+                                                  grpc_error *error) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  gpr_timespec token_lifetime;
+  grpc_credentials_status status;
+
+  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
+
+  gpr_mu_lock(&c->mu);
+  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
+      &r->response, &c->access_token_md, &token_lifetime);
+  if (status == GRPC_CREDENTIALS_OK) {
+    c->token_expiration =
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
+    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
+          c->access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
+  } else {
+    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+    r->cb(exec_ctx, r->user_data, NULL, 0, status,
+          "Error occured when fetching oauth2 token.");
+  }
+  gpr_mu_unlock(&c->mu);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void oauth2_token_fetcher_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+  grpc_credentials_md_store *cached_access_token_md = NULL;
+  {
+    gpr_mu_lock(&c->mu);
+    if (c->access_token_md != NULL &&
+        (gpr_time_cmp(
+             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+             refresh_threshold) > 0)) {
+      cached_access_token_md =
+          grpc_credentials_md_store_ref(c->access_token_md);
+    }
+    gpr_mu_unlock(&c->mu);
+  }
+  if (cached_access_token_md != NULL) {
+    cb(exec_ctx, user_data, cached_access_token_md->entries,
+       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
+    grpc_credentials_md_store_unref(cached_access_token_md);
+  } else {
+    c->fetch_func(
+        exec_ctx,
+        grpc_credentials_metadata_request_create(creds, cb, user_data),
+        &c->httpcli_context, pollent, on_oauth2_token_fetcher_http_response,
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+  }
+}
+
+static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
+                                      grpc_fetch_oauth2_func fetch_func) {
+  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  gpr_ref_init(&c->base.refcount, 1);
+  gpr_mu_init(&c->mu);
+  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+  c->fetch_func = fetch_func;
+  grpc_httpcli_context_init(&c->httpcli_context);
+}
+
+//
+//  Google Compute Engine credentials.
+//
+
+static grpc_call_credentials_vtable compute_engine_vtable = {
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void compute_engine_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_polling_entity *pollent,
+    grpc_iomgr_cb_func response_cb, gpr_timespec deadline) {
+  grpc_http_header header = {"Metadata-Flavor", "Google"};
+  grpc_httpcli_request request;
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
+  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  grpc_httpcli_get(exec_ctx, httpcli_context, pollent, &request, deadline,
+                   grpc_closure_create(response_cb, metadata_req),
+                   &metadata_req->response);
+}
+
+grpc_call_credentials *grpc_google_compute_engine_credentials_create(
+    void *reserved) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
+  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
+  c->base.vtable = &compute_engine_vtable;
+  return &c->base;
+}
+
+//
+// Google Refresh Token credentials.
+//
+
+static void refresh_token_destruct(grpc_call_credentials *creds) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)creds;
+  grpc_auth_refresh_token_destruct(&c->refresh_token);
+  oauth2_token_fetcher_destruct(&c->base.base);
+}
+
+static grpc_call_credentials_vtable refresh_token_vtable = {
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void refresh_token_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_polling_entity *pollent,
+    grpc_iomgr_cb_func response_cb, gpr_timespec deadline) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)metadata_req->creds;
+  grpc_http_header header = {"Content-Type",
+                             "application/x-www-form-urlencoded"};
+  grpc_httpcli_request request;
+  char *body = NULL;
+  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
+               c->refresh_token.client_id, c->refresh_token.client_secret,
+               c->refresh_token.refresh_token);
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
+  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  request.handshaker = &grpc_httpcli_ssl;
+  grpc_httpcli_post(exec_ctx, httpcli_context, pollent, &request, body,
+                    strlen(body), deadline,
+                    grpc_closure_create(response_cb, metadata_req),
+                    &metadata_req->response);
+  gpr_free(body);
+}
+
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token refresh_token) {
+  grpc_google_refresh_token_credentials *c;
+  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
+    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
+  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
+  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
+  c->base.base.vtable = &refresh_token_vtable;
+  c->refresh_token = refresh_token;
+  return &c->base.base;
+}
+
+grpc_call_credentials *grpc_google_refresh_token_credentials_create(
+    const char *json_refresh_token, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
+      "reserved=%p)",
+      2, (json_refresh_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
+      grpc_auth_refresh_token_create_from_string(json_refresh_token));
+}
+
+//
+// Oauth2 Access Token credentials.
+//
+
+static void access_token_destruct(grpc_call_credentials *creds) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+}
+
+static void access_token_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_polling_entity *pollent, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK,
+     NULL);
+}
+
+static grpc_call_credentials_vtable access_token_vtable = {
+    access_token_destruct, access_token_get_request_metadata};
+
+grpc_call_credentials *grpc_access_token_credentials_create(
+    const char *access_token, void *reserved) {
+  grpc_access_token_credentials *c =
+      gpr_malloc(sizeof(grpc_access_token_credentials));
+  char *token_md_value;
+  GRPC_API_TRACE(
+      "grpc_access_token_credentials_create(access_token=%s, "
+      "reserved=%p)",
+      2, (access_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_access_token_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &access_token_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->access_token_md = grpc_credentials_md_store_create(1);
+  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
+  grpc_credentials_md_store_add_cstrings(
+      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  gpr_free(token_md_value);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
new file mode 100644
index 0000000000..7f6f205c22
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/security/credentials/credentials.h"
+
+// auth_refresh_token parsing.
+typedef struct {
+  const char *type;
+  char *client_id;
+  char *client_secret;
+  char *refresh_token;
+} grpc_auth_refresh_token;
+
+/// Returns 1 if the object is valid, 0 otherwise.
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token);
+
+/// Creates a refresh token object from string. Returns an invalid object if a
+/// parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string);
+
+/// Creates a refresh token object from parsed json. Returns an invalid object
+/// if a parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json);
+
+/// Destructs the object.
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
+
+// -- Oauth2 Token Fetcher credentials --
+//
+//  This object is a base for credentials that need to acquire an oauth2 token
+//  from an http service.
+
+typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
+                                       grpc_credentials_metadata_request *req,
+                                       grpc_httpcli_context *http_context,
+                                       grpc_polling_entity *pollent,
+                                       grpc_iomgr_cb_func cb,
+                                       gpr_timespec deadline);
+typedef struct {
+  grpc_call_credentials base;
+  gpr_mu mu;
+  grpc_credentials_md_store *access_token_md;
+  gpr_timespec token_expiration;
+  grpc_httpcli_context httpcli_context;
+  grpc_fetch_oauth2_func fetch_func;
+} grpc_oauth2_token_fetcher_credentials;
+
+// Google refresh token credentials.
+typedef struct {
+  grpc_oauth2_token_fetcher_credentials base;
+  grpc_auth_refresh_token refresh_token;
+} grpc_google_refresh_token_credentials;
+
+// Access token credentials.
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *access_token_md;
+} grpc_access_token_credentials;
+
+// Private constructor for refresh token credentials from an already parsed
+// refresh token. Takes ownership of the refresh token.
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token token);
+
+// Exposed for testing only.
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const struct grpc_http_response *response,
+    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
new file mode 100644
index 0000000000..824ff081dc
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -0,0 +1,130 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/plugin/plugin_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+typedef struct {
+  void *user_data;
+  grpc_credentials_metadata_cb cb;
+} grpc_metadata_plugin_request;
+
+static void plugin_destruct(grpc_call_credentials *creds) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
+    c->plugin.destroy(c->plugin.state);
+  }
+}
+
+static void plugin_md_request_metadata_ready(void *request,
+                                             const grpc_metadata *md,
+                                             size_t num_md,
+                                             grpc_status_code status,
+                                             const char *error_details) {
+  /* called from application code */
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
+  if (status != GRPC_STATUS_OK) {
+    if (error_details != NULL) {
+      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
+              error_details);
+    }
+    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
+          error_details);
+  } else {
+    size_t i;
+    grpc_credentials_md *md_array = NULL;
+    if (num_md > 0) {
+      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
+      for (i = 0; i < num_md; i++) {
+        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
+        md_array[i].value =
+            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
+      }
+    }
+    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK, NULL);
+    if (md_array != NULL) {
+      for (i = 0; i < num_md; i++) {
+        gpr_slice_unref(md_array[i].key);
+        gpr_slice_unref(md_array[i].value);
+      }
+      gpr_free(md_array);
+    }
+  }
+  gpr_free(r);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_credentials *creds,
+                                        grpc_polling_entity *pollent,
+                                        grpc_auth_metadata_context context,
+                                        grpc_credentials_metadata_cb cb,
+                                        void *user_data) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.get_metadata != NULL) {
+    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
+    memset(request, 0, sizeof(*request));
+    request->user_data = user_data;
+    request->cb = cb;
+    c->plugin.get_metadata(c->plugin.state, context,
+                           plugin_md_request_metadata_ready, request);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
+  }
+}
+
+static grpc_call_credentials_vtable plugin_vtable = {
+    plugin_destruct, plugin_get_request_metadata};
+
+grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
+    grpc_metadata_credentials_plugin plugin, void *reserved) {
+  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
+  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(*c));
+  c->base.type = plugin.type;
+  c->base.vtable = &plugin_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->plugin = plugin;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
new file mode 100644
index 0000000000..89073cb3d1
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_credentials_md_store *plugin_md;
+} grpc_plugin_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c
new file mode 100644
index 0000000000..545bca9d98
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c
@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/ssl/ssl_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+//
+// Utils
+//
+
+static void ssl_copy_key_material(const char *input, unsigned char **output,
+                                  size_t *output_size) {
+  *output_size = strlen(input);
+  *output = gpr_malloc(*output_size);
+  memcpy(*output, input, *output_size);
+}
+
+//
+// SSL Channel Credentials.
+//
+
+static void ssl_destruct(grpc_channel_credentials *creds) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
+  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
+}
+
+static grpc_security_status ssl_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_OK;
+  size_t i = 0;
+  const char *overridden_target_name = NULL;
+  grpc_arg new_arg;
+
+  for (i = 0; args && i < args->num_args; i++) {
+    grpc_arg *arg = &args->args[i];
+    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
+        arg->type == GRPC_ARG_STRING) {
+      overridden_target_name = arg->value.string;
+      break;
+    }
+  }
+  status = grpc_ssl_channel_security_connector_create(
+      call_creds, &c->config, target, overridden_target_name, sc);
+  if (status != GRPC_SECURITY_OK) {
+    return status;
+  }
+  new_arg.type = GRPC_ARG_STRING;
+  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
+  new_arg.value.string = "https";
+  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
+  return status;
+}
+
+static grpc_channel_credentials_vtable ssl_vtable = {
+    ssl_destruct, ssl_create_security_connector};
+
+static void ssl_build_config(const char *pem_root_certs,
+                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+                             grpc_ssl_config *config) {
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (pem_key_cert_pair != NULL) {
+    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pair->private_key,
+                          &config->pem_private_key,
+                          &config->pem_private_key_size);
+    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
+                          &config->pem_cert_chain,
+                          &config->pem_cert_chain_size);
+  }
+}
+
+grpc_channel_credentials *grpc_ssl_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+    void *reserved) {
+  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_credentials_create(pem_root_certs=%s, "
+      "pem_key_cert_pair=%p, "
+      "reserved=%p)",
+      3, (pem_root_certs, pem_key_cert_pair, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  c->base.vtable = &ssl_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
+  return &c->base;
+}
+
+//
+// SSL Server Credentials.
+//
+
+static void ssl_server_destruct(grpc_server_credentials *creds) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
+    if (c->config.pem_private_keys[i] != NULL) {
+      gpr_free(c->config.pem_private_keys[i]);
+    }
+    if (c->config.pem_cert_chains[i] != NULL) {
+      gpr_free(c->config.pem_cert_chains[i]);
+    }
+  }
+  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
+  if (c->config.pem_private_keys_sizes != NULL) {
+    gpr_free(c->config.pem_private_keys_sizes);
+  }
+  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
+  if (c->config.pem_cert_chains_sizes != NULL) {
+    gpr_free(c->config.pem_cert_chains_sizes);
+  }
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+}
+
+static grpc_security_status ssl_server_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  return grpc_ssl_server_security_connector_create(&c->config, sc);
+}
+
+static grpc_server_credentials_vtable ssl_server_vtable = {
+    ssl_server_destruct, ssl_server_create_security_connector};
+
+static void ssl_build_server_config(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    grpc_ssl_server_config *config) {
+  size_t i;
+  config->client_certificate_request = client_certificate_request;
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (num_key_cert_pairs > 0) {
+    GPR_ASSERT(pem_key_cert_pairs != NULL);
+    config->pem_private_keys =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_cert_chains =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_private_keys_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+    config->pem_cert_chains_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+  }
+  config->num_key_cert_pairs = num_key_cert_pairs;
+  for (i = 0; i < num_key_cert_pairs; i++) {
+    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
+                          &config->pem_private_keys[i],
+                          &config->pem_private_keys_sizes[i]);
+    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
+                          &config->pem_cert_chains[i],
+                          &config->pem_cert_chains_sizes[i]);
+  }
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
+  return grpc_ssl_server_credentials_create_ex(
+      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
+      force_client_auth
+          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
+          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+      reserved);
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    void *reserved) {
+  grpc_ssl_server_credentials *c =
+      gpr_malloc(sizeof(grpc_ssl_server_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_server_credentials_create_ex("
+      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
+      "client_certificate_request=%d, reserved=%p)",
+      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
+          client_certificate_request, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_server_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &ssl_server_vtable;
+  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
+                          num_key_cert_pairs, client_certificate_request,
+                          &c->config);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.h b/src/core/lib/security/credentials/ssl/ssl_credentials.h
new file mode 100644
index 0000000000..f23dbdbe49
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.h
@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_ssl_config config;
+} grpc_ssl_credentials;
+
+typedef struct {
+  grpc_server_credentials base;
+  grpc_ssl_server_config config;
+} grpc_ssl_server_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H */
diff --git a/src/core/lib/security/auth_filters.h b/src/core/lib/security/transport/auth_filters.h
index 7fb56c3f3a..f688d4ed21 100644
--- a/src/core/lib/security/auth_filters.h
+++ b/src/core/lib/security/transport/auth_filters.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
-#define GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
 
 #include "src/core/lib/channel/channel_stack.h"
 
 extern const grpc_channel_filter grpc_client_auth_filter;
 extern const grpc_channel_filter grpc_server_auth_filter;
 
-#endif /* GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H */
diff --git a/src/core/lib/security/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index 8b58cb86bf..14ccf72dc9 100644
--- a/src/core/lib/security/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <string.h>
 
@@ -40,9 +40,9 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
@@ -54,11 +54,11 @@ typedef struct {
   grpc_call_credentials *creds;
   grpc_mdstr *host;
   grpc_mdstr *method;
-  /* pollset bound to this call; if we need to make external
-     network requests, they should be done under this pollset
-     so that work can progress when this call wants work to
-     progress */
-  grpc_pollset *pollset;
+  /* pollset{_set} bound to this call; if we need to make external
+     network requests, they should be done under a pollset added to this
+     pollset_set so that work can progress when this call wants work to progress
+  */
+  grpc_polling_entity *pollent;
   grpc_transport_stream_op op;
   uint8_t security_context_set;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
@@ -91,14 +91,16 @@ static void bubble_up_error(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                             grpc_status_code status, const char *error_msg) {
   call_data *calld = elem->call_data;
   gpr_log(GPR_ERROR, "Client side authentication failure: %s", error_msg);
-  grpc_transport_stream_op_add_cancellation(&calld->op, status);
+  gpr_slice error_slice = gpr_slice_from_copied_string(error_msg);
+  grpc_transport_stream_op_add_close(&calld->op, status, &error_slice);
   grpc_call_next_op(exec_ctx, elem, &calld->op);
 }
 
 static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data,
                                     grpc_credentials_md *md_elems,
                                     size_t num_md,
-                                    grpc_credentials_status status) {
+                                    grpc_credentials_status status,
+                                    const char *error_details) {
   grpc_call_element *elem = (grpc_call_element *)user_data;
   call_data *calld = elem->call_data;
   grpc_transport_stream_op *op = &calld->op;
@@ -107,7 +109,9 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data,
   reset_auth_metadata_context(&calld->auth_md_context);
   if (status != GRPC_CREDENTIALS_OK) {
     bubble_up_error(exec_ctx, elem, GRPC_STATUS_UNAUTHENTICATED,
-                    "Credentials failed to get metadata.");
+                    (error_details != NULL && strlen(error_details) > 0)
+                        ? error_details
+                        : "Credentials failed to get metadata.");
     return;
   }
   GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT);
@@ -184,9 +188,9 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
   build_auth_metadata_context(&chand->security_connector->base,
                               chand->auth_context, calld);
   calld->op = *op; /* Copy op (originates from the caller's stack). */
-  GPR_ASSERT(calld->pollset);
+  GPR_ASSERT(calld->pollent != NULL);
   grpc_call_credentials_get_request_metadata(
-      exec_ctx, calld->creds, calld->pollset, calld->auth_md_context,
+      exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
       on_credentials_metadata, elem);
 }
 
@@ -220,8 +224,7 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
   grpc_linked_mdelem *l;
   grpc_client_security_context *sec_ctx = NULL;
 
-  if (calld->security_context_set == 0 &&
-      op->cancel_with_status == GRPC_STATUS_OK) {
+  if (calld->security_context_set == 0 && op->cancel_error == GRPC_ERROR_NONE) {
     calld->security_context_set = 1;
     GPR_ASSERT(op->context);
     if (op->context[GRPC_CONTEXT_SECURITY].value == NULL) {
@@ -270,15 +273,16 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   memset(calld, 0, sizeof(*calld));
 }
 
-static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                        grpc_pollset *pollset) {
+static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx,
+                                       grpc_call_element *elem,
+                                       grpc_polling_entity *pollent) {
   call_data *calld = elem->call_data;
-  calld->pollset = pollset;
+  calld->pollent = pollent;
 }
 
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {
+                              const grpc_call_stats *stats, void *ignored) {
   call_data *calld = elem->call_data;
   grpc_call_credentials_unref(calld->creds);
   if (calld->host != NULL) {
@@ -329,8 +333,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   GRPC_AUTH_CONTEXT_UNREF(chand->auth_context, "client_auth_filter");
 }
 
-const grpc_channel_filter grpc_client_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem,          set_pollset,          destroy_call_elem,
-    sizeof(channel_data),    init_channel_elem,    destroy_channel_elem,
-    grpc_call_next_get_peer, "client-auth"};
+const grpc_channel_filter grpc_client_auth_filter = {auth_start_transport_op,
+                                                     grpc_channel_next_op,
+                                                     sizeof(call_data),
+                                                     init_call_elem,
+                                                     set_pollset_or_pollset_set,
+                                                     destroy_call_elem,
+                                                     sizeof(channel_data),
+                                                     init_channel_elem,
+                                                     destroy_channel_elem,
+                                                     grpc_call_next_get_peer,
+                                                     "client-auth"};
diff --git a/src/core/lib/security/handshake.c b/src/core/lib/security/transport/handshake.c
index d5fe0c7b7d..b374ca7633 100644
--- a/src/core/lib/security/handshake.c
+++ b/src/core/lib/security/transport/handshake.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -39,8 +39,10 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/tsi_error.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
@@ -60,13 +62,16 @@ typedef struct {
   grpc_closure on_handshake_data_sent_to_peer;
   grpc_closure on_handshake_data_received_from_peer;
   grpc_auth_context *auth_context;
+  grpc_timer timer;
+  gpr_refcount refs;
 } grpc_security_handshake;
 
 static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
-                                                 void *setup, bool success);
+                                                 void *setup,
+                                                 grpc_error *error);
 
 static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void *setup,
-                                           bool success);
+                                           grpc_error *error);
 
 static void security_connector_remove_handshake(grpc_security_handshake *h) {
   GPR_ASSERT(!h->is_client_side);
@@ -95,16 +100,34 @@ static void security_connector_remove_handshake(grpc_security_handshake *h) {
   gpr_mu_unlock(&sc->mu);
 }
 
+static void unref_handshake(grpc_security_handshake *h) {
+  if (gpr_unref(&h->refs)) {
+    if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker);
+    if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer);
+    gpr_slice_buffer_destroy(&h->left_overs);
+    gpr_slice_buffer_destroy(&h->outgoing);
+    gpr_slice_buffer_destroy(&h->incoming);
+    GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
+    GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
+    gpr_free(h);
+  }
+}
+
 static void security_handshake_done(grpc_exec_ctx *exec_ctx,
                                     grpc_security_handshake *h,
-                                    int is_success) {
+                                    grpc_error *error) {
+  grpc_timer_cancel(exec_ctx, &h->timer);
   if (!h->is_client_side) {
     security_connector_remove_handshake(h);
   }
-  if (is_success) {
+  if (error == GRPC_ERROR_NONE) {
     h->cb(exec_ctx, h->user_data, GRPC_SECURITY_OK, h->secure_endpoint,
           h->auth_context);
   } else {
+    const char *msg = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "Security handshake failed: %s", msg);
+    grpc_error_free_string(msg);
+
     if (h->secure_endpoint != NULL) {
       grpc_endpoint_shutdown(exec_ctx, h->secure_endpoint);
       grpc_endpoint_destroy(exec_ctx, h->secure_endpoint);
@@ -113,14 +136,8 @@ static void security_handshake_done(grpc_exec_ctx *exec_ctx,
     }
     h->cb(exec_ctx, h->user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   }
-  if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker);
-  if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer);
-  gpr_slice_buffer_destroy(&h->left_overs);
-  gpr_slice_buffer_destroy(&h->outgoing);
-  gpr_slice_buffer_destroy(&h->incoming);
-  GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
-  GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
-  gpr_free(h);
+  unref_handshake(h);
+  GRPC_ERROR_UNREF(error);
 }
 
 static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
@@ -130,17 +147,20 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
   tsi_frame_protector *protector;
   tsi_result result;
   if (status != GRPC_SECURITY_OK) {
-    gpr_log(GPR_ERROR, "Error checking peer.");
-    security_handshake_done(exec_ctx, h, 0);
+    security_handshake_done(
+        exec_ctx, h,
+        grpc_error_set_int(GRPC_ERROR_CREATE("Error checking peer."),
+                           GRPC_ERROR_INT_SECURITY_STATUS, status));
     return;
   }
   h->auth_context = GRPC_AUTH_CONTEXT_REF(auth_context, "handshake");
   result =
       tsi_handshaker_create_frame_protector(h->handshaker, NULL, &protector);
   if (result != TSI_OK) {
-    gpr_log(GPR_ERROR, "Frame protector creation failed with error %s.",
-            tsi_result_to_string(result));
-    security_handshake_done(exec_ctx, h, 0);
+    security_handshake_done(
+        exec_ctx, h,
+        grpc_set_tsi_error_result(
+            GRPC_ERROR_CREATE("Frame protector creation failed"), result));
     return;
   }
   h->secure_endpoint =
@@ -148,7 +168,7 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
                                   h->left_overs.slices, h->left_overs.count);
   h->left_overs.count = 0;
   h->left_overs.length = 0;
-  security_handshake_done(exec_ctx, h, 1);
+  security_handshake_done(exec_ctx, h, GRPC_ERROR_NONE);
   return;
 }
 
@@ -157,9 +177,9 @@ static void check_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) {
   tsi_result result = tsi_handshaker_extract_peer(h->handshaker, &peer);
 
   if (result != TSI_OK) {
-    gpr_log(GPR_ERROR, "Peer extraction failed with error %s",
-            tsi_result_to_string(result));
-    security_handshake_done(exec_ctx, h, 0);
+    security_handshake_done(
+        exec_ctx, h, grpc_set_tsi_error_result(
+                         GRPC_ERROR_CREATE("Peer extraction failed"), result));
     return;
   }
   grpc_security_connector_check_peer(exec_ctx, h->connector, peer,
@@ -185,9 +205,9 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
   } while (result == TSI_INCOMPLETE_DATA);
 
   if (result != TSI_OK) {
-    gpr_log(GPR_ERROR, "Handshake failed with error %s",
-            tsi_result_to_string(result));
-    security_handshake_done(exec_ctx, h, 0);
+    security_handshake_done(exec_ctx, h,
+                            grpc_set_tsi_error_result(
+                                GRPC_ERROR_CREATE("Handshake failed"), result));
     return;
   }
 
@@ -203,7 +223,7 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
 
 static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
                                                  void *handshake,
-                                                 bool success) {
+                                                 grpc_error *error) {
   grpc_security_handshake *h = handshake;
   size_t consumed_slice_size = 0;
   tsi_result result = TSI_OK;
@@ -211,9 +231,10 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
   size_t num_left_overs;
   int has_left_overs_in_current_slice = 0;
 
-  if (!success) {
-    gpr_log(GPR_ERROR, "Read failed.");
-    security_handshake_done(exec_ctx, h, 0);
+  if (error != GRPC_ERROR_NONE) {
+    security_handshake_done(
+        exec_ctx, h,
+        GRPC_ERROR_CREATE_REFERENCING("Handshake read failed", &error, 1));
     return;
   }
 
@@ -238,9 +259,9 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
   }
 
   if (result != TSI_OK) {
-    gpr_log(GPR_ERROR, "Handshake failed with error %s",
-            tsi_result_to_string(result));
-    security_handshake_done(exec_ctx, h, 0);
+    security_handshake_done(exec_ctx, h,
+                            grpc_set_tsi_error_result(
+                                GRPC_ERROR_CREATE("Handshake failed"), result));
     return;
   }
 
@@ -270,13 +291,15 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
 
 /* If handshake is NULL, the handshake is done. */
 static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
-                                           void *handshake, bool success) {
+                                           void *handshake, grpc_error *error) {
   grpc_security_handshake *h = handshake;
 
   /* Make sure that write is OK. */
-  if (!success) {
-    gpr_log(GPR_ERROR, "Write failed.");
-    if (handshake != NULL) security_handshake_done(exec_ctx, h, 0);
+  if (error != GRPC_ERROR_NONE) {
+    if (handshake != NULL)
+      security_handshake_done(
+          exec_ctx, h,
+          GRPC_ERROR_CREATE_REFERENCING("Handshake write failed", &error, 1));
     return;
   }
 
@@ -291,13 +314,19 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
-                                tsi_handshaker *handshaker,
-                                grpc_security_connector *connector,
-                                bool is_client_side,
-                                grpc_endpoint *nonsecure_endpoint,
-                                grpc_security_handshake_done_cb cb,
-                                void *user_data) {
+static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  grpc_security_handshake *h = arg;
+  if (error == GRPC_ERROR_NONE) {
+    grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
+  }
+  unref_handshake(h);
+}
+
+void grpc_do_security_handshake(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector, bool is_client_side,
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data) {
   grpc_security_connector_handshake_list *handshake_node;
   grpc_security_handshake *h = gpr_malloc(sizeof(grpc_security_handshake));
   memset(h, 0, sizeof(grpc_security_handshake));
@@ -309,6 +338,7 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
   h->wrapped_endpoint = nonsecure_endpoint;
   h->user_data = user_data;
   h->cb = cb;
+  gpr_ref_init(&h->refs, 2); /* timer and handshake proper each get a ref */
   grpc_closure_init(&h->on_handshake_data_sent_to_peer,
                     on_handshake_data_sent_to_peer, h);
   grpc_closure_init(&h->on_handshake_data_received_from_peer,
@@ -327,6 +357,8 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
     gpr_mu_unlock(&server_connector->mu);
   }
   send_handshake_bytes_to_peer(exec_ctx, h);
+  grpc_timer_init(exec_ctx, &h->timer, deadline, on_timeout, h,
+                  gpr_now(deadline.clock_type));
 }
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/lib/security/handshake.h b/src/core/lib/security/transport/handshake.h
index f34476ed49..c0906dd6af 100644
--- a/src/core/lib/security/handshake.h
+++ b/src/core/lib/security/transport/handshake.h
@@ -31,21 +31,19 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
-#define GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
 
 #include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 /* Calls the callback upon completion. Takes owership of handshaker. */
-void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
-                                tsi_handshaker *handshaker,
-                                grpc_security_connector *connector,
-                                bool is_client_side,
-                                grpc_endpoint *nonsecure_endpoint,
-                                grpc_security_handshake_done_cb cb,
-                                void *user_data);
+void grpc_do_security_handshake(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector, bool is_client_side,
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data);
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
 
-#endif /* GRPC_CORE_LIB_SECURITY_HANDSHAKE_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H */
diff --git a/src/core/lib/security/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
index 27b0e98910..7650d68e89 100644
--- a/src/core/lib/security/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -31,13 +31,14 @@
  *
  */
 
-#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/sync.h>
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/security/transport/tsi_error.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
 
@@ -126,8 +127,8 @@ static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur,
 }
 
 static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep,
-                         bool success) {
-  if (grpc_trace_secure_endpoint) {
+                         grpc_error *error) {
+  if (false && grpc_trace_secure_endpoint) {
     size_t i;
     for (i = 0; i < ep->read_buffer->count; i++) {
       char *data = gpr_dump_slice(ep->read_buffer->slices[i],
@@ -137,11 +138,12 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep,
     }
   }
   ep->read_buffer = NULL;
-  grpc_exec_ctx_enqueue(exec_ctx, ep->read_cb, success, NULL);
+  grpc_exec_ctx_sched(exec_ctx, ep->read_cb, error, NULL);
   SECURE_ENDPOINT_UNREF(exec_ctx, ep, "read");
 }
 
-static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
+static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
+                    grpc_error *error) {
   unsigned i;
   uint8_t keep_looping = 0;
   tsi_result result = TSI_OK;
@@ -149,9 +151,10 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
   uint8_t *cur = GPR_SLICE_START_PTR(ep->read_staging_buffer);
   uint8_t *end = GPR_SLICE_END_PTR(ep->read_staging_buffer);
 
-  if (!success) {
+  if (error != GRPC_ERROR_NONE) {
     gpr_slice_buffer_reset_and_unref(ep->read_buffer);
-    call_read_cb(exec_ctx, ep, 0);
+    call_read_cb(exec_ctx, ep, GRPC_ERROR_CREATE_REFERENCING(
+                                   "Secure read failed", &error, 1));
     return;
   }
 
@@ -208,11 +211,12 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
 
   if (result != TSI_OK) {
     gpr_slice_buffer_reset_and_unref(ep->read_buffer);
-    call_read_cb(exec_ctx, ep, 0);
+    call_read_cb(exec_ctx, ep, grpc_set_tsi_error_result(
+                                   GRPC_ERROR_CREATE("Unwrap failed"), result));
     return;
   }
 
-  call_read_cb(exec_ctx, ep, 1);
+  call_read_cb(exec_ctx, ep, GRPC_ERROR_NONE);
 }
 
 static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
@@ -226,7 +230,7 @@ static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
   if (ep->leftover_bytes.count) {
     gpr_slice_buffer_swap(&ep->leftover_bytes, &ep->source_buffer);
     GPR_ASSERT(ep->leftover_bytes.count == 0);
-    on_read(exec_ctx, ep, 1);
+    on_read(exec_ctx, ep, GRPC_ERROR_NONE);
     return;
   }
 
@@ -252,7 +256,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
 
   gpr_slice_buffer_reset_and_unref(&ep->output_buffer);
 
-  if (grpc_trace_secure_endpoint) {
+  if (false && grpc_trace_secure_endpoint) {
     for (i = 0; i < slices->count; i++) {
       char *data =
           gpr_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
@@ -315,7 +319,10 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
   if (result != TSI_OK) {
     /* TODO(yangg) do different things according to the error type? */
     gpr_slice_buffer_reset_and_unref(&ep->output_buffer);
-    grpc_exec_ctx_enqueue(exec_ctx, cb, false, NULL);
+    grpc_exec_ctx_sched(
+        exec_ctx, cb,
+        grpc_set_tsi_error_result(GRPC_ERROR_CREATE("Wrap failed"), result),
+        NULL);
     return;
   }
 
diff --git a/src/core/lib/security/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h
index ff1c6639de..d00075b769 100644
--- a/src/core/lib/security/secure_endpoint.h
+++ b/src/core/lib/security/transport/secure_endpoint.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
-#define GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
 
 #include <grpc/support/slice.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -46,4 +46,4 @@ grpc_endpoint *grpc_secure_endpoint_create(
     struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
     gpr_slice *leftover_slices, size_t leftover_nslices);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */
diff --git a/src/core/lib/security/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 2d2023bdf5..f0ee6770e5 100644
--- a/src/core/lib/security/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -43,12 +43,12 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/alpn/alpn.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/handshake.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/iomgr/load_file.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/handshake.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/support/env.h"
-#include "src/core/lib/support/load_file.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/fake_transport_security.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
@@ -127,23 +127,25 @@ void grpc_server_security_connector_shutdown(
 
 void grpc_channel_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
-    grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb,
-    void *user_data) {
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data) {
   if (sc == NULL || nonsecure_endpoint == NULL) {
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
-    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data);
+    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
 void grpc_server_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
     grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_security_handshake_done_cb cb, void *user_data) {
+    gpr_timespec deadline, grpc_security_handshake_done_cb cb,
+    void *user_data) {
   if (sc == NULL || nonsecure_endpoint == NULL) {
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
-    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, cb, user_data);
+    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, deadline, cb,
+                     user_data);
   }
 }
 
@@ -310,20 +312,23 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
 static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx,
                                       grpc_channel_security_connector *sc,
                                       grpc_endpoint *nonsecure_endpoint,
+                                      gpr_timespec deadline,
                                       grpc_security_handshake_done_cb cb,
                                       void *user_data) {
   grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base,
-                             true, nonsecure_endpoint, cb, user_data);
+                             true, nonsecure_endpoint, deadline, cb, user_data);
 }
 
 static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx,
                                      grpc_server_security_connector *sc,
                                      grpc_tcp_server_acceptor *acceptor,
                                      grpc_endpoint *nonsecure_endpoint,
+                                     gpr_timespec deadline,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
   grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base,
-                             false, nonsecure_endpoint, cb, user_data);
+                             false, nonsecure_endpoint, deadline, cb,
+                             user_data);
 }
 
 static grpc_security_connector_vtable fake_channel_vtable = {
@@ -413,6 +418,7 @@ static grpc_security_status ssl_create_handshaker(
 static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_security_connector *sc,
                                      grpc_endpoint *nonsecure_endpoint,
+                                     gpr_timespec deadline,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
   grpc_ssl_channel_security_connector *c =
@@ -427,7 +433,7 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
     cb(exec_ctx, user_data, status, NULL, NULL);
   } else {
     grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, cb, user_data);
+                               nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
@@ -435,6 +441,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx,
                                     grpc_server_security_connector *sc,
                                     grpc_tcp_server_acceptor *acceptor,
                                     grpc_endpoint *nonsecure_endpoint,
+                                    gpr_timespec deadline,
                                     grpc_security_handshake_done_cb cb,
                                     void *user_data) {
   grpc_ssl_server_security_connector *c =
@@ -446,7 +453,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx,
     cb(exec_ctx, user_data, status, NULL, NULL);
   } else {
     grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false,
-                               nonsecure_endpoint, cb, user_data);
+                               nonsecure_endpoint, deadline, cb, user_data);
   }
 }
 
@@ -635,7 +642,8 @@ static gpr_slice compute_default_pem_root_certs_once(void) {
   char *default_root_certs_path =
       gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
   if (default_root_certs_path != NULL) {
-    result = gpr_load_file(default_root_certs_path, 0, NULL);
+    GRPC_LOG_IF_ERROR("load_file",
+                      grpc_load_file(default_root_certs_path, 0, &result));
     gpr_free(default_root_certs_path);
   }
 
@@ -653,7 +661,8 @@ static gpr_slice compute_default_pem_root_certs_once(void) {
   /* Fall back to installed certs if needed. */
   if (GPR_SLICE_IS_EMPTY(result) &&
       ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) {
-    result = gpr_load_file(installed_roots_path, 0, NULL);
+    GRPC_LOG_IF_ERROR("load_file",
+                      grpc_load_file(installed_roots_path, 0, &result));
   }
   return result;
 }
diff --git a/src/core/lib/security/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 2c893cd5e9..c2ddf5ee1e 100644
--- a/src/core/lib/security/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -143,7 +143,7 @@ struct grpc_channel_security_connector {
                           grpc_security_call_host_check_cb cb, void *user_data);
   void (*do_handshake)(grpc_exec_ctx *exec_ctx,
                        grpc_channel_security_connector *sc,
-                       grpc_endpoint *nonsecure_endpoint,
+                       grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
                        grpc_security_handshake_done_cb cb, void *user_data);
 };
 
@@ -156,8 +156,8 @@ void grpc_channel_security_connector_check_call_host(
 /* Handshake. */
 void grpc_channel_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
-    grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb,
-    void *user_data);
+    grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
+    grpc_security_handshake_done_cb cb, void *user_data);
 
 /* --- server_security_connector object. ---
 
@@ -174,14 +174,14 @@ struct grpc_server_security_connector {
   void (*do_handshake)(grpc_exec_ctx *exec_ctx,
                        grpc_server_security_connector *sc,
                        grpc_tcp_server_acceptor *acceptor,
-                       grpc_endpoint *nonsecure_endpoint,
+                       grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline,
                        grpc_security_handshake_done_cb cb, void *user_data);
 };
 
 void grpc_server_security_connector_do_handshake(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
     grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_security_handshake_done_cb cb, void *user_data);
+    gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data);
 
 void grpc_server_security_connector_shutdown(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector);
@@ -263,4 +263,4 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     const grpc_auth_context *auth_context);
 void tsi_shallow_peer_destruct(tsi_peer *peer);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H */
diff --git a/src/core/lib/security/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
index 3320497d21..12e789bde9 100644
--- a/src/core/lib/security/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -33,9 +33,9 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -128,7 +128,7 @@ static void on_md_processing_done(
     grpc_metadata_batch_filter(calld->recv_initial_metadata, remove_consumed_md,
                                elem);
     grpc_metadata_array_destroy(&calld->md);
-    calld->on_done_recv->cb(&exec_ctx, calld->on_done_recv->cb_arg, 1);
+    grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL);
   } else {
     gpr_slice message;
     grpc_transport_stream_op close_op;
@@ -146,18 +146,21 @@ static void on_md_processing_done(
     calld->transport_op.send_trailing_metadata = NULL;
     grpc_transport_stream_op_add_close(&close_op, status, &message);
     grpc_call_next_op(&exec_ctx, elem, &close_op);
-    calld->on_done_recv->cb(&exec_ctx, calld->on_done_recv->cb_arg, 0);
+    grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv,
+                        grpc_error_set_int(GRPC_ERROR_CREATE(error_details),
+                                           GRPC_ERROR_INT_GRPC_STATUS, status),
+                        NULL);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
-                         bool success) {
+                         grpc_error *error) {
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     if (chand->creds->processor.process != NULL) {
       calld->md = metadata_batch_to_md_array(calld->recv_initial_metadata);
       chand->creds->processor.process(
@@ -166,7 +169,8 @@ static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
       return;
     }
   }
-  calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success);
+  grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv, GRPC_ERROR_REF(error),
+                      NULL);
 }
 
 static void set_recv_ops_md_callbacks(grpc_call_element *elem,
@@ -220,12 +224,9 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
       grpc_server_security_context_destroy;
 }
 
-static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                        grpc_pollset *pollset) {}
-
 /* Destructor for call_data */
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {}
+                              const grpc_call_stats *stats, void *ignored) {}
 
 /* Constructor for channel_data */
 static void init_channel_elem(grpc_exec_ctx *exec_ctx,
@@ -258,7 +259,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 }
 
 const grpc_channel_filter grpc_server_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op, sizeof(call_data),
-    init_call_elem,          set_pollset,          destroy_call_elem,
-    sizeof(channel_data),    init_channel_elem,    destroy_channel_elem,
-    grpc_call_next_get_peer, "server-auth"};
+    auth_start_transport_op,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    "server-auth"};
diff --git a/src/core/lib/security/transport/tsi_error.c b/src/core/lib/security/transport/tsi_error.c
new file mode 100644
index 0000000000..afc1733567
--- /dev/null
+++ b/src/core/lib/security/transport/tsi_error.c
@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/transport/tsi_error.h"
+
+grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result) {
+  return grpc_error_set_int(grpc_error_set_str(error, GRPC_ERROR_STR_TSI_ERROR,
+                                               tsi_result_to_string(result)),
+                            GRPC_ERROR_INT_TSI_CODE, result);
+}
diff --git a/src/core/lib/security/transport/tsi_error.h b/src/core/lib/security/transport/tsi_error.h
new file mode 100644
index 0000000000..636fbb89cf
--- /dev/null
+++ b/src/core/lib/security/transport/tsi_error.h
@@ -0,0 +1,42 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_TSI_ERROR_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_TSI_ERROR_H
+
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/tsi/transport_security_interface.h"
+
+grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result);
+
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_TSI_ERROR_H */
diff --git a/src/core/lib/security/b64.c b/src/core/lib/security/util/b64.c
index 87f0e05280..9da42e4e73 100644
--- a/src/core/lib/security/b64.c
+++ b/src/core/lib/security/util/b64.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 
 #include <stdint.h>
 #include <string.h>
diff --git a/src/core/lib/security/b64.h b/src/core/lib/security/util/b64.h
index c515e7af2c..6908095287 100644
--- a/src/core/lib/security/b64.h
+++ b/src/core/lib/security/util/b64.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_B64_H
-#define GRPC_CORE_LIB_SECURITY_B64_H
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_B64_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_B64_H
 
 #include <grpc/support/slice.h>
 
@@ -49,4 +49,4 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe);
 gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
                                       int url_safe);
 
-#endif /* GRPC_CORE_LIB_SECURITY_B64_H */
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_B64_H */
diff --git a/src/core/lib/security/util/json_util.c b/src/core/lib/security/util/json_util.c
new file mode 100644
index 0000000000..7eed039baa
--- /dev/null
+++ b/src/core/lib/security/util/json_util.c
@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/util/json_util.h"
+
+#include <string.h>
+
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name) {
+  grpc_json *child;
+  for (child = json->child; child != NULL; child = child->next) {
+    if (strcmp(child->key, prop_name) == 0) break;
+  }
+  if (child == NULL || child->type != GRPC_JSON_STRING) {
+    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
+    return NULL;
+  }
+  return child->value;
+}
+
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name,
+                                    char **copied_value) {
+  const char *prop_value = grpc_json_get_string_property(json, prop_name);
+  if (prop_value == NULL) return false;
+  *copied_value = gpr_strdup(prop_value);
+  return true;
+}
diff --git a/src/core/lib/security/util/json_util.h b/src/core/lib/security/util/json_util.h
new file mode 100644
index 0000000000..137900593f
--- /dev/null
+++ b/src/core/lib/security/util/json_util.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+
+#include <stdbool.h>
+
+#include "src/core/lib/json/json.h"
+
+// Constants.
+#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
+#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
+#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
+
+// Gets a child property from a json node.
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name);
+
+// Copies the value of the json child property specified by prop_name.
+// Returns false if the property was not found.
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name, char **copied_value);
+
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H */
diff --git a/src/core/lib/support/avl.c b/src/core/lib/support/avl.c
index 8d3ce23e6c..acf8fd5a55 100644
--- a/src/core/lib/support/avl.c
+++ b/src/core/lib/support/avl.c
@@ -124,6 +124,15 @@ void *gpr_avl_get(gpr_avl avl, void *key) {
   return node ? node->value : NULL;
 }
 
+int gpr_avl_maybe_get(gpr_avl avl, void *key, void **value) {
+  gpr_avl_node *node = get(avl.vtable, avl.root, key);
+  if (node != NULL) {
+    *value = node->value;
+    return 1;
+  }
+  return 0;
+}
+
 static gpr_avl_node *rotate_left(const gpr_avl_vtable *vtable, void *key,
                                  void *value, gpr_avl_node *left,
                                  gpr_avl_node *right) {
@@ -286,3 +295,5 @@ gpr_avl gpr_avl_ref(gpr_avl avl) {
 }
 
 void gpr_avl_unref(gpr_avl avl) { unref_node(avl.vtable, avl.root); }
+
+int gpr_avl_is_empty(gpr_avl avl) { return avl.root == NULL; }
diff --git a/src/core/lib/support/cpu_windows.c b/src/core/lib/support/cpu_windows.c
index ce32eb0a9d..34d006bfc8 100644
--- a/src/core/lib/support/cpu_windows.c
+++ b/src/core/lib/support/cpu_windows.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 #include <grpc/support/log.h>
 
 unsigned gpr_cpu_num_cores(void) {
@@ -44,4 +44,4 @@ unsigned gpr_cpu_num_cores(void) {
 
 unsigned gpr_cpu_current_cpu(void) { return GetCurrentProcessorNumber(); }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/env_win32.c b/src/core/lib/support/env_windows.c
index e670e1e8d0..9116959442 100644
--- a/src/core/lib/support/env_win32.c
+++ b/src/core/lib/support/env_windows.c
@@ -33,13 +33,13 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_ENV
+#ifdef GPR_WINDOWS_ENV
 
 #include <windows.h>
 
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -76,4 +76,4 @@ void gpr_setenv(const char *name, const char *value) {
   GPR_ASSERT(res);
 }
 
-#endif /* GPR_WIN32_ENV */
+#endif /* GPR_WINDOWS_ENV */
diff --git a/src/core/lib/support/log.c b/src/core/lib/support/log.c
index 882abf673c..bae0957df7 100644
--- a/src/core/lib/support/log.c
+++ b/src/core/lib/support/log.c
@@ -96,4 +96,6 @@ void gpr_log_verbosity_init() {
   }
 }
 
-void gpr_set_log_function(gpr_log_func f) { g_log_func = f; }
+void gpr_set_log_function(gpr_log_func f) {
+  g_log_func = f ? f : gpr_default_log;
+}
diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c
index ca04c022e3..508fae4eec 100644
--- a/src/core/lib/support/log_linux.c
+++ b/src/core/lib/support/log_linux.c
@@ -95,9 +95,9 @@ void gpr_default_log(gpr_log_func_args *args) {
     strcpy(time_buffer, "error:strftime");
   }
 
-  gpr_asprintf(&prefix, "%s%s.%09d %7tu %s:%d]",
+  gpr_asprintf(&prefix, "%s%s.%09" PRId32 " %7ld %s:%d]",
                gpr_log_severity_string(args->severity), time_buffer,
-               (int)(now.tv_nsec), gettid(), display_file, args->line);
+               now.tv_nsec, gettid(), display_file, args->line);
 
   fprintf(stderr, "%-60s %s\n", prefix, args->message);
   gpr_free(prefix);
diff --git a/src/core/lib/support/log_win32.c b/src/core/lib/support/log_windows.c
index 29735bd18c..ea898c359d 100644
--- a/src/core/lib/support/log_win32.c
+++ b/src/core/lib/support/log_windows.c
@@ -33,19 +33,19 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_LOG
+#ifdef GPR_WINDOWS_LOG
 
 #include <stdarg.h>
 #include <stdio.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
 #include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 
 void gpr_log(const char *file, int line, gpr_log_severity severity,
              const char *format, ...) {
@@ -109,4 +109,4 @@ void gpr_default_log(gpr_log_func_args *args) {
   fflush(stderr);
 }
 
-#endif /* GPR_WIN32_LOG */
+#endif /* GPR_WINDOWS_LOG */
diff --git a/src/core/lib/support/murmur_hash.c b/src/core/lib/support/murmur_hash.c
index 5711fff0c0..7137c1f313 100644
--- a/src/core/lib/support/murmur_hash.c
+++ b/src/core/lib/support/murmur_hash.c
@@ -33,6 +33,8 @@
 
 #include "src/core/lib/support/murmur_hash.h"
 
+#include <string.h>
+
 #define ROTL32(x, r) ((x) << (r)) | ((x) >> (32 - (r)))
 
 #define FMIX32(h)    \
@@ -42,10 +44,6 @@
   (h) *= 0xc2b2ae35; \
   (h) ^= (h) >> 16;
 
-/* Block read - if your platform needs to do endian-swapping or can only
-   handle aligned reads, do the conversion here */
-#define GETBLOCK32(p, i) (p)[(i)]
-
 uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
   const uint8_t *data = (const uint8_t *)key;
   const size_t nblocks = len / 4;
@@ -62,7 +60,7 @@ uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
 
   /* body */
   for (i = -(int)nblocks; i; i++) {
-    k1 = GETBLOCK32(blocks, i);
+    memcpy(&k1, blocks + i, sizeof(uint32_t));
 
     k1 *= c1;
     k1 = ROTL32(k1, 15);
diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c
index a2ab6c5f1f..30c1e67647 100644
--- a/src/core/lib/support/string.c
+++ b/src/core/lib/support/string.c
@@ -194,6 +194,16 @@ int int64_ttoa(int64_t value, char *string) {
   return i;
 }
 
+char *gpr_leftpad(const char *str, char flag, size_t length) {
+  const size_t str_length = strlen(str);
+  const size_t out_length = str_length > length ? str_length : length;
+  char *out = gpr_malloc(out_length + 1);
+  memset(out, flag, out_length - str_length);
+  memcpy(out + out_length - str_length, str, str_length);
+  out[out_length] = 0;
+  return out;
+}
+
 char *gpr_strjoin(const char **strs, size_t nstrs, size_t *final_length) {
   return gpr_strjoin_sep(strs, nstrs, "", final_length);
 }
diff --git a/src/core/lib/support/string.h b/src/core/lib/support/string.h
index ea58610914..2b6bb3eec6 100644
--- a/src/core/lib/support/string.h
+++ b/src/core/lib/support/string.h
@@ -83,6 +83,10 @@ int int64_ttoa(int64_t value, char *output);
 /* Reverse a run of bytes */
 void gpr_reverse_bytes(char *str, int len);
 
+/* Pad a string with flag characters. The given length specifies the minimum
+   field width. The input string is never truncated. */
+char *gpr_leftpad(const char *str, char flag, size_t length);
+
 /* Join a set of strings, returning the resulting string.
    Total combined length (excluding null terminator) is returned in total_length
    if it is non-null. */
diff --git a/src/core/lib/support/string_util_win32.c b/src/core/lib/support/string_util_windows.c
index f3cb0c050f..049c9a8c04 100644
--- a/src/core/lib/support/string_util_win32.c
+++ b/src/core/lib/support/string_util_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 /* Some platforms (namely msys) need wchar to be included BEFORE
    anything else, especially strsafe.h. */
@@ -83,7 +83,7 @@ char *gpr_format_message(int messageid) {
   DWORD status = FormatMessage(
       FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
           FORMAT_MESSAGE_IGNORE_INSERTS,
-      NULL, (DWORD)messageid, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+      NULL, (DWORD)messageid, MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
       (LPTSTR)(&tmessage), 0, NULL);
   if (status == 0) return gpr_strdup("Unable to retrieve error string");
   message = gpr_tchar_to_char(tmessage);
@@ -91,4 +91,4 @@ char *gpr_format_message(int messageid) {
   return message;
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/string_win32.c b/src/core/lib/support/string_windows.c
index 6b92f79253..ecc2a3a4e5 100644
--- a/src/core/lib/support/string_win32.c
+++ b/src/core/lib/support/string_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_STRING
+#ifdef GPR_WINDOWS_STRING
 
 #include <stdarg.h>
 #include <stdio.h>
@@ -80,4 +80,4 @@ int gpr_asprintf(char **strp, const char *format, ...) {
   return -1;
 }
 
-#endif /* GPR_WIN32_STRING */
+#endif /* GPR_WINDOWS_STRING */
diff --git a/src/core/lib/support/string_win32.h b/src/core/lib/support/string_windows.h
index ff4a694ca9..899563b72d 100644
--- a/src/core/lib/support/string_win32.h
+++ b/src/core/lib/support/string_windows.h
@@ -31,17 +31,17 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H
-#define GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H
+#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H
+#define GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 /* These allocate new strings using gpr_malloc to convert from and to utf-8. */
 LPTSTR gpr_char_to_tchar(LPCSTR input);
 LPSTR gpr_tchar_to_char(LPCTSTR input);
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
 
-#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H */
+#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H */
diff --git a/src/core/lib/support/subprocess_windows.c b/src/core/lib/support/subprocess_windows.c
index 264306f1bd..dee8c44ac1 100644
--- a/src/core/lib/support/subprocess_windows.c
+++ b/src/core/lib/support/subprocess_windows.c
@@ -43,7 +43,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/subprocess.h>
 #include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 
 struct gpr_subprocess {
   PROCESS_INFORMATION pi;
diff --git a/src/core/lib/support/sync_win32.c b/src/core/lib/support/sync_windows.c
index 470a9f9704..8f0e8ff69f 100644
--- a/src/core/lib/support/sync_win32.c
+++ b/src/core/lib/support/sync_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -130,4 +130,4 @@ void gpr_once_init(gpr_once *once, void (*init_function)(void)) {
   InitOnceExecuteOnce(once, run_once_func, &arg, &dummy);
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/thd_win32.c b/src/core/lib/support/thd_windows.c
index 6deb3140eb..74d2250df4 100644
--- a/src/core/lib/support/thd_win32.c
+++ b/src/core/lib/support/thd_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -114,4 +114,4 @@ void gpr_thd_join(gpr_thd_id t) {
   destroy_thread(info);
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/core/lib/support/time_win32.c b/src/core/lib/support/time_windows.c
index 9e924ab3f4..6459732879 100644
--- a/src/core/lib/support/time_win32.c
+++ b/src/core/lib/support/time_windows.c
@@ -35,7 +35,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_TIME
+#ifdef GPR_WINDOWS_TIME
 
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
@@ -107,4 +107,4 @@ void gpr_sleep_until(gpr_timespec until) {
   }
 }
 
-#endif /* GPR_WIN32_TIME */
+#endif /* GPR_WINDOWS_TIME */
diff --git a/src/core/lib/support/tmpfile_msys.c b/src/core/lib/support/tmpfile_msys.c
index 2fdc89a64f..4f566c4c28 100644
--- a/src/core/lib/support/tmpfile_msys.c
+++ b/src/core/lib/support/tmpfile_msys.c
@@ -44,7 +44,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 #include "src/core/lib/support/tmpfile.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
diff --git a/src/core/lib/support/tmpfile_win32.c b/src/core/lib/support/tmpfile_windows.c
index 9ac73128c3..542f53e589 100644
--- a/src/core/lib/support/tmpfile_win32.c
+++ b/src/core/lib/support/tmpfile_windows.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32_TMPFILE
+#ifdef GPR_WINDOWS_TMPFILE
 
 #include <io.h>
 #include <stdio.h>
@@ -44,7 +44,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string_win32.h"
+#include "src/core/lib/support/string_windows.h"
 #include "src/core/lib/support/tmpfile.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
@@ -81,4 +81,4 @@ end:
   return result;
 }
 
-#endif /* GPR_WIN32_TMPFILE */
+#endif /* GPR_WINDOWS_TMPFILE */
diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c
index 2cf2f00b31..aa9d60ee6a 100644
--- a/src/core/lib/surface/alarm.c
+++ b/src/core/lib/surface/alarm.c
@@ -48,9 +48,9 @@ struct grpc_alarm {
 static void do_nothing_end_completion(grpc_exec_ctx *exec_ctx, void *arg,
                                       grpc_cq_completion *c) {}
 
-static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_alarm *alarm = arg;
-  grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, success,
+  grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, error,
                  do_nothing_end_completion, NULL, &alarm->completion);
 }
 
diff --git a/src/core/lib/surface/byte_buffer_reader.c b/src/core/lib/surface/byte_buffer_reader.c
index 809fd5f1fa..c97079f638 100644
--- a/src/core/lib/surface/byte_buffer_reader.c
+++ b/src/core/lib/surface/byte_buffer_reader.c
@@ -62,12 +62,19 @@ void grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader,
     case GRPC_BB_RAW:
       gpr_slice_buffer_init(&decompressed_slices_buffer);
       if (is_compressed(reader->buffer_in)) {
-        grpc_msg_decompress(reader->buffer_in->data.raw.compression,
-                            &reader->buffer_in->data.raw.slice_buffer,
-                            &decompressed_slices_buffer);
-        reader->buffer_out =
-            grpc_raw_byte_buffer_create(decompressed_slices_buffer.slices,
-                                        decompressed_slices_buffer.count);
+        if (grpc_msg_decompress(reader->buffer_in->data.raw.compression,
+                                &reader->buffer_in->data.raw.slice_buffer,
+                                &decompressed_slices_buffer) == 0) {
+          gpr_log(GPR_ERROR,
+                  "Unexpected error decompressing data for algorithm with enum "
+                  "value '%d'. Reading data as if it were uncompressed.",
+                  reader->buffer_in->data.raw.compression);
+          reader->buffer_out = reader->buffer_in;
+        } else { /* all fine */
+          reader->buffer_out =
+              grpc_raw_byte_buffer_create(decompressed_slices_buffer.slices,
+                                          decompressed_slices_buffer.count);
+        }
         gpr_slice_buffer_destroy(&decompressed_slices_buffer);
       } else { /* not compressed, use the input buffer as output */
         reader->buffer_out = reader->buffer_in;
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index 9b2b94eedf..708ea3502a 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -40,6 +40,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/slice.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
@@ -52,7 +53,9 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/transport/metadata.h"
 #include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport.h"
 
 /** The maximum number of concurrent batches possible.
     Based upon the maximum number of individually queueable ops in the batch
@@ -65,12 +68,6 @@
       - status/close recv (depending on client/server) */
 #define MAX_CONCURRENT_BATCHES 6
 
-typedef struct {
-  grpc_ioreq_completion_func on_complete;
-  void *user_data;
-  int success;
-} completed_request;
-
 #define MAX_SEND_EXTRA_METADATA_COUNT 3
 
 /* Status data for a request can come from several sources; this
@@ -97,31 +94,13 @@ typedef struct {
   grpc_mdstr *details;
 } received_status;
 
-/* How far through the GRPC stream have we read? */
-typedef enum {
-  /* We are still waiting for initial metadata to complete */
-  READ_STATE_INITIAL = 0,
-  /* We have gotten initial metadata, and are reading either
-     messages or trailing metadata */
-  READ_STATE_GOT_INITIAL_METADATA,
-  /* The stream is closed for reading */
-  READ_STATE_READ_CLOSED,
-  /* The stream is closed for reading & writing */
-  READ_STATE_STREAM_CLOSED
-} read_state;
-
-typedef enum {
-  WRITE_STATE_INITIAL = 0,
-  WRITE_STATE_STARTED,
-  WRITE_STATE_WRITE_CLOSED
-} write_state;
-
 typedef struct batch_control {
   grpc_call *call;
   grpc_cq_completion cq_completion;
   grpc_closure finish_batch;
   void *notify_tag;
   gpr_refcount steps_to_complete;
+  grpc_error *error;
 
   uint8_t send_initial_metadata;
   uint8_t send_message;
@@ -130,11 +109,11 @@ typedef struct batch_control {
   uint8_t recv_message;
   uint8_t recv_final_op;
   uint8_t is_notify_tag_closure;
-  uint8_t success;
 } batch_control;
 
 struct grpc_call {
   grpc_completion_queue *cq;
+  grpc_polling_entity pollent;
   grpc_channel *channel;
   grpc_call *parent;
   grpc_call *first_child;
@@ -176,10 +155,10 @@ struct grpc_call {
   received_status status[STATUS_SOURCE_COUNT];
 
   /* Call stats: only valid after trailing metadata received */
-  grpc_transport_stream_stats stats;
+  grpc_call_stats stats;
 
-  /* Compression algorithm for the call */
-  grpc_compression_algorithm compression_algorithm;
+  /* Compression algorithm for *incoming* data */
+  grpc_compression_algorithm incoming_compression_algorithm;
   /* Supported encodings (compression algorithms), a bitset */
   uint32_t encodings_accepted_by_peer;
 
@@ -238,18 +217,19 @@ static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
 static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
                                           grpc_status_code status,
                                           const char *description);
+static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
+                                         grpc_status_code status,
+                                         const char *description);
 static void destroy_call(grpc_exec_ctx *exec_ctx, void *call_stack,
-                         bool success);
+                         grpc_error *error);
 static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
-                                  bool success);
-
-grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
-                            uint32_t propagation_mask,
-                            grpc_completion_queue *cq,
-                            const void *server_transport_data,
-                            grpc_mdelem **add_initial_metadata,
-                            size_t add_initial_metadata_count,
-                            gpr_timespec send_deadline) {
+                                  grpc_error *error);
+
+grpc_call *grpc_call_create(
+    grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+    grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative,
+    const void *server_transport_data, grpc_mdelem **add_initial_metadata,
+    size_t add_initial_metadata_count, gpr_timespec send_deadline) {
   size_t i, j;
   grpc_channel_stack *channel_stack = grpc_channel_get_channel_stack(channel);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -261,6 +241,8 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
   call->channel = channel;
   call->cq = cq;
   call->parent = parent_call;
+  /* Always support no compression */
+  GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE);
   call->is_client = server_transport_data == NULL;
   if (call->is_client) {
     GPR_ASSERT(add_initial_metadata_count < MAX_SEND_EXTRA_METADATA_COUNT);
@@ -284,9 +266,20 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
                        call->context, server_transport_data,
                        CALL_STACK_FROM_CALL(call));
   if (cq != NULL) {
+    GPR_ASSERT(
+        pollset_set_alternative == NULL &&
+        "Only one of 'cq' and 'pollset_set_alternative' should be non-NULL.");
     GRPC_CQ_INTERNAL_REF(cq, "bind");
-    grpc_call_stack_set_pollset(&exec_ctx, CALL_STACK_FROM_CALL(call),
-                                grpc_cq_pollset(cq));
+    call->pollent =
+        grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
+  }
+  if (pollset_set_alternative != NULL) {
+    call->pollent =
+        grpc_polling_entity_create_from_pollset_set(pollset_set_alternative);
+  }
+  if (!grpc_polling_entity_is_empty(&call->pollent)) {
+    grpc_call_stack_set_pollset_or_pollset_set(
+        &exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent);
   }
   if (parent_call != NULL) {
     GRPC_CALL_INTERNAL_REF(parent_call, "child");
@@ -341,10 +334,16 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
 void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call,
                                     grpc_completion_queue *cq) {
   GPR_ASSERT(cq);
+
+  if (grpc_polling_entity_pollset_set(&call->pollent) != NULL) {
+    gpr_log(GPR_ERROR, "A pollset_set is already registered for this call.");
+    abort();
+  }
   call->cq = cq;
   GRPC_CQ_INTERNAL_REF(cq, "bind");
-  grpc_call_stack_set_pollset(exec_ctx, CALL_STACK_FROM_CALL(call),
-                              grpc_cq_pollset(cq));
+  call->pollent = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
+  grpc_call_stack_set_pollset_or_pollset_set(
+      exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent);
 }
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
@@ -361,7 +360,8 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) {
   GRPC_CALL_STACK_UNREF(exec_ctx, CALL_STACK_FROM_CALL(c), REF_REASON);
 }
 
-static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, bool success) {
+static void destroy_call(grpc_exec_ctx *exec_ctx, void *call,
+                         grpc_error *error) {
   size_t i;
   int ii;
   grpc_call *c = call;
@@ -391,7 +391,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, bool success) {
     GRPC_CQ_INTERNAL_UNREF(c->cq, "bind");
   }
   grpc_channel *channel = c->channel;
-  grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), c);
+  grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->stats, c);
   GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call");
   GPR_TIMER_END("destroy_call", 0);
 }
@@ -402,24 +402,74 @@ static void set_status_code(grpc_call *call, status_source source,
 
   call->status[source].is_set = 1;
   call->status[source].code = (grpc_status_code)status;
+}
+
+static void set_status_details(grpc_call *call, status_source source,
+                               grpc_mdstr *status) {
+  if (call->status[source].details != NULL) {
+    GRPC_MDSTR_UNREF(status);
+  } else {
+    call->status[source].details = status;
+  }
+}
+
+static void get_final_status(grpc_call *call,
+                             void (*set_value)(grpc_status_code code,
+                                               void *user_data),
+                             void *set_value_user_data) {
+  int i;
+  for (i = 0; i < STATUS_SOURCE_COUNT; i++) {
+    if (call->status[i].is_set) {
+      set_value(call->status[i].code, set_value_user_data);
+      return;
+    }
+  }
+  if (call->is_client) {
+    set_value(GRPC_STATUS_UNKNOWN, set_value_user_data);
+  } else {
+    set_value(GRPC_STATUS_OK, set_value_user_data);
+  }
+}
 
-  /* TODO(ctiller): what to do about the flush that was previously here */
+static void set_status_from_error(grpc_call *call, status_source source,
+                                  grpc_error *error) {
+  intptr_t status;
+  if (grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &status)) {
+    set_status_code(call, source, (uint32_t)status);
+  } else {
+    set_status_code(call, source, GRPC_STATUS_INTERNAL);
+  }
+  const char *msg = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE);
+  bool free_msg = false;
+  if (msg == NULL) {
+    free_msg = true;
+    msg = grpc_error_string(error);
+  }
+  set_status_details(call, source, grpc_mdstr_from_string(msg));
+  if (free_msg) grpc_error_free_string(msg);
 }
 
-static void set_compression_algorithm(grpc_call *call,
-                                      grpc_compression_algorithm algo) {
-  call->compression_algorithm = algo;
+static void set_incoming_compression_algorithm(
+    grpc_call *call, grpc_compression_algorithm algo) {
+  GPR_ASSERT(algo < GRPC_COMPRESS_ALGORITHMS_COUNT);
+  call->incoming_compression_algorithm = algo;
 }
 
 grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
     grpc_call *call) {
   grpc_compression_algorithm algorithm;
   gpr_mu_lock(&call->mu);
-  algorithm = call->compression_algorithm;
+  algorithm = call->incoming_compression_algorithm;
   gpr_mu_unlock(&call->mu);
   return algorithm;
 }
 
+static grpc_compression_algorithm compression_algorithm_for_level_locked(
+    grpc_call *call, grpc_compression_level level) {
+  return grpc_compression_algorithm_for_level(level,
+                                              call->encodings_accepted_by_peer);
+}
+
 uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) {
   uint32_t flags;
   gpr_mu_lock(&call->mu);
@@ -485,32 +535,6 @@ uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call) {
   return encodings_accepted_by_peer;
 }
 
-static void set_status_details(grpc_call *call, status_source source,
-                               grpc_mdstr *status) {
-  if (call->status[source].details != NULL) {
-    GRPC_MDSTR_UNREF(call->status[source].details);
-  }
-  call->status[source].details = status;
-}
-
-static void get_final_status(grpc_call *call,
-                             void (*set_value)(grpc_status_code code,
-                                               void *user_data),
-                             void *set_value_user_data) {
-  int i;
-  for (i = 0; i < STATUS_SOURCE_COUNT; i++) {
-    if (call->status[i].is_set) {
-      set_value(call->status[i].code, set_value_user_data);
-      return;
-    }
-  }
-  if (call->is_client) {
-    set_value(GRPC_STATUS_UNKNOWN, set_value_user_data);
-  } else {
-    set_value(GRPC_STATUS_OK, set_value_user_data);
-  }
-}
-
 static void get_final_details(grpc_call *call, char **out_details,
                               size_t *out_details_capacity) {
   int i;
@@ -545,15 +569,28 @@ static grpc_linked_mdelem *linked_from_md(grpc_metadata *md) {
   return (grpc_linked_mdelem *)&md->internal_data;
 }
 
+static grpc_metadata *get_md_elem(grpc_metadata *metadata,
+                                  grpc_metadata *additional_metadata, int i,
+                                  int count) {
+  grpc_metadata *res =
+      i < count ? &metadata[i] : &additional_metadata[i - count];
+  GPR_ASSERT(res);
+  return res;
+}
+
 static int prepare_application_metadata(grpc_call *call, int count,
                                         grpc_metadata *metadata,
                                         int is_trailing,
-                                        int prepend_extra_metadata) {
+                                        int prepend_extra_metadata,
+                                        grpc_metadata *additional_metadata,
+                                        int additional_metadata_count) {
+  int total_count = count + additional_metadata_count;
   int i;
   grpc_metadata_batch *batch =
       &call->metadata_batch[0 /* is_receiving */][is_trailing];
-  for (i = 0; i < count; i++) {
-    grpc_metadata *md = &metadata[i];
+  for (i = 0; i < total_count; i++) {
+    const grpc_metadata *md =
+        get_md_elem(metadata, additional_metadata, i, count);
     grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
     GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data));
     l->md = grpc_mdelem_from_string_and_buffer(
@@ -572,9 +609,10 @@ static int prepare_application_metadata(grpc_call *call, int count,
       break;
     }
   }
-  if (i != count) {
+  if (i != total_count) {
     for (int j = 0; j <= i; j++) {
-      grpc_metadata *md = &metadata[j];
+      const grpc_metadata *md =
+          get_md_elem(metadata, additional_metadata, j, count);
       grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
       GRPC_MDELEM_UNREF(l->md);
     }
@@ -595,24 +633,36 @@ static int prepare_application_metadata(grpc_call *call, int count,
       }
     }
   }
-  for (i = 1; i < count; i++) {
-    linked_from_md(&metadata[i])->prev = linked_from_md(&metadata[i - 1]);
+  for (i = 1; i < total_count; i++) {
+    grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
+    grpc_metadata *prev_md =
+        get_md_elem(metadata, additional_metadata, i - 1, count);
+    linked_from_md(md)->prev = linked_from_md(prev_md);
   }
-  for (i = 0; i < count - 1; i++) {
-    linked_from_md(&metadata[i])->next = linked_from_md(&metadata[i + 1]);
+  for (i = 0; i < total_count - 1; i++) {
+    grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
+    grpc_metadata *next_md =
+        get_md_elem(metadata, additional_metadata, i + 1, count);
+    linked_from_md(md)->next = linked_from_md(next_md);
   }
-  switch (prepend_extra_metadata * 2 + (count != 0)) {
+
+  switch (prepend_extra_metadata * 2 + (total_count != 0)) {
     case 0:
       /* no prepend, no metadata => nothing to do */
       batch->list.head = batch->list.tail = NULL;
       break;
-    case 1:
+    case 1: {
       /* metadata, but no prepend */
-      batch->list.head = linked_from_md(&metadata[0]);
-      batch->list.tail = linked_from_md(&metadata[count - 1]);
+      grpc_metadata *first_md =
+          get_md_elem(metadata, additional_metadata, 0, count);
+      grpc_metadata *last_md =
+          get_md_elem(metadata, additional_metadata, total_count - 1, count);
+      batch->list.head = linked_from_md(first_md);
+      batch->list.tail = linked_from_md(last_md);
       batch->list.head->prev = NULL;
       batch->list.tail->next = NULL;
       break;
+    }
     case 2:
       /* prepend, but no md */
       batch->list.head = &call->send_extra_metadata[0];
@@ -621,17 +671,22 @@ static int prepare_application_metadata(grpc_call *call, int count,
       batch->list.head->prev = NULL;
       batch->list.tail->next = NULL;
       break;
-    case 3:
+    case 3: {
       /* prepend AND md */
+      grpc_metadata *first_md =
+          get_md_elem(metadata, additional_metadata, 0, count);
+      grpc_metadata *last_md =
+          get_md_elem(metadata, additional_metadata, total_count - 1, count);
       batch->list.head = &call->send_extra_metadata[0];
       call->send_extra_metadata[call->send_extra_metadata_count - 1].next =
-          linked_from_md(&metadata[0]);
-      linked_from_md(&metadata[0])->prev =
+          linked_from_md(first_md);
+      linked_from_md(first_md)->prev =
           &call->send_extra_metadata[call->send_extra_metadata_count - 1];
-      batch->list.tail = linked_from_md(&metadata[count - 1]);
+      batch->list.tail = linked_from_md(last_md);
       batch->list.head->prev = NULL;
       batch->list.tail->next = NULL;
       break;
+    }
     default:
       GPR_UNREACHABLE_CODE(return 0);
   }
@@ -700,48 +755,98 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
   return r;
 }
 
-typedef struct cancel_closure {
+typedef struct termination_closure {
   grpc_closure closure;
   grpc_call *call;
-  grpc_status_code status;
-} cancel_closure;
+  grpc_error *error;
+  grpc_closure *op_closure;
+  enum { TC_CANCEL, TC_CLOSE } type;
+} termination_closure;
+
+static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp,
+                             grpc_error *error) {
+  termination_closure *tc = tcp;
+  switch (tc->type) {
+    case TC_CANCEL:
+      GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "cancel");
+      break;
+    case TC_CLOSE:
+      GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "close");
+      break;
+  }
+  GRPC_ERROR_UNREF(tc->error);
+  grpc_exec_ctx_sched(exec_ctx, tc->op_closure, GRPC_ERROR_NONE, NULL);
+  gpr_free(tc);
+}
 
-static void done_cancel(grpc_exec_ctx *exec_ctx, void *ccp, bool success) {
-  cancel_closure *cc = ccp;
-  GRPC_CALL_INTERNAL_UNREF(exec_ctx, cc->call, "cancel");
-  gpr_free(cc);
+static void send_cancel(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
+  grpc_transport_stream_op op;
+  termination_closure *tc = tcp;
+  memset(&op, 0, sizeof(op));
+  op.cancel_error = tc->error;
+  /* reuse closure to catch completion */
+  grpc_closure_init(&tc->closure, done_termination, tc);
+  op.on_complete = &tc->closure;
+  execute_op(exec_ctx, tc->call, &op);
 }
 
-static void send_cancel(grpc_exec_ctx *exec_ctx, void *ccp, bool success) {
+static void send_close(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
   grpc_transport_stream_op op;
-  cancel_closure *cc = ccp;
+  termination_closure *tc = tcp;
   memset(&op, 0, sizeof(op));
-  op.cancel_with_status = cc->status;
+  op.close_error = tc->error;
   /* reuse closure to catch completion */
-  grpc_closure_init(&cc->closure, done_cancel, cc);
-  op.on_complete = &cc->closure;
-  execute_op(exec_ctx, cc->call, &op);
+  grpc_closure_init(&tc->closure, done_termination, tc);
+  tc->op_closure = op.on_complete;
+  op.on_complete = &tc->closure;
+  execute_op(exec_ctx, tc->call, &op);
+}
+
+static grpc_call_error terminate_with_status(grpc_exec_ctx *exec_ctx,
+                                             termination_closure *tc) {
+  set_status_from_error(tc->call, STATUS_FROM_API_OVERRIDE, tc->error);
+
+  if (tc->type == TC_CANCEL) {
+    grpc_closure_init(&tc->closure, send_cancel, tc);
+    GRPC_CALL_INTERNAL_REF(tc->call, "cancel");
+  } else if (tc->type == TC_CLOSE) {
+    grpc_closure_init(&tc->closure, send_close, tc);
+    GRPC_CALL_INTERNAL_REF(tc->call, "close");
+  }
+  grpc_exec_ctx_sched(exec_ctx, &tc->closure, GRPC_ERROR_NONE, NULL);
+  return GRPC_CALL_OK;
 }
 
 static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
                                           grpc_status_code status,
                                           const char *description) {
-  grpc_mdstr *details =
-      description ? grpc_mdstr_from_string(description) : NULL;
-  cancel_closure *cc = gpr_malloc(sizeof(*cc));
-
   GPR_ASSERT(status != GRPC_STATUS_OK);
+  termination_closure *tc = gpr_malloc(sizeof(*tc));
+  memset(tc, 0, sizeof(termination_closure));
+  tc->type = TC_CANCEL;
+  tc->call = c;
+  tc->error = grpc_error_set_int(
+      grpc_error_set_str(GRPC_ERROR_CREATE(description),
+                         GRPC_ERROR_STR_GRPC_MESSAGE, description),
+      GRPC_ERROR_INT_GRPC_STATUS, status);
+
+  return terminate_with_status(exec_ctx, tc);
+}
 
-  set_status_code(c, STATUS_FROM_API_OVERRIDE, (uint32_t)status);
-  set_status_details(c, STATUS_FROM_API_OVERRIDE, details);
-
-  grpc_closure_init(&cc->closure, send_cancel, cc);
-  cc->call = c;
-  cc->status = status;
-  GRPC_CALL_INTERNAL_REF(c, "cancel");
-  grpc_exec_ctx_enqueue(exec_ctx, &cc->closure, true, NULL);
-
-  return GRPC_CALL_OK;
+static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
+                                         grpc_status_code status,
+                                         const char *description) {
+  GPR_ASSERT(status != GRPC_STATUS_OK);
+  termination_closure *tc = gpr_malloc(sizeof(*tc));
+  memset(tc, 0, sizeof(termination_closure));
+  tc->type = TC_CLOSE;
+  tc->call = c;
+  tc->error = grpc_error_set_int(
+      grpc_error_set_str(GRPC_ERROR_CREATE(description),
+                         GRPC_ERROR_STR_GRPC_MESSAGE, description),
+      GRPC_ERROR_INT_GRPC_STATUS, status);
+
+  return terminate_with_status(exec_ctx, tc);
 }
 
 static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
@@ -775,11 +880,11 @@ grpc_call *grpc_call_from_top_element(grpc_call_element *elem) {
   return CALL_FROM_TOP_ELEM(elem);
 }
 
-static void call_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void call_alarm(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_call *call = arg;
   gpr_mu_lock(&call->mu);
   call->have_alarm = 0;
-  if (success) {
+  if (error != GRPC_ERROR_CANCELLED) {
     cancel_with_status(exec_ctx, call, GRPC_STATUS_DEADLINE_EXCEEDED,
                        "Deadline Exceeded");
   }
@@ -828,12 +933,16 @@ static uint32_t decode_status(grpc_mdelem *md) {
   return status;
 }
 
-static uint32_t decode_compression(grpc_mdelem *md) {
+static grpc_compression_algorithm decode_compression(grpc_mdelem *md) {
   grpc_compression_algorithm algorithm =
       grpc_compression_algorithm_from_mdstr(md->value);
   if (algorithm == GRPC_COMPRESS_ALGORITHMS_COUNT) {
     const char *md_c_str = grpc_mdstr_as_c_string(md->value);
-    gpr_log(GPR_ERROR, "Invalid compression algorithm: '%s'", md_c_str);
+    gpr_log(GPR_ERROR,
+            "Invalid incoming compression algorithm: '%s'. Interpreting "
+            "incoming data as uncompressed.",
+            md_c_str);
+    return GRPC_COMPRESS_NONE;
   }
   return algorithm;
 }
@@ -878,9 +987,9 @@ static grpc_mdelem *recv_initial_filter(void *callp, grpc_mdelem *elem) {
   if (elem == NULL) {
     return NULL;
   } else if (elem->key == GRPC_MDSTR_GRPC_ENCODING) {
-    GPR_TIMER_BEGIN("compression_algorithm", 0);
-    set_compression_algorithm(call, decode_compression(elem));
-    GPR_TIMER_END("compression_algorithm", 0);
+    GPR_TIMER_BEGIN("incoming_compression_algorithm", 0);
+    set_incoming_compression_algorithm(call, decode_compression(elem));
+    GPR_TIMER_END("incoming_compression_algorithm", 0);
     return NULL;
   } else if (elem->key == GRPC_MDSTR_GRPC_ACCEPT_ENCODING) {
     GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0);
@@ -961,7 +1070,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,
                                   batch_control *bctl) {
   grpc_call *call = bctl->call;
   if (bctl->is_notify_tag_closure) {
-    grpc_exec_ctx_enqueue(exec_ctx, bctl->notify_tag, bctl->success, NULL);
+    /* unrefs bctl->error */
+    grpc_exec_ctx_sched(exec_ctx, bctl->notify_tag, bctl->error, NULL);
     gpr_mu_lock(&call->mu);
     bctl->call->used_batches =
         (uint8_t)(bctl->call->used_batches &
@@ -969,7 +1079,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,
     gpr_mu_unlock(&call->mu);
     GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion");
   } else {
-    grpc_cq_end_op(exec_ctx, bctl->call->cq, bctl->notify_tag, bctl->success,
+    /* unrefs bctl->error */
+    grpc_cq_end_op(exec_ctx, bctl->call->cq, bctl->notify_tag, bctl->error,
                    finish_batch_completion, bctl, &bctl->cq_completion);
   }
 }
@@ -1001,15 +1112,18 @@ static void continue_receiving_slices(grpc_exec_ctx *exec_ctx,
 }
 
 static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
-                                  bool success) {
+                                  grpc_error *error) {
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
 
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer,
                          call->receiving_slice);
     continue_receiving_slices(exec_ctx, bctl);
   } else {
+    if (grpc_trace_operation_failures) {
+      GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
+    }
     grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
     call->receiving_stream = NULL;
     grpc_byte_buffer_destroy(*call->receiving_buffer);
@@ -1043,9 +1157,9 @@ static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl,
   } else {
     call->test_only_last_message_flags = call->receiving_stream->flags;
     if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) &&
-        (call->compression_algorithm > GRPC_COMPRESS_NONE)) {
+        (call->incoming_compression_algorithm > GRPC_COMPRESS_NONE)) {
       *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create(
-          NULL, 0, call->compression_algorithm);
+          NULL, 0, call->incoming_compression_algorithm);
     } else {
       *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0);
     }
@@ -1058,35 +1172,89 @@ static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl,
 }
 
 static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
-                                   bool success) {
+                                   grpc_error *error) {
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
 
   gpr_mu_lock(&bctl->call->mu);
-  if (bctl->call->has_initial_md_been_received || !success ||
+  if (bctl->call->has_initial_md_been_received || error != GRPC_ERROR_NONE ||
       call->receiving_stream == NULL) {
     gpr_mu_unlock(&bctl->call->mu);
-    process_data_after_md(exec_ctx, bctlp, success);
+    process_data_after_md(exec_ctx, bctlp, error);
   } else {
     call->saved_receiving_stream_ready_bctlp = bctlp;
     gpr_mu_unlock(&bctl->call->mu);
   }
 }
 
+static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
+                                       batch_control *bctl) {
+  grpc_call *call = bctl->call;
+  /* validate call->incoming_compression_algorithm */
+  if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) {
+    const grpc_compression_algorithm algo =
+        call->incoming_compression_algorithm;
+    char *error_msg = NULL;
+    const grpc_compression_options compression_options =
+        grpc_channel_compression_options(call->channel);
+    /* check if algorithm is known */
+    if (algo >= GRPC_COMPRESS_ALGORITHMS_COUNT) {
+      gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.",
+                   algo);
+      gpr_log(GPR_ERROR, "%s", error_msg);
+      close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg);
+    } else if (grpc_compression_options_is_algorithm_enabled(
+                   &compression_options, algo) == 0) {
+      /* check if algorithm is supported by current channel config */
+      char *algo_name;
+      grpc_compression_algorithm_name(algo, &algo_name);
+      gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.",
+                   algo_name);
+      gpr_log(GPR_ERROR, "%s", error_msg);
+      close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg);
+    } else {
+      call->incoming_compression_algorithm = algo;
+    }
+    gpr_free(error_msg);
+  }
+
+  /* make sure the received grpc-encoding is amongst the ones listed in
+   * grpc-accept-encoding */
+  GPR_ASSERT(call->encodings_accepted_by_peer != 0);
+  if (!GPR_BITGET(call->encodings_accepted_by_peer,
+                  call->incoming_compression_algorithm)) {
+    extern int grpc_compression_trace;
+    if (grpc_compression_trace) {
+      char *algo_name;
+      grpc_compression_algorithm_name(call->incoming_compression_algorithm,
+                                      &algo_name);
+      gpr_log(GPR_ERROR,
+              "Compression algorithm (grpc-encoding = '%s') not present in "
+              "the bitset of accepted encodings (grpc-accept-encodings: "
+              "'0x%x')",
+              algo_name, call->encodings_accepted_by_peer);
+    }
+  }
+}
+
 static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
-                                             void *bctlp, bool success) {
+                                             void *bctlp, grpc_error *error) {
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
 
   gpr_mu_lock(&call->mu);
 
-  if (!success) {
-    bctl->success = false;
+  if (error != GRPC_ERROR_NONE) {
+    bctl->error = GRPC_ERROR_REF(error);
   } else {
     grpc_metadata_batch *md =
         &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
     grpc_metadata_batch_filter(md, recv_initial_filter, call);
 
+    GPR_TIMER_BEGIN("validate_filtered_metadata", 0);
+    validate_filtered_metadata(exec_ctx, bctl);
+    GPR_TIMER_END("validate_filtered_metadata", 0);
+
     if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) !=
             0 &&
         !call->is_client) {
@@ -1101,7 +1269,7 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
     grpc_closure *saved_rsr_closure = grpc_closure_create(
         receiving_stream_ready, call->saved_receiving_stream_ready_bctlp);
     call->saved_receiving_stream_ready_bctlp = NULL;
-    grpc_exec_ctx_enqueue(exec_ctx, saved_rsr_closure, success, NULL);
+    grpc_exec_ctx_sched(exec_ctx, saved_rsr_closure, error, NULL);
   }
 
   gpr_mu_unlock(&call->mu);
@@ -1111,15 +1279,18 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
+static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp,
+                         grpc_error *error) {
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
   grpc_call *child_call;
   grpc_call *next_child_call;
 
+  GRPC_ERROR_REF(error);
+
   gpr_mu_lock(&call->mu);
   if (bctl->send_initial_metadata) {
-    if (!success) {
+    if (error != GRPC_ERROR_NONE) {
       set_status_code(call, STATUS_FROM_CORE, GRPC_STATUS_UNAVAILABLE);
     }
     grpc_metadata_batch_destroy(
@@ -1165,13 +1336,17 @@ static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
                        call->final_op.server.cancelled);
     }
 
-    success = 1;
+    GRPC_ERROR_UNREF(error);
+    error = GRPC_ERROR_NONE;
   }
-  bctl->success = success != 0;
+  GRPC_ERROR_UNREF(bctl->error);
+  bctl->error = GRPC_ERROR_REF(error);
   gpr_mu_unlock(&call->mu);
   if (gpr_unref(&bctl->steps_to_complete)) {
     post_batch_completion(exec_ctx, bctl);
   }
+
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
@@ -1201,7 +1376,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
 
   if (nops == 0) {
     GRPC_CALL_INTERNAL_REF(call, "completion");
-    bctl->success = 1;
+    bctl->error = GRPC_ERROR_NONE;
     if (!is_notify_tag_closure) {
       grpc_cq_begin_op(call->cq, notify_tag);
     }
@@ -1229,7 +1404,40 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
           goto done_with_error;
         }
-        if (op->data.send_initial_metadata.count > INT_MAX) {
+        /* process compression level */
+        grpc_metadata compression_md;
+        memset(&compression_md, 0, sizeof(grpc_metadata));
+        size_t additional_metadata_count = 0;
+        grpc_compression_level effective_compression_level;
+        bool level_set = false;
+        if (op->data.send_initial_metadata.maybe_compression_level.is_set) {
+          effective_compression_level =
+              op->data.send_initial_metadata.maybe_compression_level.level;
+          level_set = true;
+        } else {
+          const grpc_compression_options copts =
+              grpc_channel_compression_options(call->channel);
+          level_set = copts.default_level.is_set;
+          if (level_set) {
+            effective_compression_level = copts.default_level.level;
+          }
+        }
+        if (level_set && !call->is_client) {
+          const grpc_compression_algorithm calgo =
+              compression_algorithm_for_level_locked(
+                  call, effective_compression_level);
+          char *calgo_name;
+          grpc_compression_algorithm_name(calgo, &calgo_name);
+          // the following will be picked up by the compress filter and used as
+          // the call's compression algorithm.
+          compression_md.key = GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY;
+          compression_md.value = calgo_name;
+          compression_md.value_length = strlen(calgo_name);
+          additional_metadata_count++;
+        }
+
+        if (op->data.send_initial_metadata.count + additional_metadata_count >
+            INT_MAX) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1237,7 +1445,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         call->sent_initial_metadata = 1;
         if (!prepare_application_metadata(
                 call, (int)op->data.send_initial_metadata.count,
-                op->data.send_initial_metadata.metadata, 0, call->is_client)) {
+                op->data.send_initial_metadata.metadata, 0, call->is_client,
+                &compression_md, (int)additional_metadata_count)) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1325,7 +1534,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         if (!prepare_application_metadata(
                 call,
                 (int)op->data.send_status_from_server.trailing_metadata_count,
-                op->data.send_status_from_server.trailing_metadata, 1, 1)) {
+                op->data.send_status_from_server.trailing_metadata, 1, 1, NULL,
+                0)) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1397,7 +1607,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         bctl->recv_final_op = 1;
         stream_op.recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
-        stream_op.collect_stats = &call->stats;
+        stream_op.collect_stats = &call->stats.transport_stream_stats;
         break;
       case GRPC_OP_RECV_CLOSE_ON_SERVER:
         /* Flag validation: currently allow no flags */
@@ -1419,7 +1629,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         bctl->recv_final_op = 1;
         stream_op.recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
-        stream_op.collect_stats = &call->stats;
+        stream_op.collect_stats = &call->stats.transport_stream_stats;
         break;
     }
   }
@@ -1474,7 +1684,8 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
   grpc_call_error err;
 
   GRPC_API_TRACE(
-      "grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, reserved=%p)",
+      "grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, "
+      "reserved=%p)",
       5, (call, ops, (unsigned long)nops, tag, reserved));
 
   if (reserved != NULL) {
@@ -1513,9 +1724,10 @@ uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; }
 grpc_compression_algorithm grpc_call_compression_for_level(
     grpc_call *call, grpc_compression_level level) {
   gpr_mu_lock(&call->mu);
-  const uint32_t accepted_encodings = call->encodings_accepted_by_peer;
+  grpc_compression_algorithm algo =
+      compression_algorithm_for_level_locked(call, level);
   gpr_mu_unlock(&call->mu);
-  return grpc_compression_algorithm_for_level(level, accepted_encodings);
+  return algo;
 }
 
 const char *grpc_call_error_to_string(grpc_call_error error) {
diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h
index 2725e060b8..3a78fe3aa3 100644
--- a/src/core/lib/surface/call.h
+++ b/src/core/lib/surface/call.h
@@ -37,7 +37,6 @@
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/surface/api_trace.h"
-#include "src/core/lib/surface/surface_trace.h"
 
 #include <grpc/grpc.h>
 #include <grpc/impl/codegen/compression_types.h>
@@ -53,6 +52,8 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx,
 grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
                             uint32_t propagation_mask,
                             grpc_completion_queue *cq,
+                            /* if not NULL, it'll be used in lieu of \a cq */
+                            grpc_pollset_set *pollset_set_alternative,
                             const void *server_transport_data,
                             grpc_mdelem **add_initial_metadata,
                             size_t add_initial_metadata_count,
diff --git a/src/core/lib/surface/call_log_batch.c b/src/core/lib/surface/call_log_batch.c
index a6d1d5149f..31c074f15d 100644
--- a/src/core/lib/surface/call_log_batch.c
+++ b/src/core/lib/surface/call_log_batch.c
@@ -112,7 +112,7 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity,
   size_t i;
   for (i = 0; i < nops; i++) {
     tmp = grpc_op_string(&ops[i]);
-    gpr_log(file, line, severity, "ops[%d]: %s", i, tmp);
+    gpr_log(file, line, severity, "ops[%" PRIuPTR "]: %s", i, tmp);
     gpr_free(tmp);
   }
 }
diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c
index b6b760b5d8..2cf6d8890a 100644
--- a/src/core/lib/surface/channel.c
+++ b/src/core/lib/surface/channel.c
@@ -36,16 +36,17 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <grpc/compression.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel_init.h"
-#include "src/core/lib/surface/init.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 /** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS.
@@ -64,10 +65,12 @@ typedef struct registered_call {
 struct grpc_channel {
   int is_client;
   uint32_t max_message_length;
+  grpc_compression_options compression_options;
   grpc_mdelem *default_authority;
 
   gpr_mu registered_call_mu;
   registered_call *registered_calls;
+
   char *target;
 };
 
@@ -80,7 +83,8 @@ struct grpc_channel {
 /* the protobuf library will (by default) start warning at 100megs */
 #define DEFAULT_MAX_MESSAGE_LENGTH (100 * 1024 * 1024)
 
-static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg, bool success);
+static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error);
 
 grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
                                   const grpc_channel_args *input_args,
@@ -111,6 +115,7 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
   channel->registered_calls = NULL;
 
   channel->max_message_length = DEFAULT_MAX_MESSAGE_LENGTH;
+  grpc_compression_options_init(&channel->compression_options);
   if (args) {
     for (size_t i = 0; i < args->num_args; i++) {
       if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_MESSAGE_LENGTH)) {
@@ -151,6 +156,27 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
                 ":authority", args->args[i].value.string);
           }
         }
+      } else if (0 == strcmp(args->args[i].key,
+                             GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) {
+        channel->compression_options.default_level.is_set = true;
+        GPR_ASSERT(args->args[i].value.integer >= 0 &&
+                   args->args[i].value.integer < GRPC_COMPRESS_LEVEL_COUNT);
+        channel->compression_options.default_level.level =
+            (grpc_compression_level)args->args[i].value.integer;
+      } else if (0 == strcmp(args->args[i].key,
+                             GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) {
+        channel->compression_options.default_algorithm.is_set = true;
+        GPR_ASSERT(args->args[i].value.integer >= 0 &&
+                   args->args[i].value.integer <
+                       GRPC_COMPRESS_ALGORITHMS_COUNT);
+        channel->compression_options.default_algorithm.algorithm =
+            (grpc_compression_algorithm)args->args[i].value.integer;
+      } else if (0 ==
+                 strcmp(args->args[i].key,
+                        GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) {
+        channel->compression_options.enabled_algorithms_bitset =
+            (uint32_t)args->args[i].value.integer |
+            0x1; /* always support no compression */
       }
     }
     grpc_channel_args_destroy(args);
@@ -166,12 +192,14 @@ char *grpc_channel_get_target(grpc_channel *channel) {
 
 static grpc_call *grpc_channel_create_call_internal(
     grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
-    grpc_completion_queue *cq, grpc_mdelem *path_mdelem,
-    grpc_mdelem *authority_mdelem, gpr_timespec deadline) {
+    grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative,
+    grpc_mdelem *path_mdelem, grpc_mdelem *authority_mdelem,
+    gpr_timespec deadline) {
   grpc_mdelem *send_metadata[2];
   size_t num_metadata = 0;
 
   GPR_ASSERT(channel->is_client);
+  GPR_ASSERT(!(cq != NULL && pollset_set_alternative != NULL));
 
   send_metadata[num_metadata++] = path_mdelem;
   if (authority_mdelem != NULL) {
@@ -180,8 +208,9 @@ static grpc_call *grpc_channel_create_call_internal(
     send_metadata[num_metadata++] = GRPC_MDELEM_REF(channel->default_authority);
   }
 
-  return grpc_call_create(channel, parent_call, propagation_mask, cq, NULL,
-                          send_metadata, num_metadata, deadline);
+  return grpc_call_create(channel, parent_call, propagation_mask, cq,
+                          pollset_set_alternative, NULL, send_metadata,
+                          num_metadata, deadline);
 }
 
 grpc_call *grpc_channel_create_call(grpc_channel *channel,
@@ -194,14 +223,30 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel,
       "grpc_channel_create_call("
       "channel=%p, parent_call=%p, propagation_mask=%x, cq=%p, method=%s, "
       "host=%s, "
-      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %" PRId64
+      ", tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
-      10, (channel, parent_call, (unsigned)propagation_mask, cq, method, host,
-           (long long)deadline.tv_sec, (int)deadline.tv_nsec,
-           (int)deadline.clock_type, reserved));
+      10,
+      (channel, parent_call, (unsigned)propagation_mask, cq, method, host,
+       deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
-      channel, parent_call, propagation_mask, cq,
+      channel, parent_call, propagation_mask, cq, NULL,
+      grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH,
+                                        grpc_mdstr_from_string(method)),
+      host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY,
+                                               grpc_mdstr_from_string(host))
+           : NULL,
+      deadline);
+}
+
+grpc_call *grpc_channel_create_pollset_set_call(
+    grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+    grpc_pollset_set *pollset_set, const char *method, const char *host,
+    gpr_timespec deadline, void *reserved) {
+  GPR_ASSERT(!reserved);
+  return grpc_channel_create_call_internal(
+      channel, parent_call, propagation_mask, NULL, pollset_set,
       grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH,
                                         grpc_mdstr_from_string(method)),
       host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY,
@@ -238,14 +283,15 @@ grpc_call *grpc_channel_create_registered_call(
       "grpc_channel_create_registered_call("
       "channel=%p, parent_call=%p, propagation_mask=%x, completion_queue=%p, "
       "registered_call_handle=%p, "
-      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %" PRId64
+      ", tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
       9, (channel, parent_call, (unsigned)propagation_mask, completion_queue,
-          registered_call_handle, (long long)deadline.tv_sec,
-          (int)deadline.tv_nsec, (int)deadline.clock_type, reserved));
+          registered_call_handle, deadline.tv_sec, deadline.tv_nsec,
+          (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
-      channel, parent_call, propagation_mask, completion_queue,
+      channel, parent_call, propagation_mask, completion_queue, NULL,
       GRPC_MDELEM_REF(rc->path),
       rc->authority ? GRPC_MDELEM_REF(rc->authority) : NULL, deadline);
 }
@@ -267,7 +313,7 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
 }
 
 static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg,
-                            bool iomgr_success) {
+                            grpc_error *error) {
   grpc_channel *channel = arg;
   grpc_channel_stack_destroy(exec_ctx, CHANNEL_STACK_FROM_CHANNEL(channel));
   while (channel->registered_calls) {
@@ -293,7 +339,7 @@ void grpc_channel_destroy(grpc_channel *channel) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   GRPC_API_TRACE("grpc_channel_destroy(channel=%p)", 1, (channel));
   memset(&op, 0, sizeof(op));
-  op.disconnect = 1;
+  op.disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed");
   elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0);
   elem->filter->start_transport_op(&exec_ctx, elem, &op);
 
@@ -306,6 +352,11 @@ grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel) {
   return CHANNEL_STACK_FROM_CHANNEL(channel);
 }
 
+grpc_compression_options grpc_channel_compression_options(
+    const grpc_channel *channel) {
+  return channel->compression_options;
+}
+
 grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) {
   char tmp[GPR_LTOA_MIN_BUFSIZE];
   switch (i) {
diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h
index 22dae930e4..7eff7b8883 100644
--- a/src/core/lib/surface/channel.h
+++ b/src/core/lib/surface/channel.h
@@ -42,6 +42,11 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
                                   grpc_channel_stack_type channel_stack_type,
                                   grpc_transport *optional_transport);
 
+grpc_call *grpc_channel_create_pollset_set_call(
+    grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+    grpc_pollset_set *pollset_set, const char *method, const char *host,
+    gpr_timespec deadline, void *reserved);
+
 /** Get a (borrowed) pointer to this channels underlying channel stack */
 grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel);
 
@@ -71,4 +76,8 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
   grpc_channel_internal_unref(exec_ctx, channel)
 #endif
 
+/** Return the channel's compression options. */
+grpc_compression_options grpc_channel_compression_options(
+    const grpc_channel *channel);
+
 #endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_H */
diff --git a/src/core/lib/surface/channel_ping.c b/src/core/lib/surface/channel_ping.c
index 5a50698695..9818f9d2f2 100644
--- a/src/core/lib/surface/channel_ping.c
+++ b/src/core/lib/surface/channel_ping.c
@@ -53,10 +53,10 @@ static void ping_destroy(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_free(arg);
 }
 
-static void ping_done(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void ping_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   ping_result *pr = arg;
-  grpc_cq_end_op(exec_ctx, pr->cq, pr->tag, success, ping_destroy, pr,
-                 &pr->completion_storage);
+  grpc_cq_end_op(exec_ctx, pr->cq, pr->tag, GRPC_ERROR_REF(error), ping_destroy,
+                 pr, &pr->completion_storage);
 }
 
 void grpc_channel_ping(grpc_channel *channel, grpc_completion_queue *cq,
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index 1f82c3bad2..5978884db8 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -48,7 +48,8 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/event_string.h"
-#include "src/core/lib/surface/surface_trace.h"
+
+int grpc_trace_operation_failures;
 
 typedef struct {
   grpc_pollset_worker **worker;
@@ -70,6 +71,8 @@ struct grpc_completion_queue {
   int shutdown;
   int shutdown_called;
   int is_server_cq;
+  /** Can the server cq accept incoming channels */
+  int is_non_listening_server_cq;
   int num_pluckers;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
   grpc_closure pollset_shutdown_done;
@@ -84,12 +87,24 @@ struct grpc_completion_queue {
 };
 
 #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
+#define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
 
 static gpr_mu g_freelist_mu;
 static grpc_completion_queue *g_freelist;
 
+int grpc_cq_pluck_trace;
+int grpc_cq_event_timeout_trace;
+
+#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)                  \
+  if (grpc_api_trace &&                                               \
+      (grpc_cq_pluck_trace || (event)->type != GRPC_QUEUE_TIMEOUT)) { \
+    char *_ev = grpc_event_string(event);                             \
+    gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev);               \
+    gpr_free(_ev);                                                    \
+  }
+
 static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc,
-                                     bool success);
+                                     grpc_error *error);
 
 void grpc_cq_global_init(void) { gpr_mu_init(&g_freelist_mu); }
 
@@ -149,6 +164,7 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
   cc->shutdown = 0;
   cc->shutdown_called = 0;
   cc->is_server_cq = 0;
+  cc->is_non_listening_server_cq = 0;
   cc->num_pluckers = 0;
 #ifndef NDEBUG
   cc->outstanding_tag_count = 0;
@@ -172,7 +188,7 @@ void grpc_cq_internal_ref(grpc_completion_queue *cc) {
 }
 
 static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg,
-                                     bool success) {
+                                     grpc_error *error) {
   grpc_completion_queue *cc = arg;
   GRPC_CQ_INTERNAL_UNREF(cc, "pollset_destroy");
 }
@@ -215,7 +231,7 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) {
    event, then enter shutdown mode */
 /* Queue a GRPC_OP_COMPLETED operation */
 void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
-                    void *tag, int success,
+                    void *tag, grpc_error *error,
                     void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
                                  grpc_cq_completion *storage),
                     void *done_arg, grpc_cq_completion *storage) {
@@ -227,16 +243,24 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
 #endif
 
   GPR_TIMER_BEGIN("grpc_cq_end_op", 0);
-  GRPC_API_TRACE(
-      "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, success=%d, done=%p, "
-      "done_arg=%p, storage=%p)",
-      7, (exec_ctx, cc, tag, success, done, done_arg, storage));
+  if (grpc_api_trace ||
+      (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) {
+    const char *errmsg = grpc_error_string(error);
+    GRPC_API_TRACE(
+        "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, "
+        "done_arg=%p, storage=%p)",
+        7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage));
+    if (grpc_trace_operation_failures && error != GRPC_ERROR_NONE) {
+      gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
+    }
+    grpc_error_free_string(errmsg);
+  }
 
   storage->tag = tag;
   storage->done = done;
   storage->done_arg = done_arg;
-  storage->next =
-      ((uintptr_t)&cc->completed_head) | ((uintptr_t)(success != 0));
+  storage->next = ((uintptr_t)&cc->completed_head) |
+                  ((uintptr_t)(error == GRPC_ERROR_NONE));
 
   gpr_mu_lock(cc->mu);
 #ifndef NDEBUG
@@ -263,8 +287,15 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
         break;
       }
     }
-    grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker);
+    grpc_error *kick_error =
+        grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker);
     gpr_mu_unlock(cc->mu);
+    if (kick_error != GRPC_ERROR_NONE) {
+      const char *msg = grpc_error_string(kick_error);
+      gpr_log(GPR_ERROR, "Kick failed: %s", msg);
+      grpc_error_free_string(msg);
+      GRPC_ERROR_UNREF(kick_error);
+    }
   } else {
     cc->completed_tail->next =
         ((uintptr_t)storage) | (1u & (uintptr_t)cc->completed_tail->next);
@@ -278,6 +309,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
   }
 
   GPR_TIMER_END("grpc_cq_end_op", 0);
+
+  GRPC_ERROR_UNREF(error);
 }
 
 grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
@@ -293,10 +326,11 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
   GRPC_API_TRACE(
       "grpc_completion_queue_next("
       "cc=%p, "
-      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "deadline=gpr_timespec { tv_sec: %" PRId64
+      ", tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
-      5, (cc, (long long)deadline.tv_sec, (int)deadline.tv_nsec,
-          (int)deadline.clock_type, reserved));
+      5, (cc, deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
+          reserved));
   GPR_ASSERT(!reserved);
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
@@ -343,8 +377,18 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
       gpr_mu_lock(cc->mu);
       continue;
     } else {
-      grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now,
-                        iteration_deadline);
+      grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc),
+                                          &worker, now, iteration_deadline);
+      if (err != GRPC_ERROR_NONE) {
+        gpr_mu_unlock(cc->mu);
+        const char *msg = grpc_error_string(err);
+        gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
+        grpc_error_free_string(msg);
+        GRPC_ERROR_UNREF(err);
+        memset(&ret, 0, sizeof(ret));
+        ret.type = GRPC_QUEUE_TIMEOUT;
+        break;
+      }
     }
   }
   GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret);
@@ -392,13 +436,16 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
 
   GPR_TIMER_BEGIN("grpc_completion_queue_pluck", 0);
 
-  GRPC_API_TRACE(
-      "grpc_completion_queue_pluck("
-      "cc=%p, tag=%p, "
-      "deadline=gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
-      "reserved=%p)",
-      6, (cc, tag, (long long)deadline.tv_sec, (int)deadline.tv_nsec,
-          (int)deadline.clock_type, reserved));
+  if (grpc_cq_pluck_trace) {
+    GRPC_API_TRACE(
+        "grpc_completion_queue_pluck("
+        "cc=%p, tag=%p, "
+        "deadline=gpr_timespec { tv_sec: %" PRId64
+        ", tv_nsec: %d, clock_type: %d }, "
+        "reserved=%p)",
+        6, (cc, tag, deadline.tv_sec, deadline.tv_nsec,
+            (int)deadline.clock_type, reserved));
+  }
   GPR_ASSERT(!reserved);
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
@@ -460,8 +507,19 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
       grpc_exec_ctx_flush(&exec_ctx);
       gpr_mu_lock(cc->mu);
     } else {
-      grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now,
-                        iteration_deadline);
+      grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc),
+                                          &worker, now, iteration_deadline);
+      if (err != GRPC_ERROR_NONE) {
+        del_plucker(cc, tag, &worker);
+        gpr_mu_unlock(cc->mu);
+        const char *msg = grpc_error_string(err);
+        gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
+        grpc_error_free_string(msg);
+        GRPC_ERROR_UNREF(err);
+        memset(&ret, 0, sizeof(ret));
+        ret.type = GRPC_QUEUE_TIMEOUT;
+        break;
+      }
     }
     del_plucker(cc, tag, &worker);
   }
@@ -511,6 +569,18 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
   return POLLSET_FROM_CQ(cc);
 }
 
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) {
+  return CQ_FROM_POLLSET(ps);
+}
+
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc) {
+  cc->is_non_listening_server_cq = 1;
+}
+
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) {
+  return (cc->is_non_listening_server_cq == 1);
+}
+
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; }
 
 int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; }
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index eef82cf014..3049284f68 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,6 +39,12 @@
 #include <grpc/grpc.h>
 #include "src/core/lib/iomgr/pollset.h"
 
+/* These trace flags default to 1. The corresponding lines are only traced
+   if grpc_api_trace is also truthy */
+extern int grpc_cq_pluck_trace;
+extern int grpc_cq_event_timeout_trace;
+extern int grpc_trace_operation_failures;
+
 typedef struct grpc_cq_completion {
   /** user supplied tag */
   void *tag;
@@ -75,13 +81,16 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag);
 /* Queue a GRPC_OP_COMPLETED operation; tag must correspond to the tag passed to
    grpc_cq_begin_op */
 void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
-                    void *tag, int success,
+                    void *tag, grpc_error *error,
                     void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
                                  grpc_cq_completion *storage),
                     void *done_arg, grpc_cq_completion *storage);
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps);
 
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc);
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc);
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
 int grpc_cq_is_server_cq(grpc_completion_queue *cc);
 
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
index 57c6897626..5397913a21 100644
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@@ -57,7 +57,6 @@
 #include "src/core/lib/surface/init.h"
 #include "src/core/lib/surface/lame_client.h"
 #include "src/core/lib/surface/server.h"
-#include "src/core/lib/surface/surface_trace.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/transport_impl.h"
 
@@ -164,7 +163,14 @@ void grpc_init(void) {
     grpc_register_tracer("channel_stack_builder",
                          &grpc_trace_channel_stack_builder);
     grpc_register_tracer("http1", &grpc_http1_trace);
-    grpc_register_tracer("compression", &grpc_compress_filter_trace);
+    grpc_register_tracer("compression", &grpc_compression_trace);
+    grpc_register_tracer("queue_pluck", &grpc_cq_pluck_trace);
+    // Default pluck trace to 1
+    grpc_cq_pluck_trace = 1;
+    grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace);
+    // Default timeout trace to 1
+    grpc_cq_event_timeout_trace = 1;
+    grpc_register_tracer("op_failure", &grpc_trace_operation_failures);
     grpc_security_pre_init();
     grpc_iomgr_init();
     grpc_executor_init();
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 3fda2c9e1e..7ee7b51568 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -37,10 +37,10 @@
 #include <string.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
 
diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.c
index f50ec54cea..5ea4cba5d1 100644
--- a/src/core/lib/surface/lame_client.c
+++ b/src/core/lib/surface/lame_client.c
@@ -80,7 +80,8 @@ static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
   } else if (op->recv_trailing_metadata != NULL) {
     fill_metadata(elem, op->recv_trailing_metadata);
   }
-  grpc_transport_stream_op_finish_with_failure(exec_ctx, op);
+  grpc_transport_stream_op_finish_with_failure(
+      exec_ctx, op, GRPC_ERROR_CREATE("lame client channel"));
 }
 
 static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
@@ -91,23 +92,26 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
                                     grpc_channel_element *elem,
                                     grpc_transport_op *op) {
   if (op->on_connectivity_state_change) {
-    GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE);
-    *op->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE;
-    op->on_connectivity_state_change->cb(
-        exec_ctx, op->on_connectivity_state_change->cb_arg, 1);
+    GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_SHUTDOWN);
+    *op->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+    grpc_exec_ctx_sched(exec_ctx, op->on_connectivity_state_change,
+                        GRPC_ERROR_NONE, NULL);
   }
   if (op->on_consumed != NULL) {
-    op->on_consumed->cb(exec_ctx, op->on_consumed->cb_arg, 1);
+    grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
   }
   if (op->send_ping != NULL) {
-    op->send_ping->cb(exec_ctx, op->send_ping->cb_arg, 0);
+    grpc_exec_ctx_sched(exec_ctx, op->send_ping,
+                        GRPC_ERROR_CREATE("lame client channel"), NULL);
   }
+  GRPC_ERROR_UNREF(op->disconnect_with_error);
 }
 
 static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                            grpc_call_element_args *args) {}
 
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                              const grpc_call_stats *stats,
                               void *and_free_memory) {
   gpr_free(and_free_memory);
 }
@@ -127,7 +131,7 @@ const grpc_channel_filter grpc_lame_filter = {
     lame_start_transport_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index 2db95b9cdf..def6e5068b 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,11 +69,6 @@ typedef struct call_data call_data;
 typedef struct channel_data channel_data;
 typedef struct registered_method registered_method;
 
-typedef struct {
-  call_data *next;
-  call_data *prev;
-} call_link;
-
 typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
 
 typedef struct requested_call {
@@ -81,7 +76,6 @@ typedef struct requested_call {
   void *tag;
   grpc_server *server;
   grpc_completion_queue *cq_bound_to_call;
-  grpc_completion_queue *cq_for_notification;
   grpc_call **call;
   grpc_cq_completion completion;
   grpc_metadata_array *initial_metadata;
@@ -108,6 +102,7 @@ struct channel_data {
   grpc_server *server;
   grpc_connectivity_state connectivity_state;
   grpc_channel *channel;
+  size_t cq_idx;
   /* linked list of all channels on a server */
   channel_data *next;
   channel_data *prev;
@@ -172,7 +167,7 @@ struct request_matcher {
   grpc_server *server;
   call_data *pending_head;
   call_data *pending_tail;
-  gpr_stack_lockfree *requests;
+  gpr_stack_lockfree **requests_per_cq;
 };
 
 struct registered_method {
@@ -180,6 +175,7 @@ struct registered_method {
   char *host;
   grpc_server_register_method_payload_handling payload_handling;
   uint32_t flags;
+  /* one request matcher per method */
   request_matcher request_matcher;
   registered_method *next;
 };
@@ -195,6 +191,7 @@ struct grpc_server {
   grpc_completion_queue **cqs;
   grpc_pollset **pollsets;
   size_t cq_count;
+  bool started;
 
   /* The two following mutexes control access to server-state
      mu_global controls access to non-call-related state (e.g., channel state)
@@ -207,6 +204,7 @@ struct grpc_server {
   gpr_mu mu_call;   /* mutex for call-specific state */
 
   registered_method *registered_methods;
+  /** one request matcher for unregistered methods */
   request_matcher unregistered_request_matcher;
   /** free list of available requested_calls indices */
   gpr_stack_lockfree *request_freelist;
@@ -232,9 +230,10 @@ struct grpc_server {
 #define SERVER_FROM_CALL_ELEM(elem) \
   (((channel_data *)(elem)->channel_data)->server)
 
-static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld, bool success);
+static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld,
+                            grpc_error *error);
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc);
+                      size_t cq_idx, requested_call *rc, grpc_error *error);
 /* Before calling maybe_finish_shutdown, we must hold mu_global and not
    hold mu_call */
 static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_server *server);
@@ -265,14 +264,14 @@ struct shutdown_cleanup_args {
 };
 
 static void shutdown_cleanup(grpc_exec_ctx *exec_ctx, void *arg,
-                             bool iomgr_status_ignored) {
+                             grpc_error *error) {
   struct shutdown_cleanup_args *a = arg;
   gpr_slice_unref(a->slice);
   gpr_free(a);
 }
 
 static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
-                          int send_goaway, int send_disconnect) {
+                          int send_goaway, grpc_error *send_disconnect) {
   grpc_transport_op op;
   struct shutdown_cleanup_args *sc;
   grpc_channel_element *elem;
@@ -283,7 +282,7 @@ static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
   sc->slice = gpr_slice_from_copied_string("Server shutdown");
   op.goaway_message = &sc->slice;
   op.goaway_status = GRPC_STATUS_OK;
-  op.disconnect = send_disconnect;
+  op.disconnect_with_error = send_disconnect;
   grpc_closure_init(&sc->closure, shutdown_cleanup, sc);
   op.on_consumed = &sc->closure;
 
@@ -294,14 +293,16 @@ static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
 static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx,
                                          channel_broadcaster *cb,
                                          int send_goaway,
-                                         int force_disconnect) {
+                                         grpc_error *force_disconnect) {
   size_t i;
 
   for (i = 0; i < cb->num_channels; i++) {
-    send_shutdown(exec_ctx, cb->channels[i], send_goaway, force_disconnect);
+    send_shutdown(exec_ctx, cb->channels[i], send_goaway,
+                  GRPC_ERROR_REF(force_disconnect));
     GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, cb->channels[i], "broadcast");
   }
   gpr_free(cb->channels);
+  GRPC_ERROR_UNREF(force_disconnect);
 }
 
 /*
@@ -312,15 +313,23 @@ static void request_matcher_init(request_matcher *rm, size_t entries,
                                  grpc_server *server) {
   memset(rm, 0, sizeof(*rm));
   rm->server = server;
-  rm->requests = gpr_stack_lockfree_create(entries);
+  rm->requests_per_cq =
+      gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    rm->requests_per_cq[i] = gpr_stack_lockfree_create(entries);
+  }
 }
 
 static void request_matcher_destroy(request_matcher *rm) {
-  GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests) == -1);
-  gpr_stack_lockfree_destroy(rm->requests);
+  for (size_t i = 0; i < rm->server->cq_count; i++) {
+    GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1);
+    gpr_stack_lockfree_destroy(rm->requests_per_cq[i]);
+  }
+  gpr_free(rm->requests_per_cq);
 }
 
-static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, bool success) {
+static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem,
+                        grpc_error *error) {
   grpc_call_destroy(grpc_call_from_top_element(elem));
 }
 
@@ -335,17 +344,24 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx,
     grpc_closure_init(
         &calld->kill_zombie_closure, kill_zombie,
         grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
-    grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
+                        NULL);
   }
 }
 
 static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx,
                                           grpc_server *server,
-                                          request_matcher *rm) {
+                                          request_matcher *rm,
+                                          grpc_error *error) {
   int request_id;
-  while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
-    fail_call(exec_ctx, server, &server->requested_calls[request_id]);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    while ((request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[i])) !=
+           -1) {
+      fail_call(exec_ctx, server, i, &server->requested_calls[request_id],
+                GRPC_ERROR_REF(error));
+    }
   }
+  GRPC_ERROR_UNREF(error);
 }
 
 /*
@@ -364,15 +380,19 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
   gpr_mu_destroy(&server->mu_call);
   while ((rm = server->registered_methods) != NULL) {
     server->registered_methods = rm->next;
-    request_matcher_destroy(&rm->request_matcher);
+    if (server->started) {
+      request_matcher_destroy(&rm->request_matcher);
+    }
     gpr_free(rm->method);
     gpr_free(rm->host);
     gpr_free(rm);
   }
+  if (server->started) {
+    request_matcher_destroy(&server->unregistered_request_matcher);
+  }
   for (i = 0; i < server->cq_count; i++) {
     GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server");
   }
-  request_matcher_destroy(&server->unregistered_request_matcher);
   gpr_stack_lockfree_destroy(server->request_freelist);
   gpr_free(server->cqs);
   gpr_free(server->pollsets);
@@ -398,7 +418,7 @@ static void orphan_channel(channel_data *chand) {
 }
 
 static void finish_destroy_channel(grpc_exec_ctx *exec_ctx, void *cd,
-                                   bool success) {
+                                   grpc_error *error) {
   channel_data *chand = cd;
   grpc_server *server = chand->server;
   GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, chand->channel, "server");
@@ -453,11 +473,11 @@ static void done_request_event(grpc_exec_ctx *exec_ctx, void *req,
 }
 
 static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                         call_data *calld, requested_call *rc) {
+                         call_data *calld, size_t cq_idx, requested_call *rc) {
   grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
   grpc_call *call = calld->call;
   *rc->call = call;
-  calld->cq_new = rc->cq_for_notification;
+  calld->cq_new = server->cqs[cq_idx];
   GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
   switch (rc->type) {
     case BATCH_CALL:
@@ -487,46 +507,57 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
       grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
   channel_data *chand = elem->channel_data;
   server_ref(chand->server);
-  grpc_cq_end_op(exec_ctx, calld->cq_new, rc->tag, true, done_request_event, rc,
-                 &rc->completion);
+  grpc_cq_end_op(exec_ctx, calld->cq_new, rc->tag, GRPC_ERROR_NONE,
+                 done_request_event, rc, &rc->completion);
 }
 
-static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
-  call_data *calld = arg;
+static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  grpc_call_element *call_elem = arg;
+  call_data *calld = call_elem->call_data;
+  channel_data *chand = call_elem->channel_data;
   request_matcher *rm = calld->request_matcher;
   grpc_server *server = rm->server;
 
-  if (!success || gpr_atm_acq_load(&server->shutdown_flag)) {
+  if (error != GRPC_ERROR_NONE || gpr_atm_acq_load(&server->shutdown_flag)) {
     gpr_mu_lock(&calld->mu_state);
     calld->state = ZOMBIED;
     gpr_mu_unlock(&calld->mu_state);
     grpc_closure_init(
         &calld->kill_zombie_closure, kill_zombie,
         grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
-    grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, error, NULL);
     return;
   }
 
-  int request_id = gpr_stack_lockfree_pop(rm->requests);
-  if (request_id == -1) {
-    gpr_mu_lock(&server->mu_call);
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = PENDING;
-    gpr_mu_unlock(&calld->mu_state);
-    if (rm->pending_head == NULL) {
-      rm->pending_tail = rm->pending_head = calld;
+  for (size_t i = 0; i < server->cq_count; i++) {
+    size_t cq_idx = (chand->cq_idx + i) % server->cq_count;
+    int request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+    if (request_id == -1) {
+      continue;
     } else {
-      rm->pending_tail->pending_next = calld;
-      rm->pending_tail = calld;
+      gpr_mu_lock(&calld->mu_state);
+      calld->state = ACTIVATED;
+      gpr_mu_unlock(&calld->mu_state);
+      publish_call(exec_ctx, server, calld, cq_idx,
+                   &server->requested_calls[request_id]);
+      return; /* early out */
     }
-    calld->pending_next = NULL;
-    gpr_mu_unlock(&server->mu_call);
+  }
+
+  /* no cq to take the request found: queue it on the slow list */
+  gpr_mu_lock(&server->mu_call);
+  gpr_mu_lock(&calld->mu_state);
+  calld->state = PENDING;
+  gpr_mu_unlock(&calld->mu_state);
+  if (rm->pending_head == NULL) {
+    rm->pending_tail = rm->pending_head = calld;
   } else {
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = ACTIVATED;
-    gpr_mu_unlock(&calld->mu_state);
-    publish_call(exec_ctx, server, calld, &server->requested_calls[request_id]);
+    rm->pending_tail->pending_next = calld;
+    rm->pending_tail = calld;
   }
+  calld->pending_next = NULL;
+  gpr_mu_unlock(&server->mu_call);
 }
 
 static void finish_start_new_rpc(
@@ -540,7 +571,8 @@ static void finish_start_new_rpc(
     calld->state = ZOMBIED;
     gpr_mu_unlock(&calld->mu_state);
     grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
-    grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
+                        NULL);
     return;
   }
 
@@ -548,14 +580,14 @@ static void finish_start_new_rpc(
 
   switch (payload_handling) {
     case GRPC_SRM_PAYLOAD_NONE:
-      publish_new_rpc(exec_ctx, calld, true);
+      publish_new_rpc(exec_ctx, elem, GRPC_ERROR_NONE);
       break;
     case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: {
       grpc_op op;
       memset(&op, 0, sizeof(op));
       op.op = GRPC_OP_RECV_MESSAGE;
       op.data.recv_message = &calld->payload;
-      grpc_closure_init(&calld->publish, publish_new_rpc, calld);
+      grpc_closure_init(&calld->publish, publish_new_rpc, elem);
       grpc_call_start_batch_and_execute(exec_ctx, calld->call, &op, 1,
                                         &calld->publish);
       break;
@@ -636,16 +668,21 @@ static int num_channels(grpc_server *server) {
 }
 
 static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx,
-                                     grpc_server *server) {
-  registered_method *rm;
-  request_matcher_kill_requests(exec_ctx, server,
-                                &server->unregistered_request_matcher);
-  request_matcher_zombify_all_pending_calls(
-      exec_ctx, &server->unregistered_request_matcher);
-  for (rm = server->registered_methods; rm; rm = rm->next) {
-    request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
-    request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+                                     grpc_server *server, grpc_error *error) {
+  if (server->started) {
+    request_matcher_kill_requests(exec_ctx, server,
+                                  &server->unregistered_request_matcher,
+                                  GRPC_ERROR_REF(error));
+    request_matcher_zombify_all_pending_calls(
+        exec_ctx, &server->unregistered_request_matcher);
+    for (registered_method *rm = server->registered_methods; rm;
+         rm = rm->next) {
+      request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher,
+                                    GRPC_ERROR_REF(error));
+      request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+    }
   }
+  GRPC_ERROR_UNREF(error);
 }
 
 static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
@@ -655,7 +692,8 @@ static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
     return;
   }
 
-  kill_pending_work_locked(exec_ctx, server);
+  kill_pending_work_locked(exec_ctx, server,
+                           GRPC_ERROR_CREATE("Server Shutdown"));
 
   if (server->root_channel_data.next != &server->root_channel_data ||
       server->listeners_destroyed < num_listeners(server)) {
@@ -676,7 +714,8 @@ static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
   for (i = 0; i < server->num_shutdown_tags; i++) {
     server_ref(server);
     grpc_cq_end_op(exec_ctx, server->shutdown_tags[i].cq,
-                   server->shutdown_tags[i].tag, 1, done_shutdown_event, server,
+                   server->shutdown_tags[i].tag, GRPC_ERROR_NONE,
+                   done_shutdown_event, server,
                    &server->shutdown_tags[i].completion);
   }
 }
@@ -699,11 +738,12 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
 }
 
 static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
-                                            bool success) {
+                                            grpc_error *error) {
   grpc_call_element *elem = ptr;
   call_data *calld = elem->call_data;
   gpr_timespec op_deadline;
 
+  GRPC_ERROR_REF(error);
   grpc_metadata_batch_filter(calld->recv_initial_metadata, server_filter, elem);
   op_deadline = calld->recv_initial_metadata->deadline;
   if (0 != gpr_time_cmp(op_deadline, gpr_inf_future(op_deadline.clock_type))) {
@@ -712,11 +752,13 @@ static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
   if (calld->host && calld->path) {
     /* do nothing */
   } else {
-    success = 0;
+    GRPC_ERROR_UNREF(error);
+    error =
+        GRPC_ERROR_CREATE_REFERENCING("Missing :authority or :path", &error, 1);
   }
 
-  calld->on_done_recv_initial_metadata->cb(
-      exec_ctx, calld->on_done_recv_initial_metadata->cb_arg, success);
+  grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv_initial_metadata, error,
+                      NULL);
 }
 
 static void server_mutate_op(grpc_call_element *elem,
@@ -741,10 +783,10 @@ static void server_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
 }
 
 static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
-                                 bool success) {
+                                 grpc_error *error) {
   grpc_call_element *elem = ptr;
   call_data *calld = elem->call_data;
-  if (success) {
+  if (error == GRPC_ERROR_NONE) {
     start_new_rpc(exec_ctx, elem);
   } else {
     gpr_mu_lock(&calld->mu_state);
@@ -752,7 +794,8 @@ static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
       calld->state = ZOMBIED;
       gpr_mu_unlock(&calld->mu_state);
       grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
-      grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+      grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure,
+                          GRPC_ERROR_NONE, NULL);
     } else if (calld->state == PENDING) {
       calld->state = ZOMBIED;
       gpr_mu_unlock(&calld->mu_state);
@@ -769,9 +812,9 @@ static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd,
                           const void *transport_server_data) {
   channel_data *chand = cd;
   /* create a call */
-  grpc_call *call =
-      grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data,
-                       NULL, 0, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+  grpc_call *call = grpc_call_create(chand->channel, NULL, 0, NULL, NULL,
+                                     transport_server_data, NULL, 0,
+                                     gpr_inf_future(GPR_CLOCK_MONOTONIC));
   grpc_call_element *elem =
       grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
   call_data *calld = elem->call_data;
@@ -785,10 +828,10 @@ static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd,
 }
 
 static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd,
-                                         bool iomgr_status_ignored) {
+                                         grpc_error *error) {
   channel_data *chand = cd;
   grpc_server *server = chand->server;
-  if (chand->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) {
+  if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
     grpc_transport_op op;
     memset(&op, 0, sizeof(op));
     op.on_connectivity_state_change = &chand->channel_connectivity_changed,
@@ -821,7 +864,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                              void *ignored) {
+                              const grpc_call_stats *stats, void *ignored) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
 
@@ -886,7 +929,7 @@ const grpc_channel_filter grpc_server_top_filter = {
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
-    grpc_call_stack_ignore_set_pollset,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
     destroy_call_elem,
     sizeof(channel_data),
     init_channel_elem,
@@ -895,25 +938,46 @@ const grpc_channel_filter grpc_server_top_filter = {
     "server",
 };
 
-void grpc_server_register_completion_queue(grpc_server *server,
-                                           grpc_completion_queue *cq,
-                                           void *reserved) {
+static void register_completion_queue(grpc_server *server,
+                                      grpc_completion_queue *cq,
+                                      bool is_non_listening, void *reserved) {
   size_t i, n;
-  GRPC_API_TRACE(
-      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
-      (server, cq, reserved));
   GPR_ASSERT(!reserved);
   for (i = 0; i < server->cq_count; i++) {
     if (server->cqs[i] == cq) return;
   }
-  GRPC_CQ_INTERNAL_REF(cq, "server");
+
   grpc_cq_mark_server_cq(cq);
+
+  if (is_non_listening) {
+    grpc_cq_mark_non_listening_server_cq(cq);
+  }
+
+  GRPC_CQ_INTERNAL_REF(cq, "server");
   n = server->cq_count++;
   server->cqs = gpr_realloc(server->cqs,
                             server->cq_count * sizeof(grpc_completion_queue *));
   server->cqs[n] = cq;
 }
 
+void grpc_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq,
+                                           void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
+      (server, cq, reserved));
+  register_completion_queue(server, cq, false, reserved);
+}
+
+void grpc_server_register_non_listening_completion_queue(
+    grpc_server *server, grpc_completion_queue *cq, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, "
+      "reserved=%p)",
+      3, (server, cq, reserved));
+  register_completion_queue(server, cq, true, reserved);
+}
+
 grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   size_t i;
 
@@ -940,8 +1004,6 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   for (i = 0; i < (size_t)server->max_requested_calls; i++) {
     gpr_stack_lockfree_push(server->request_freelist, (int)i);
   }
-  request_matcher_init(&server->unregistered_request_matcher,
-                       server->max_requested_calls, server);
   server->requested_calls = gpr_malloc(server->max_requested_calls *
                                        sizeof(*server->requested_calls));
 
@@ -985,8 +1047,6 @@ void *grpc_server_register_method(
   }
   m = gpr_malloc(sizeof(registered_method));
   memset(m, 0, sizeof(*m));
-  request_matcher_init(&m->request_matcher, server->max_requested_calls,
-                       server);
   m->method = gpr_strdup(method);
   m->host = gpr_strdup(host);
   m->next = server->registered_methods;
@@ -1003,13 +1063,23 @@ void grpc_server_start(grpc_server *server) {
 
   GRPC_API_TRACE("grpc_server_start(server=%p)", 1, (server));
 
+  server->started = true;
+  size_t pollset_count = 0;
   server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
   for (i = 0; i < server->cq_count; i++) {
-    server->pollsets[i] = grpc_cq_pollset(server->cqs[i]);
+    if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) {
+      server->pollsets[pollset_count++] = grpc_cq_pollset(server->cqs[i]);
+    }
+  }
+  request_matcher_init(&server->unregistered_request_matcher,
+                       server->max_requested_calls, server);
+  for (registered_method *rm = server->registered_methods; rm; rm = rm->next) {
+    request_matcher_init(&rm->request_matcher, server->max_requested_calls,
+                         server);
   }
 
   for (l = server->listeners; l; l = l->next) {
-    l->start(&exec_ctx, server, l->arg, server->pollsets, server->cq_count);
+    l->start(&exec_ctx, server, l->arg, server->pollsets, pollset_count);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
@@ -1017,8 +1087,8 @@ void grpc_server_start(grpc_server *server) {
 
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args) {
-  size_t i;
   size_t num_registered_methods;
   size_t alloc;
   registered_method *rm;
@@ -1033,12 +1103,6 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   uint32_t max_probes = 0;
   grpc_transport_op op;
 
-  for (i = 0; i < s->cq_count; i++) {
-    memset(&op, 0, sizeof(op));
-    op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
-    grpc_transport_perform_op(exec_ctx, transport, &op);
-  }
-
   channel =
       grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
   chand = (channel_data *)grpc_channel_stack_element(
@@ -1048,6 +1112,17 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   server_ref(s);
   chand->channel = channel;
 
+  size_t cq_idx;
+  grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
+  for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
+    if (s->cqs[cq_idx] == accepting_cq) break;
+  }
+  if (cq_idx == s->cq_count) {
+    /* completion queue not found: pick a random one to publish new calls to */
+    cq_idx = (size_t)rand() % s->cq_count;
+  }
+  chand->cq_idx = cq_idx;
+
   num_registered_methods = 0;
   for (rm = s->registered_methods; rm; rm = rm->next) {
     num_registered_methods++;
@@ -1092,7 +1167,9 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   op.set_accept_stream_user_data = chand;
   op.on_connectivity_state_change = &chand->channel_connectivity_changed;
   op.connectivity_state = &chand->connectivity_state;
-  op.disconnect = gpr_atm_acq_load(&s->shutdown_flag) != 0;
+  if (gpr_atm_acq_load(&s->shutdown_flag) != 0) {
+    op.disconnect_with_error = GRPC_ERROR_CREATE("Server shutdown");
+  }
   grpc_transport_perform_op(exec_ctx, transport, &op);
 }
 
@@ -1103,7 +1180,7 @@ void done_published_shutdown(grpc_exec_ctx *exec_ctx, void *done_arg,
 }
 
 static void listener_destroy_done(grpc_exec_ctx *exec_ctx, void *s,
-                                  bool success) {
+                                  grpc_error *error) {
   grpc_server *server = s;
   gpr_mu_lock(&server->mu_global);
   server->listeners_destroyed++;
@@ -1125,8 +1202,8 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
   gpr_mu_lock(&server->mu_global);
   grpc_cq_begin_op(cq, tag);
   if (server->shutdown_published) {
-    grpc_cq_end_op(&exec_ctx, cq, tag, 1, done_published_shutdown, NULL,
-                   gpr_malloc(sizeof(grpc_cq_completion)));
+    grpc_cq_end_op(&exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown,
+                   NULL, gpr_malloc(sizeof(grpc_cq_completion)));
     gpr_mu_unlock(&server->mu_global);
     goto done;
   }
@@ -1149,7 +1226,8 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
 
   /* collect all unregistered then registered calls */
   gpr_mu_lock(&server->mu_call);
-  kill_pending_work_locked(&exec_ctx, server);
+  kill_pending_work_locked(&exec_ctx, server,
+                           GRPC_ERROR_CREATE("Server Shutdown"));
   gpr_mu_unlock(&server->mu_call);
 
   maybe_finish_shutdown(&exec_ctx, server);
@@ -1177,7 +1255,8 @@ void grpc_server_cancel_all_calls(grpc_server *server) {
   channel_broadcaster_init(server, &broadcaster);
   gpr_mu_unlock(&server->mu_global);
 
-  channel_broadcaster_shutdown(&exec_ctx, &broadcaster, 0, 1);
+  channel_broadcaster_shutdown(&exec_ctx, &broadcaster, 0,
+                               GRPC_ERROR_CREATE("Cancelling all calls"));
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
@@ -1218,19 +1297,21 @@ void grpc_server_add_listener(
 }
 
 static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
-                                          grpc_server *server,
+                                          grpc_server *server, size_t cq_idx,
                                           requested_call *rc) {
   call_data *calld = NULL;
   request_matcher *rm = NULL;
   int request_id;
   if (gpr_atm_acq_load(&server->shutdown_flag)) {
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc,
+              GRPC_ERROR_CREATE("Server Shutdown"));
     return GRPC_CALL_OK;
   }
   request_id = gpr_stack_lockfree_pop(server->request_freelist);
   if (request_id == -1) {
     /* out of request ids: just fail this one */
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc,
+              GRPC_ERROR_CREATE("Server Shutdown"));
     return GRPC_CALL_OK;
   }
   switch (rc->type) {
@@ -1243,12 +1324,12 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
   }
   server->requested_calls[request_id] = *rc;
   gpr_free(rc);
-  if (gpr_stack_lockfree_push(rm->requests, request_id)) {
+  if (gpr_stack_lockfree_push(rm->requests_per_cq[cq_idx], request_id)) {
     /* this was the first queued request: we need to lock and start
        matching calls */
     gpr_mu_lock(&server->mu_call);
     while ((calld = rm->pending_head) != NULL) {
-      request_id = gpr_stack_lockfree_pop(rm->requests);
+      request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
       if (request_id == -1) break;
       rm->pending_head = calld->pending_next;
       gpr_mu_unlock(&server->mu_call);
@@ -1258,13 +1339,13 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
         grpc_closure_init(
             &calld->kill_zombie_closure, kill_zombie,
             grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
-        grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true,
-                              NULL);
+        grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure,
+                            GRPC_ERROR_NONE, NULL);
       } else {
         GPR_ASSERT(calld->state == PENDING);
         calld->state = ACTIVATED;
         gpr_mu_unlock(&calld->mu_state);
-        publish_call(exec_ctx, server, calld,
+        publish_call(exec_ctx, server, calld, cq_idx,
                      &server->requested_calls[request_id]);
       }
       gpr_mu_lock(&server->mu_call);
@@ -1288,7 +1369,13 @@ grpc_call_error grpc_server_request_call(
       "cq_bound_to_call=%p, cq_for_notification=%p, tag=%p)",
       7, (server, call, details, initial_metadata, cq_bound_to_call,
           cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1299,11 +1386,10 @@ grpc_call_error grpc_server_request_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.batch.details = details;
   rc->initial_metadata = initial_metadata;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
@@ -1325,7 +1411,14 @@ grpc_call_error grpc_server_request_registered_call(
       "tag=%p)",
       9, (server, rmp, call, deadline, initial_metadata, optional_payload,
           cq_bound_to_call, cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1341,25 +1434,25 @@ grpc_call_error grpc_server_request_registered_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.registered.registered_method = rm;
   rc->data.registered.deadline = deadline;
   rc->initial_metadata = initial_metadata;
   rc->data.registered.optional_payload = optional_payload;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
 }
 
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc) {
+                      size_t cq_idx, requested_call *rc, grpc_error *error) {
   *rc->call = NULL;
   rc->initial_metadata->count = 0;
+  GPR_ASSERT(error != GRPC_ERROR_NONE);
 
   server_ref(server);
-  grpc_cq_end_op(exec_ctx, rc->cq_for_notification, rc->tag, 0,
+  grpc_cq_end_op(exec_ctx, server->cqs[cq_idx], rc->tag, error,
                  done_request_event, rc, &rc->completion);
 }
 
diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h
index 470ef23c69..fb6e4d60c5 100644
--- a/src/core/lib/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -53,6 +53,7 @@ void grpc_server_add_listener(
    server */
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *server,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args);
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c
index c18ea7bb9f..53f3c43854 100644
--- a/src/core/lib/surface/version.c
+++ b/src/core/lib/surface/version.c
@@ -36,4 +36,4 @@
 
 #include <grpc/grpc.h>
 
-const char *grpc_version_string(void) { return "0.14.2-pre1"; }
+const char *grpc_version_string(void) { return "0.16.0-dev"; }
diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c
index e24ee638fd..054f112127 100644
--- a/src/core/lib/transport/connectivity_state.c
+++ b/src/core/lib/transport/connectivity_state.c
@@ -51,8 +51,8 @@ const char *grpc_connectivity_state_name(grpc_connectivity_state state) {
       return "READY";
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
       return "TRANSIENT_FAILURE";
-    case GRPC_CHANNEL_FATAL_FAILURE:
-      return "FATAL_FAILURE";
+    case GRPC_CHANNEL_SHUTDOWN:
+      return "SHUTDOWN";
   }
   GPR_UNREACHABLE_CODE(return "UNKNOWN");
 }
@@ -61,35 +61,40 @@ void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker,
                                   grpc_connectivity_state init_state,
                                   const char *name) {
   tracker->current_state = init_state;
+  tracker->current_error = GRPC_ERROR_NONE;
   tracker->watchers = NULL;
   tracker->name = gpr_strdup(name);
 }
 
 void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx,
                                      grpc_connectivity_state_tracker *tracker) {
-  int success;
+  grpc_error *error;
   grpc_connectivity_state_watcher *w;
   while ((w = tracker->watchers)) {
     tracker->watchers = w->next;
 
-    if (GRPC_CHANNEL_FATAL_FAILURE != *w->current) {
-      *w->current = GRPC_CHANNEL_FATAL_FAILURE;
-      success = 1;
+    if (GRPC_CHANNEL_SHUTDOWN != *w->current) {
+      *w->current = GRPC_CHANNEL_SHUTDOWN;
+      error = GRPC_ERROR_NONE;
     } else {
-      success = 0;
+      error = GRPC_ERROR_CREATE("Shutdown connectivity owner");
     }
-    grpc_exec_ctx_enqueue(exec_ctx, w->notify, success, NULL);
+    grpc_exec_ctx_sched(exec_ctx, w->notify, error, NULL);
     gpr_free(w);
   }
+  GRPC_ERROR_UNREF(tracker->current_error);
   gpr_free(tracker->name);
 }
 
 grpc_connectivity_state grpc_connectivity_state_check(
-    grpc_connectivity_state_tracker *tracker) {
+    grpc_connectivity_state_tracker *tracker, grpc_error **error) {
   if (grpc_connectivity_state_trace) {
     gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
             grpc_connectivity_state_name(tracker->current_state));
   }
+  if (error != NULL) {
+    *error = GRPC_ERROR_REF(tracker->current_error);
+  }
   return tracker->current_state;
 }
 
@@ -109,7 +114,7 @@ int grpc_connectivity_state_notify_on_state_change(
   if (current == NULL) {
     grpc_connectivity_state_watcher *w = tracker->watchers;
     if (w != NULL && w->notify == notify) {
-      grpc_exec_ctx_enqueue(exec_ctx, notify, false, NULL);
+      grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
       tracker->watchers = w->next;
       gpr_free(w);
       return 0;
@@ -117,7 +122,7 @@ int grpc_connectivity_state_notify_on_state_change(
     while (w != NULL) {
       grpc_connectivity_state_watcher *rm_candidate = w->next;
       if (rm_candidate != NULL && rm_candidate->notify == notify) {
-        grpc_exec_ctx_enqueue(exec_ctx, notify, false, NULL);
+        grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
         w->next = w->next->next;
         gpr_free(rm_candidate);
         return 0;
@@ -128,7 +133,8 @@ int grpc_connectivity_state_notify_on_state_change(
   } else {
     if (tracker->current_state != *current) {
       *current = tracker->current_state;
-      grpc_exec_ctx_enqueue(exec_ctx, notify, true, NULL);
+      grpc_exec_ctx_sched(exec_ctx, notify,
+                          GRPC_ERROR_REF(tracker->current_error), NULL);
     } else {
       grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w));
       w->current = current;
@@ -143,22 +149,38 @@ int grpc_connectivity_state_notify_on_state_change(
 void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
                                  grpc_connectivity_state_tracker *tracker,
                                  grpc_connectivity_state state,
-                                 const char *reason) {
+                                 grpc_error *error, const char *reason) {
   grpc_connectivity_state_watcher *w;
   if (grpc_connectivity_state_trace) {
-    gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s]", tracker, tracker->name,
-            grpc_connectivity_state_name(tracker->current_state),
-            grpc_connectivity_state_name(state), reason);
+    const char *error_string = grpc_error_string(error);
+    gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker,
+            tracker->name, grpc_connectivity_state_name(tracker->current_state),
+            grpc_connectivity_state_name(state), reason, error, error_string);
+    grpc_error_free_string(error_string);
+  }
+  switch (state) {
+    case GRPC_CHANNEL_CONNECTING:
+    case GRPC_CHANNEL_IDLE:
+    case GRPC_CHANNEL_READY:
+      GPR_ASSERT(error == GRPC_ERROR_NONE);
+      break;
+    case GRPC_CHANNEL_SHUTDOWN:
+    case GRPC_CHANNEL_TRANSIENT_FAILURE:
+      GPR_ASSERT(error != GRPC_ERROR_NONE);
+      break;
   }
+  GRPC_ERROR_UNREF(tracker->current_error);
+  tracker->current_error = error;
   if (tracker->current_state == state) {
     return;
   }
-  GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_FATAL_FAILURE);
+  GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_SHUTDOWN);
   tracker->current_state = state;
   while ((w = tracker->watchers) != NULL) {
     *w->current = tracker->current_state;
     tracker->watchers = w->next;
-    grpc_exec_ctx_enqueue(exec_ctx, w->notify, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, w->notify,
+                        GRPC_ERROR_REF(tracker->current_error), NULL);
     gpr_free(w);
   }
 }
diff --git a/src/core/lib/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h
index 2eb7e09124..7a2fa52c10 100644
--- a/src/core/lib/transport/connectivity_state.h
+++ b/src/core/lib/transport/connectivity_state.h
@@ -49,6 +49,8 @@ typedef struct grpc_connectivity_state_watcher {
 typedef struct {
   /** current connectivity state */
   grpc_connectivity_state current_state;
+  /** error associated with state */
+  grpc_error *current_error;
   /** all our watchers */
   grpc_connectivity_state_watcher *watchers;
   /** a name to help debugging */
@@ -70,10 +72,11 @@ void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx,
 void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
                                  grpc_connectivity_state_tracker *tracker,
                                  grpc_connectivity_state state,
+                                 grpc_error *associated_error,
                                  const char *reason);
 
 grpc_connectivity_state grpc_connectivity_state_check(
-    grpc_connectivity_state_tracker *tracker);
+    grpc_connectivity_state_tracker *tracker, grpc_error **current_error);
 
 /** Return 1 if the channel should start connecting, 0 otherwise.
     If current==NULL cancel notify if it is already queued (success==0 in that
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 5847ec9053..0677f29766 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -129,7 +129,10 @@ typedef struct mdtab_shard {
   internal_metadata **elems;
   size_t count;
   size_t capacity;
-  size_t free;
+  /** Estimate of the number of unreferenced mdelems in the hash table.
+      This will eventually converge to the exact number, but it's instantaneous
+      accuracy is not guaranteed */
+  gpr_atm free_estimate;
 } mdtab_shard;
 
 #define LOG2_STRTAB_SHARD_COUNT 5
@@ -217,7 +220,7 @@ void grpc_mdctx_global_init(void) {
     mdtab_shard *shard = &g_mdtab_shard[i];
     gpr_mu_init(&shard->mu);
     shard->count = 0;
-    shard->free = 0;
+    gpr_atm_no_barrier_store(&shard->free_estimate, 0);
     shard->capacity = INITIAL_MDTAB_CAPACITY;
     shard->elems = gpr_malloc(sizeof(*shard->elems) * shard->capacity);
     memset(shard->elems, 0, sizeof(*shard->elems) * shard->capacity);
@@ -232,7 +235,7 @@ void grpc_mdctx_global_shutdown(void) {
     gc_mdtab(shard);
     /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
     if (shard->count != 0) {
-      gpr_log(GPR_DEBUG, "WARNING: %d metadata elements were leaked",
+      gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata elements were leaked",
               shard->count);
       if (grpc_iomgr_abort_on_leaks()) {
         abort();
@@ -245,7 +248,7 @@ void grpc_mdctx_global_shutdown(void) {
     gpr_mu_destroy(&shard->mu);
     /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
     if (shard->count != 0) {
-      gpr_log(GPR_DEBUG, "WARNING: %d metadata strings were leaked",
+      gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked",
               shard->count);
       for (size_t j = 0; j < shard->capacity; j++) {
         for (internal_string *s = shard->strs[j]; s; s = s->bucket_next) {
@@ -281,10 +284,8 @@ static void ref_md_locked(mdtab_shard *shard,
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->value));
 #endif
-  if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 2)) {
-    shard->free--;
-  } else {
-    GPR_ASSERT(1 != gpr_atm_no_barrier_fetch_add(&md->refcnt, -1));
+  if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 1)) {
+    gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -1);
   }
 }
 
@@ -373,7 +374,8 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) {
     ss = g_static_strtab[idx];
     if (ss == NULL) break;
     if (ss->hash == hash && GPR_SLICE_LENGTH(ss->slice) == length &&
-        0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length)) {
+        (length == 0 ||
+         0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length))) {
       GPR_TIMER_END("grpc_mdstr_from_buffer", 0);
       return ss;
     }
@@ -446,6 +448,7 @@ static void gc_mdtab(mdtab_shard *shard) {
   size_t i;
   internal_metadata **prev_next;
   internal_metadata *md, *next;
+  gpr_atm num_freed = 0;
 
   GPR_TIMER_BEGIN("gc_mdtab", 0);
   for (i = 0; i < shard->capacity; i++) {
@@ -462,13 +465,14 @@ static void gc_mdtab(mdtab_shard *shard) {
         }
         gpr_free(md);
         *prev_next = next;
-        shard->free--;
+        num_freed++;
         shard->count--;
       } else {
         prev_next = &md->bucket_next;
       }
     }
   }
+  gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -num_freed);
   GPR_TIMER_END("gc_mdtab", 0);
 }
 
@@ -503,7 +507,8 @@ static void grow_mdtab(mdtab_shard *shard) {
 }
 
 static void rehash_mdtab(mdtab_shard *shard) {
-  if (shard->free > shard->capacity / 4) {
+  if (gpr_atm_no_barrier_load(&shard->free_estimate) >
+      (gpr_atm)(shard->capacity / 4)) {
     gc_mdtab(shard);
   } else {
     grow_mdtab(shard);
@@ -552,7 +557,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdstr *mkey,
 
   /* not found: create a new pair */
   md = gpr_malloc(sizeof(internal_metadata));
-  gpr_atm_rel_store(&md->refcnt, 2);
+  gpr_atm_rel_store(&md->refcnt, 1);
   md->key = key;
   md->value = value;
   md->user_data = 0;
@@ -644,7 +649,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) {
      this function - meaning that no adjustment to mdtab_free is necessary,
      simplifying the logic here to be just an atomic increment */
   /* use C assert to have this removed in opt builds */
-  assert(gpr_atm_no_barrier_load(&md->refcnt) >= 2);
+  assert(gpr_atm_no_barrier_load(&md->refcnt) >= 1);
   gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
   return gmd;
 }
@@ -661,18 +666,13 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) {
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->value));
 #endif
-  if (2 == gpr_atm_full_fetch_add(&md->refcnt, -1)) {
-    uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash);
+  uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash);
+  if (1 == gpr_atm_full_fetch_add(&md->refcnt, -1)) {
+    /* once the refcount hits zero, some other thread can come along and
+       free md at any time: it's unsafe from this point on to access it */
     mdtab_shard *shard =
         &g_mdtab_shard[SHARD_IDX(hash, LOG2_MDTAB_SHARD_COUNT)];
-    GPR_TIMER_BEGIN("grpc_mdelem_unref.to_zero", 0);
-    gpr_mu_lock(&shard->mu);
-    if (1 == gpr_atm_no_barrier_load(&md->refcnt)) {
-      shard->free++;
-      gpr_atm_no_barrier_store(&md->refcnt, 0);
-    }
-    gpr_mu_unlock(&shard->mu);
-    GPR_TIMER_END("grpc_mdelem_unref.to_zero", 0);
+    gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1);
   }
 }
 
diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h
index 713d9e6782..6d82f4d681 100644
--- a/src/core/lib/transport/metadata.h
+++ b/src/core/lib/transport/metadata.h
@@ -147,6 +147,10 @@ const char *grpc_mdstr_as_c_string(grpc_mdstr *s);
 
 #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice))
 
+/* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */
+#define GRPC_MDELEM_LENGTH(e) \
+  (GRPC_MDSTR_LENGTH((e)->key) + GRPC_MDSTR_LENGTH((e)->value) + 32)
+
 int grpc_mdstr_is_legal_header(grpc_mdstr *s);
 int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s);
 int grpc_mdstr_is_bin_suffixed(grpc_mdstr *s);
diff --git a/src/core/lib/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c
index 4567221a48..e4398abeb7 100644
--- a/src/core/lib/transport/metadata_batch.c
+++ b/src/core/lib/transport/metadata_batch.c
@@ -192,3 +192,12 @@ int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch) {
          gpr_time_cmp(gpr_inf_future(batch->deadline.clock_type),
                       batch->deadline) == 0;
 }
+
+size_t grpc_metadata_batch_size(grpc_metadata_batch *batch) {
+  size_t size = 0;
+  for (grpc_linked_mdelem *elem = batch->list.head; elem != NULL;
+       elem = elem->next) {
+    size += GRPC_MDELEM_LENGTH(elem->md);
+  }
+  return size;
+}
diff --git a/src/core/lib/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h
index b62668876e..7af823f7ca 100644
--- a/src/core/lib/transport/metadata_batch.h
+++ b/src/core/lib/transport/metadata_batch.h
@@ -66,6 +66,9 @@ void grpc_metadata_batch_destroy(grpc_metadata_batch *batch);
 void grpc_metadata_batch_clear(grpc_metadata_batch *batch);
 int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch);
 
+/* Returns the transport size of the batch. */
+size_t grpc_metadata_batch_size(grpc_metadata_batch *batch);
+
 /** Moves the metadata information from \a src to \a dst. Upon return, \a src is
  * zeroed. */
 void grpc_metadata_batch_move(grpc_metadata_batch *dst,
diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c
index 73b0041fd4..c5f16e530d 100644
--- a/src/core/lib/transport/static_metadata.c
+++ b/src/core/lib/transport/static_metadata.c
@@ -48,7 +48,7 @@ uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 4, 8, 6, 2, 4, 8, 6, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
     {11, 35, 10, 35, 12, 35, 12, 49, 13, 35, 14, 35, 15, 35, 16, 35, 17, 35,
@@ -56,10 +56,10 @@ const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] =
      30, 18, 30, 35, 31, 35, 32, 35, 36, 35, 37, 35, 38, 35, 39, 35, 42, 33,
      42, 34, 42, 48, 42, 53, 42, 54, 42, 55, 42, 56, 43, 33, 43, 48, 43, 53,
      46, 0,  46, 1,  46, 2,  50, 35, 57, 35, 58, 35, 59, 35, 60, 35, 61, 35,
-     62, 35, 63, 35, 64, 35, 65, 35, 66, 40, 66, 68, 66, 71, 67, 79, 67, 80,
-     69, 35, 70, 35, 72, 35, 73, 35, 74, 35, 75, 35, 76, 41, 76, 51, 76, 52,
-     77, 35, 78, 35, 81, 3,  81, 4,  81, 5,  81, 6,  81, 7,  81, 8,  81, 9,
-     82, 35, 83, 84, 85, 35, 86, 35, 87, 35, 88, 35, 89, 35};
+     62, 35, 63, 35, 64, 35, 65, 35, 66, 35, 67, 40, 67, 69, 67, 72, 68, 80,
+     68, 81, 70, 35, 71, 35, 73, 35, 74, 35, 75, 35, 76, 35, 77, 41, 77, 51,
+     77, 52, 78, 35, 79, 35, 82, 3,  82, 4,  82, 5,  82, 6,  82, 7,  82, 8,
+     82, 9,  83, 35, 84, 85, 86, 35, 87, 35, 88, 35, 89, 35, 90, 35};
 
 const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "0",
@@ -126,6 +126,7 @@ const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "if-unmodified-since",
     "last-modified",
     "link",
+    "load-reporting",
     "location",
     "max-forwards",
     ":method",
diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h
index f9d8bcdc8f..5ff0d2f3bc 100644
--- a/src/core/lib/transport/static_metadata.h
+++ b/src/core/lib/transport/static_metadata.h
@@ -44,7 +44,7 @@
 
 #include "src/core/lib/transport/metadata.h"
 
-#define GRPC_STATIC_MDSTR_COUNT 90
+#define GRPC_STATIC_MDSTR_COUNT 91
 extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 /* "0" */
 #define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0])
@@ -175,60 +175,62 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[62])
 /* "link" */
 #define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[63])
+/* "load-reporting" */
+#define GRPC_MDSTR_LOAD_REPORTING (&grpc_static_mdstr_table[64])
 /* "location" */
-#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[64])
+#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[65])
 /* "max-forwards" */
-#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[65])
+#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[66])
 /* ":method" */
-#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[66])
+#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[67])
 /* ":path" */
-#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[67])
+#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[68])
 /* "POST" */
-#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[68])
+#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[69])
 /* "proxy-authenticate" */
-#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[69])
+#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[70])
 /* "proxy-authorization" */
-#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[70])
+#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[71])
 /* "PUT" */
-#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[71])
+#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[72])
 /* "range" */
-#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[72])
+#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[73])
 /* "referer" */
-#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[73])
+#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[74])
 /* "refresh" */
-#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[74])
+#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[75])
 /* "retry-after" */
-#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[75])
+#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[76])
 /* ":scheme" */
-#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[76])
+#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[77])
 /* "server" */
-#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[77])
+#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[78])
 /* "set-cookie" */
-#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[78])
+#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[79])
 /* "/" */
-#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[79])
+#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[80])
 /* "/index.html" */
-#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[80])
+#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[81])
 /* ":status" */
-#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[81])
+#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[82])
 /* "strict-transport-security" */
-#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[82])
+#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[83])
 /* "te" */
-#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[83])
+#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[84])
 /* "trailers" */
-#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[84])
+#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[85])
 /* "transfer-encoding" */
-#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[85])
+#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[86])
 /* "user-agent" */
-#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[86])
+#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[87])
 /* "vary" */
-#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[87])
+#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[88])
 /* "via" */
-#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[88])
+#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[89])
 /* "www-authenticate" */
-#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[89])
+#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[90])
 
-#define GRPC_STATIC_MDELEM_COUNT 79
+#define GRPC_STATIC_MDELEM_COUNT 80
 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 /* "accept-charset": "" */
@@ -333,71 +335,73 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];
 #define GRPC_MDELEM_LAST_MODIFIED_EMPTY (&grpc_static_mdelem_table[45])
 /* "link": "" */
 #define GRPC_MDELEM_LINK_EMPTY (&grpc_static_mdelem_table[46])
+/* "load-reporting": "" */
+#define GRPC_MDELEM_LOAD_REPORTING_EMPTY (&grpc_static_mdelem_table[47])
 /* "location": "" */
-#define GRPC_MDELEM_LOCATION_EMPTY (&grpc_static_mdelem_table[47])
+#define GRPC_MDELEM_LOCATION_EMPTY (&grpc_static_mdelem_table[48])
 /* "max-forwards": "" */
-#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (&grpc_static_mdelem_table[48])
+#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (&grpc_static_mdelem_table[49])
 /* ":method": "GET" */
-#define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[49])
+#define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[50])
 /* ":method": "POST" */
-#define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[50])
+#define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[51])
 /* ":method": "PUT" */
-#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[51])
+#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[52])
 /* ":path": "/" */
-#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[52])
+#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[53])
 /* ":path": "/index.html" */
-#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[53])
+#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[54])
 /* "proxy-authenticate": "" */
-#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[54])
+#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[55])
 /* "proxy-authorization": "" */
-#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[55])
+#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[56])
 /* "range": "" */
-#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[56])
+#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[57])
 /* "referer": "" */
-#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[57])
+#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[58])
 /* "refresh": "" */
-#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[58])
+#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[59])
 /* "retry-after": "" */
-#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[59])
+#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[60])
 /* ":scheme": "grpc" */
-#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[60])
+#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[61])
 /* ":scheme": "http" */
-#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[61])
+#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[62])
 /* ":scheme": "https" */
-#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[62])
+#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[63])
 /* "server": "" */
-#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[63])
+#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[64])
 /* "set-cookie": "" */
-#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[64])
+#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[65])
 /* ":status": "200" */
-#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[65])
+#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[66])
 /* ":status": "204" */
-#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[66])
+#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[67])
 /* ":status": "206" */
-#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[67])
+#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[68])
 /* ":status": "304" */
-#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[68])
+#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[69])
 /* ":status": "400" */
-#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[69])
+#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[70])
 /* ":status": "404" */
-#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[70])
+#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[71])
 /* ":status": "500" */
-#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[71])
+#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[72])
 /* "strict-transport-security": "" */
 #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \
-  (&grpc_static_mdelem_table[72])
+  (&grpc_static_mdelem_table[73])
 /* "te": "trailers" */
-#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[73])
+#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[74])
 /* "transfer-encoding": "" */
-#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[74])
+#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[75])
 /* "user-agent": "" */
-#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[75])
+#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[76])
 /* "vary": "" */
-#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[76])
+#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[77])
 /* "via": "" */
-#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[77])
+#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[78])
 /* "www-authenticate": "" */
-#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[78])
+#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[79])
 
 extern const uint8_t
     grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2];
diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c
index e6d524abe6..79a20e1262 100644
--- a/src/core/lib/transport/transport.c
+++ b/src/core/lib/transport/transport.c
@@ -36,6 +36,7 @@
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
+#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/transport_impl.h"
 
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
@@ -60,7 +61,7 @@ void grpc_stream_unref(grpc_exec_ctx *exec_ctx,
                        grpc_stream_refcount *refcount) {
 #endif
   if (gpr_unref(&refcount->refs)) {
-    grpc_exec_ctx_enqueue(exec_ctx, &refcount->destroy, true, NULL);
+    grpc_exec_ctx_sched(exec_ctx, &refcount->destroy, GRPC_ERROR_NONE, NULL);
   }
 }
 
@@ -125,10 +126,19 @@ void grpc_transport_perform_op(grpc_exec_ctx *exec_ctx,
   transport->vtable->perform_op(exec_ctx, transport, op);
 }
 
-void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx,
-                                grpc_transport *transport, grpc_stream *stream,
-                                grpc_pollset *pollset) {
-  transport->vtable->set_pollset(exec_ctx, transport, stream, pollset);
+void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport,
+                             grpc_stream *stream,
+                             grpc_polling_entity *pollent) {
+  grpc_pollset *pollset;
+  grpc_pollset_set *pollset_set;
+  if ((pollset = grpc_polling_entity_pollset(pollent)) != NULL) {
+    transport->vtable->set_pollset(exec_ctx, transport, stream, pollset);
+  } else if ((pollset_set = grpc_polling_entity_pollset_set(pollent)) != NULL) {
+    transport->vtable->set_pollset_set(exec_ctx, transport, stream,
+                                       pollset_set);
+  } else {
+    abort();
+  }
 }
 
 void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
@@ -143,62 +153,73 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
   return transport->vtable->get_peer(exec_ctx, transport);
 }
 
-void grpc_transport_stream_op_finish_with_failure(
-    grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op) {
-  grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, false, NULL);
-  grpc_exec_ctx_enqueue(exec_ctx, op->recv_initial_metadata_ready, false, NULL);
-  grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, false, NULL);
-}
-
-void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op,
-                                               grpc_status_code status) {
-  GPR_ASSERT(status != GRPC_STATUS_OK);
-  if (op->cancel_with_status == GRPC_STATUS_OK) {
-    op->cancel_with_status = status;
-  }
-  if (op->close_with_status != GRPC_STATUS_OK) {
-    op->close_with_status = GRPC_STATUS_OK;
-    if (op->optional_close_message != NULL) {
-      gpr_slice_unref(*op->optional_close_message);
-      op->optional_close_message = NULL;
-    }
-  }
+void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx,
+                                                  grpc_transport_stream_op *op,
+                                                  grpc_error *error) {
+  grpc_exec_ctx_sched(exec_ctx, op->recv_message_ready, GRPC_ERROR_REF(error),
+                      NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->recv_initial_metadata_ready,
+                      GRPC_ERROR_REF(error), NULL);
+  grpc_exec_ctx_sched(exec_ctx, op->on_complete, error, NULL);
 }
 
 typedef struct {
-  gpr_slice message;
+  grpc_error *error;
   grpc_closure *then_call;
   grpc_closure closure;
 } close_message_data;
 
-static void free_message(grpc_exec_ctx *exec_ctx, void *p, bool iomgr_success) {
+static void free_message(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) {
   close_message_data *cmd = p;
-  gpr_slice_unref(cmd->message);
+  GRPC_ERROR_UNREF(cmd->error);
   if (cmd->then_call != NULL) {
-    cmd->then_call->cb(exec_ctx, cmd->then_call->cb_arg, iomgr_success);
+    cmd->then_call->cb(exec_ctx, cmd->then_call->cb_arg, GRPC_ERROR_REF(error));
   }
   gpr_free(cmd);
 }
 
+static void add_error(grpc_transport_stream_op *op, grpc_error **which,
+                      grpc_error *error) {
+  close_message_data *cmd;
+  cmd = gpr_malloc(sizeof(*cmd));
+  cmd->error = error;
+  cmd->then_call = op->on_complete;
+  grpc_closure_init(&cmd->closure, free_message, cmd);
+  op->on_complete = &cmd->closure;
+  *which = error;
+}
+
+void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op,
+                                               grpc_status_code status) {
+  GPR_ASSERT(status != GRPC_STATUS_OK);
+  if (op->cancel_error == GRPC_ERROR_NONE) {
+    op->cancel_error = grpc_error_set_int(GRPC_ERROR_CANCELLED,
+                                          GRPC_ERROR_INT_GRPC_STATUS, status);
+    op->close_error = GRPC_ERROR_NONE;
+  }
+}
+
 void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op,
                                         grpc_status_code status,
                                         gpr_slice *optional_message) {
-  close_message_data *cmd;
   GPR_ASSERT(status != GRPC_STATUS_OK);
-  if (op->cancel_with_status != GRPC_STATUS_OK ||
-      op->close_with_status != GRPC_STATUS_OK) {
+  if (op->cancel_error != GRPC_ERROR_NONE ||
+      op->close_error != GRPC_ERROR_NONE) {
     if (optional_message) {
       gpr_slice_unref(*optional_message);
     }
     return;
   }
-  if (optional_message) {
-    cmd = gpr_malloc(sizeof(*cmd));
-    cmd->message = *optional_message;
-    cmd->then_call = op->on_complete;
-    grpc_closure_init(&cmd->closure, free_message, cmd);
-    op->on_complete = &cmd->closure;
-    op->optional_close_message = &cmd->message;
+  grpc_error *error;
+  if (optional_message != NULL) {
+    char *msg = gpr_dump_slice(*optional_message, GPR_DUMP_ASCII);
+    error = grpc_error_set_str(GRPC_ERROR_CREATE(msg),
+                               GRPC_ERROR_STR_GRPC_MESSAGE, msg);
+    gpr_free(msg);
+    gpr_slice_unref(*optional_message);
+  } else {
+    error = GRPC_ERROR_CREATE("Call force closed");
   }
-  op->close_with_status = status;
+  error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, status);
+  add_error(op, &op->close_error, error);
 }
diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h
index 482a9d1791..d2f6344ee3 100644
--- a/src/core/lib/transport/transport.h
+++ b/src/core/lib/transport/transport.h
@@ -37,6 +37,7 @@
 #include <stddef.h>
 
 #include "src/core/lib/channel/context.h"
+#include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 #include "src/core/lib/transport/byte_stream.h"
@@ -134,13 +135,12 @@ typedef struct grpc_transport_stream_op {
   /** Collect any stats into provided buffer, zero internal stat counters */
   grpc_transport_stream_stats *collect_stats;
 
-  /** If != GRPC_STATUS_OK, cancel this stream */
-  grpc_status_code cancel_with_status;
+  /** If != GRPC_ERROR_NONE, cancel this stream */
+  grpc_error *cancel_error;
 
-  /** If != GRPC_STATUS_OK, send grpc-status, grpc-message, and close this
+  /** If != GRPC_ERROR, send grpc-status, grpc-message, and close this
       stream for both reading and writing */
-  grpc_status_code close_with_status;
-  gpr_slice *optional_close_message;
+  grpc_error *close_error;
 
   /* Indexes correspond to grpc_context_index enum values */
   grpc_call_context_element *context;
@@ -154,7 +154,7 @@ typedef struct grpc_transport_op {
   grpc_closure *on_connectivity_state_change;
   grpc_connectivity_state *connectivity_state;
   /** should the transport be disconnected */
-  int disconnect;
+  grpc_error *disconnect_with_error;
   /** should we send a goaway?
       after a goaway is sent, once there are no more active calls on
       the transport, the transport should disconnect */
@@ -197,9 +197,8 @@ int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx,
                                grpc_stream_refcount *refcount,
                                const void *server_data);
 
-void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx,
-                                grpc_transport *transport, grpc_stream *stream,
-                                grpc_pollset *pollset);
+void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport,
+                             grpc_stream *stream, grpc_polling_entity *pollent);
 
 /* Destroy transport data for a stream.
 
@@ -216,7 +215,8 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
                                    grpc_stream *stream, void *and_free_memory);
 
 void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx,
-                                                  grpc_transport_stream_op *op);
+                                                  grpc_transport_stream_op *op,
+                                                  grpc_error *error);
 
 void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op,
                                                grpc_status_code status);
diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h
index 956155eec8..fc7140671b 100644
--- a/src/core/lib/transport/transport_impl.h
+++ b/src/core/lib/transport/transport_impl.h
@@ -53,6 +53,10 @@ typedef struct grpc_transport_vtable {
   void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                       grpc_stream *stream, grpc_pollset *pollset);
 
+  /* implementation of grpc_transport_set_pollset */
+  void (*set_pollset_set)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
+                          grpc_stream *stream, grpc_pollset_set *pollset_set);
+
   /* implementation of grpc_transport_perform_stream_op */
   void (*perform_stream_op)(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                             grpc_stream *stream, grpc_transport_stream_op *op);
diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c
index df04c61127..138591db2a 100644
--- a/src/core/lib/transport/transport_op_string.c
+++ b/src/core/lib/transport/transport_op_string.c
@@ -63,8 +63,8 @@ static void put_metadata_list(gpr_strvec *b, grpc_metadata_batch md) {
   }
   if (gpr_time_cmp(md.deadline, gpr_inf_future(md.deadline.clock_type)) != 0) {
     char *tmp;
-    gpr_asprintf(&tmp, " deadline=%lld.%09d", (long long)md.deadline.tv_sec,
-                 (int)md.deadline.tv_nsec);
+    gpr_asprintf(&tmp, " deadline=%" PRId64 ".%09d", md.deadline.tv_sec,
+                 md.deadline.tv_nsec);
     gpr_strvec_add(b, tmp);
   }
 }
@@ -119,10 +119,21 @@ char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) {
     gpr_strvec_add(&b, gpr_strdup("RECV_TRAILING_METADATA"));
   }
 
-  if (op->cancel_with_status != GRPC_STATUS_OK) {
+  if (op->cancel_error != GRPC_ERROR_NONE) {
     if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
     first = 0;
-    gpr_asprintf(&tmp, "CANCEL:%d", op->cancel_with_status);
+    const char *msg = grpc_error_string(op->cancel_error);
+    gpr_asprintf(&tmp, "CANCEL:%s", msg);
+    grpc_error_free_string(msg);
+    gpr_strvec_add(&b, tmp);
+  }
+
+  if (op->close_error != GRPC_ERROR_NONE) {
+    if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
+    first = 0;
+    const char *msg = grpc_error_string(op->close_error);
+    gpr_asprintf(&tmp, "CLOSE:%s", msg);
+    grpc_error_free_string(msg);
     gpr_strvec_add(&b, tmp);
   }
 
diff --git a/src/core/plugin_registry/grpc_cronet_plugin_registry.c b/src/core/plugin_registry/grpc_cronet_plugin_registry.c
new file mode 100644
index 0000000000..d0b5f5c702
--- /dev/null
+++ b/src/core/plugin_registry/grpc_cronet_plugin_registry.c
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/grpc.h>
+
+extern void grpc_chttp2_plugin_init(void);
+extern void grpc_chttp2_plugin_shutdown(void);
+extern void grpc_client_config_init(void);
+extern void grpc_client_config_shutdown(void);
+
+void grpc_register_built_in_plugins(void) {
+  grpc_register_plugin(grpc_chttp2_plugin_init,
+                       grpc_chttp2_plugin_shutdown);
+  grpc_register_plugin(grpc_client_config_init,
+                       grpc_client_config_shutdown);
+}
diff --git a/src/core/plugin_registry/grpc_plugin_registry.c b/src/core/plugin_registry/grpc_plugin_registry.c
index 822aa6d8b7..905cd59e23 100644
--- a/src/core/plugin_registry/grpc_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_plugin_registry.c
@@ -45,6 +45,8 @@ extern void grpc_resolver_dns_native_init(void);
 extern void grpc_resolver_dns_native_shutdown(void);
 extern void grpc_resolver_sockaddr_init(void);
 extern void grpc_resolver_sockaddr_shutdown(void);
+extern void grpc_load_reporting_plugin_init(void);
+extern void grpc_load_reporting_plugin_shutdown(void);
 extern void census_grpc_plugin_init(void);
 extern void census_grpc_plugin_shutdown(void);
 
@@ -61,6 +63,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_resolver_dns_native_shutdown);
   grpc_register_plugin(grpc_resolver_sockaddr_init,
                        grpc_resolver_sockaddr_shutdown);
+  grpc_register_plugin(grpc_load_reporting_plugin_init,
+                       grpc_load_reporting_plugin_shutdown);
   grpc_register_plugin(census_grpc_plugin_init,
                        census_grpc_plugin_shutdown);
 }
diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
index a6108ae7a9..7995078725 100644
--- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
@@ -41,6 +41,8 @@ extern void grpc_resolver_dns_native_init(void);
 extern void grpc_resolver_dns_native_shutdown(void);
 extern void grpc_resolver_sockaddr_init(void);
 extern void grpc_resolver_sockaddr_shutdown(void);
+extern void grpc_load_reporting_plugin_init(void);
+extern void grpc_load_reporting_plugin_shutdown(void);
 extern void grpc_lb_policy_pick_first_init(void);
 extern void grpc_lb_policy_pick_first_shutdown(void);
 extern void grpc_lb_policy_round_robin_init(void);
@@ -57,6 +59,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_resolver_dns_native_shutdown);
   grpc_register_plugin(grpc_resolver_sockaddr_init,
                        grpc_resolver_sockaddr_shutdown);
+  grpc_register_plugin(grpc_load_reporting_plugin_init,
+                       grpc_load_reporting_plugin_shutdown);
   grpc_register_plugin(grpc_lb_policy_pick_first_init,
                        grpc_lb_policy_pick_first_shutdown);
   grpc_register_plugin(grpc_lb_policy_round_robin_init,
diff --git a/src/cpp/README.md b/src/cpp/README.md
index f2935e52d9..8c0f85e5ff 100644
--- a/src/cpp/README.md
+++ b/src/cpp/README.md
@@ -51,7 +51,7 @@ below.
 #Build from Source
 
 ```sh
- $ git clone https://github.com/grpc/grpc.git
+ $ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc
  $ cd grpc
  $ git submodule update --init
  $ make
diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc
index 32c7794ade..0ba77a5057 100644
--- a/src/cpp/client/client_context.cc
+++ b/src/cpp/client/client_context.cc
@@ -33,15 +33,14 @@
 
 #include <grpc++/client_context.h>
 
-#include <grpc++/security/credentials.h>
-#include <grpc++/server_context.h>
-#include <grpc++/support/time.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/channel/compress_filter.h"
+#include <grpc++/security/credentials.h>
+#include <grpc++/server_context.h>
+#include <grpc++/support/time.h>
 
 namespace grpc {
 
@@ -64,6 +63,7 @@ ClientContext::ClientContext()
       call_(nullptr),
       call_canceled_(false),
       deadline_(gpr_inf_future(GPR_CLOCK_REALTIME)),
+      census_context_(nullptr),
       propagate_from_call_(nullptr) {
   g_client_callbacks->DefaultConstructor(this);
 }
@@ -112,7 +112,7 @@ void ClientContext::set_compression_algorithm(
     abort();
   }
   GPR_ASSERT(algorithm_name != nullptr);
-  AddMetadata(GRPC_COMPRESS_REQUEST_ALGORITHM_KEY, algorithm_name);
+  AddMetadata(GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, algorithm_name);
 }
 
 void ClientContext::TryCancel() {
diff --git a/src/cpp/client/create_channel_posix.cc b/src/cpp/client/create_channel_posix.cc
new file mode 100644
index 0000000000..60cfed3d62
--- /dev/null
+++ b/src/cpp/client/create_channel_posix.cc
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/channel.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/impl/grpc_library.h>
+#include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
+
+#include "src/cpp/client/create_channel_internal.h"
+
+namespace grpc {
+
+#ifdef GPR_SUPPORT_CHANNELS_FROM_FD
+
+std::shared_ptr<Channel> CreateInsecureChannelFromFd(const grpc::string& target,
+                                                     int fd) {
+  internal::GrpcLibrary init_lib;
+  init_lib.init();
+  return CreateChannelInternal(
+      "", grpc_insecure_channel_create_from_fd(target.c_str(), fd, nullptr));
+}
+
+#endif  // GPR_SUPPORT_CHANNELS_FROM_FD
+
+}  // namespace grpc
diff --git a/src/cpp/codegen/codegen_init.cc b/src/cpp/codegen/codegen_init.cc
index c5d22124b7..103a7c1258 100644
--- a/src/cpp/codegen/codegen_init.cc
+++ b/src/cpp/codegen/codegen_init.cc
@@ -34,12 +34,12 @@
 #include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/grpc_library.h>
 
-/// Initializes the global gRPC variables for the codegen library. These will
+/// Null-initializes the global gRPC variables for the codegen library. These
 /// stay null in the absence of of grpc++ library. In this case, no gRPC
 /// features such as the ability to perform calls will be available. Trying to
 /// perform them would result in a segmentation fault when trying to deference
 /// the following nulled globals. These should be associated with actual
 /// as part of the instantiation of a \a grpc::GrpcLibraryInitializer variable.
 
-grpc::CoreCodegenInterface* grpc::g_core_codegen_interface = nullptr;
-grpc::GrpcLibraryInterface* grpc::g_glip = nullptr;
+grpc::CoreCodegenInterface* grpc::g_core_codegen_interface;
+grpc::GrpcLibraryInterface* grpc::g_glip;
diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc
index db3558f192..f297ae8587 100644
--- a/src/cpp/common/channel_arguments.cc
+++ b/src/cpp/common/channel_arguments.cc
@@ -85,7 +85,7 @@ void ChannelArguments::Swap(ChannelArguments& other) {
 
 void ChannelArguments::SetCompressionAlgorithm(
     grpc_compression_algorithm algorithm) {
-  SetInt(GRPC_COMPRESSION_ALGORITHM_ARG, algorithm);
+  SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm);
 }
 
 // Note: a second call to this will add in front the result of the first call.
diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc
index 8e8d42eb29..cc35aa69ba 100644
--- a/src/cpp/common/core_codegen.cc
+++ b/src/cpp/common/core_codegen.cc
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/cpp/common/core_codegen.h"
+#include <grpc++/impl/codegen/core_codegen.h>
 
 #include <stdlib.h>
 
diff --git a/src/cpp/common/core_codegen.h b/src/cpp/common/core_codegen.h
deleted file mode 100644
index 656b11e7e7..0000000000
--- a/src/cpp/common/core_codegen.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- *
- * Copyright 2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-// This file should be compiled as part of grpc++.
-
-#include <grpc++/impl/codegen/core_codegen_interface.h>
-#include <grpc/byte_buffer.h>
-#include <grpc/impl/codegen/grpc_types.h>
-
-namespace grpc {
-
-/// Implementation of the core codegen interface.
-class CoreCodegen : public CoreCodegenInterface {
- private:
-  grpc_completion_queue* grpc_completion_queue_create(void* reserved) override;
-  void grpc_completion_queue_destroy(grpc_completion_queue* cq) override;
-  grpc_event grpc_completion_queue_pluck(grpc_completion_queue* cq, void* tag,
-                                         gpr_timespec deadline,
-                                         void* reserved) override;
-
-  void* gpr_malloc(size_t size) override;
-  void gpr_free(void* p) override;
-
-  void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) override;
-
-  void grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader,
-                                    grpc_byte_buffer* buffer) override;
-  void grpc_byte_buffer_reader_destroy(
-      grpc_byte_buffer_reader* reader) override;
-  int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader* reader,
-                                   gpr_slice* slice) override;
-
-  grpc_byte_buffer* grpc_raw_byte_buffer_create(gpr_slice* slice,
-                                                size_t nslices) override;
-
-  gpr_slice gpr_slice_malloc(size_t length) override;
-  void gpr_slice_unref(gpr_slice slice) override;
-  gpr_slice gpr_slice_split_tail(gpr_slice* s, size_t split) override;
-  void gpr_slice_buffer_add(gpr_slice_buffer* sb, gpr_slice slice) override;
-  void gpr_slice_buffer_pop(gpr_slice_buffer* sb) override;
-
-  void grpc_metadata_array_init(grpc_metadata_array* array) override;
-  void grpc_metadata_array_destroy(grpc_metadata_array* array) override;
-
-  gpr_timespec gpr_inf_future(gpr_clock_type type) override;
-
-  virtual const Status& ok() override;
-  virtual const Status& cancelled() override;
-
-  void assert_fail(const char* failed_assertion) override;
-};
-
-}  // namespace grpc
diff --git a/src/cpp/ext/proto_server_reflection.cc b/src/cpp/ext/proto_server_reflection.cc
new file mode 100644
index 0000000000..3973bfb58e
--- /dev/null
+++ b/src/cpp/ext/proto_server_reflection.cc
@@ -0,0 +1,227 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <unordered_set>
+#include <vector>
+
+#include <grpc++/grpc++.h>
+
+#include "src/cpp/ext/proto_server_reflection.h"
+
+using grpc::Status;
+using grpc::StatusCode;
+using grpc::reflection::v1alpha::ServerReflectionRequest;
+using grpc::reflection::v1alpha::ExtensionRequest;
+using grpc::reflection::v1alpha::ServerReflectionResponse;
+using grpc::reflection::v1alpha::ListServiceResponse;
+using grpc::reflection::v1alpha::ServiceResponse;
+using grpc::reflection::v1alpha::ExtensionNumberResponse;
+using grpc::reflection::v1alpha::ErrorResponse;
+using grpc::reflection::v1alpha::FileDescriptorResponse;
+
+namespace grpc {
+
+ProtoServerReflection::ProtoServerReflection()
+    : descriptor_pool_(protobuf::DescriptorPool::generated_pool()) {}
+
+void ProtoServerReflection::SetServiceList(
+    const std::vector<grpc::string>* services) {
+  services_ = services;
+}
+
+Status ProtoServerReflection::ServerReflectionInfo(
+    ServerContext* context,
+    ServerReaderWriter<ServerReflectionResponse, ServerReflectionRequest>*
+        stream) {
+  ServerReflectionRequest request;
+  ServerReflectionResponse response;
+  Status status;
+  while (stream->Read(&request)) {
+    switch (request.message_request_case()) {
+      case ServerReflectionRequest::MessageRequestCase::kFileByFilename:
+        status = GetFileByName(context, request.file_by_filename(), &response);
+        break;
+      case ServerReflectionRequest::MessageRequestCase::kFileContainingSymbol:
+        status = GetFileContainingSymbol(
+            context, request.file_containing_symbol(), &response);
+        break;
+      case ServerReflectionRequest::MessageRequestCase::
+          kFileContainingExtension:
+        status = GetFileContainingExtension(
+            context, &request.file_containing_extension(), &response);
+        break;
+      case ServerReflectionRequest::MessageRequestCase::
+          kAllExtensionNumbersOfType:
+        status = GetAllExtensionNumbers(
+            context, request.all_extension_numbers_of_type(),
+            response.mutable_all_extension_numbers_response());
+        break;
+      case ServerReflectionRequest::MessageRequestCase::kListServices:
+        status =
+            ListService(context, response.mutable_list_services_response());
+        break;
+      default:
+        status = Status(StatusCode::UNIMPLEMENTED, "");
+    }
+
+    if (!status.ok()) {
+      FillErrorResponse(status, response.mutable_error_response());
+    }
+    response.set_valid_host(request.host());
+    response.set_allocated_original_request(
+        new ServerReflectionRequest(request));
+    stream->Write(response);
+  }
+
+  return Status::OK;
+}
+
+void ProtoServerReflection::FillErrorResponse(const Status& status,
+                                              ErrorResponse* error_response) {
+  error_response->set_error_code(status.error_code());
+  error_response->set_error_message(status.error_message());
+}
+
+Status ProtoServerReflection::ListService(ServerContext* context,
+                                          ListServiceResponse* response) {
+  if (services_ == nullptr) {
+    return Status(StatusCode::NOT_FOUND, "Services not found.");
+  }
+  for (auto it = services_->begin(); it != services_->end(); ++it) {
+    ServiceResponse* service_response = response->add_service();
+    service_response->set_name(*it);
+  }
+  return Status::OK;
+}
+
+Status ProtoServerReflection::GetFileByName(
+    ServerContext* context, const grpc::string& filename,
+    ServerReflectionResponse* response) {
+  if (descriptor_pool_ == nullptr) {
+    return Status::CANCELLED;
+  }
+
+  const protobuf::FileDescriptor* file_desc =
+      descriptor_pool_->FindFileByName(filename);
+  if (file_desc == nullptr) {
+    return Status(StatusCode::NOT_FOUND, "File not found.");
+  }
+  std::unordered_set<grpc::string> seen_files;
+  FillFileDescriptorResponse(file_desc, response, &seen_files);
+  return Status::OK;
+}
+
+Status ProtoServerReflection::GetFileContainingSymbol(
+    ServerContext* context, const grpc::string& symbol,
+    ServerReflectionResponse* response) {
+  if (descriptor_pool_ == nullptr) {
+    return Status::CANCELLED;
+  }
+
+  const protobuf::FileDescriptor* file_desc =
+      descriptor_pool_->FindFileContainingSymbol(symbol);
+  if (file_desc == nullptr) {
+    return Status(StatusCode::NOT_FOUND, "Symbol not found.");
+  }
+  std::unordered_set<grpc::string> seen_files;
+  FillFileDescriptorResponse(file_desc, response, &seen_files);
+  return Status::OK;
+}
+
+Status ProtoServerReflection::GetFileContainingExtension(
+    ServerContext* context, const ExtensionRequest* request,
+    ServerReflectionResponse* response) {
+  if (descriptor_pool_ == nullptr) {
+    return Status::CANCELLED;
+  }
+
+  const protobuf::Descriptor* desc =
+      descriptor_pool_->FindMessageTypeByName(request->containing_type());
+  if (desc == nullptr) {
+    return Status(StatusCode::NOT_FOUND, "Type not found.");
+  }
+
+  const protobuf::FieldDescriptor* field_desc =
+      descriptor_pool_->FindExtensionByNumber(desc,
+                                              request->extension_number());
+  if (field_desc == nullptr) {
+    return Status(StatusCode::NOT_FOUND, "Extension not found.");
+  }
+  std::unordered_set<grpc::string> seen_files;
+  FillFileDescriptorResponse(field_desc->file(), response, &seen_files);
+  return Status::OK;
+}
+
+Status ProtoServerReflection::GetAllExtensionNumbers(
+    ServerContext* context, const grpc::string& type,
+    ExtensionNumberResponse* response) {
+  if (descriptor_pool_ == nullptr) {
+    return Status::CANCELLED;
+  }
+
+  const protobuf::Descriptor* desc =
+      descriptor_pool_->FindMessageTypeByName(type);
+  if (desc == nullptr) {
+    return Status(StatusCode::NOT_FOUND, "Type not found.");
+  }
+
+  std::vector<const protobuf::FieldDescriptor*> extensions;
+  descriptor_pool_->FindAllExtensions(desc, &extensions);
+  for (auto it = extensions.begin(); it != extensions.end(); it++) {
+    response->add_extension_number((*it)->number());
+  }
+  response->set_base_type_name(type);
+  return Status::OK;
+}
+
+void ProtoServerReflection::FillFileDescriptorResponse(
+    const protobuf::FileDescriptor* file_desc,
+    ServerReflectionResponse* response,
+    std::unordered_set<grpc::string>* seen_files) {
+  if (seen_files->find(file_desc->name()) != seen_files->end()) {
+    return;
+  }
+  seen_files->insert(file_desc->name());
+
+  protobuf::FileDescriptorProto file_desc_proto;
+  grpc::string data;
+  file_desc->CopyTo(&file_desc_proto);
+  file_desc_proto.SerializeToString(&data);
+  response->mutable_file_descriptor_response()->add_file_descriptor_proto(data);
+
+  for (int i = 0; i < file_desc->dependency_count(); ++i) {
+    FillFileDescriptorResponse(file_desc->dependency(i), response, seen_files);
+  }
+}
+
+}  // namespace grpc
diff --git a/src/cpp/ext/proto_server_reflection.h b/src/cpp/ext/proto_server_reflection.h
new file mode 100644
index 0000000000..23c130513d
--- /dev/null
+++ b/src/cpp/ext/proto_server_reflection.h
@@ -0,0 +1,94 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef GRPC_INTERNAL_CPP_EXT_PROTO_SERVER_REFLECTION_H
+#define GRPC_INTERNAL_CPP_EXT_PROTO_SERVER_REFLECTION_H
+
+#include <unordered_set>
+#include <vector>
+
+#include <grpc++/ext/reflection.grpc.pb.h>
+#include <grpc++/grpc++.h>
+
+namespace grpc {
+
+class ProtoServerReflection GRPC_FINAL
+    : public reflection::v1alpha::ServerReflection::Service {
+ public:
+  ProtoServerReflection();
+
+  // Add the full names of registered services
+  void SetServiceList(const std::vector<grpc::string>* services);
+
+  // implementation of ServerReflectionInfo(stream ServerReflectionRequest) rpc
+  // in ServerReflection service
+  Status ServerReflectionInfo(
+      ServerContext* context,
+      ServerReaderWriter<reflection::v1alpha::ServerReflectionResponse,
+                         reflection::v1alpha::ServerReflectionRequest>* stream)
+      GRPC_OVERRIDE;
+
+ private:
+  Status ListService(ServerContext* context,
+                     reflection::v1alpha::ListServiceResponse* response);
+
+  Status GetFileByName(ServerContext* context, const grpc::string& file_name,
+                       reflection::v1alpha::ServerReflectionResponse* response);
+
+  Status GetFileContainingSymbol(
+      ServerContext* context, const grpc::string& symbol,
+      reflection::v1alpha::ServerReflectionResponse* response);
+
+  Status GetFileContainingExtension(
+      ServerContext* context,
+      const reflection::v1alpha::ExtensionRequest* request,
+      reflection::v1alpha::ServerReflectionResponse* response);
+
+  Status GetAllExtensionNumbers(
+      ServerContext* context, const grpc::string& type,
+      reflection::v1alpha::ExtensionNumberResponse* response);
+
+  void FillFileDescriptorResponse(
+      const protobuf::FileDescriptor* file_desc,
+      reflection::v1alpha::ServerReflectionResponse* response,
+      std::unordered_set<grpc::string>* seen_files);
+
+  void FillErrorResponse(const Status& status,
+                         reflection::v1alpha::ErrorResponse* error_response);
+
+  const protobuf::DescriptorPool* descriptor_pool_;
+  const std::vector<string>* services_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPC_INTERNAL_CPP_EXT_PROTO_SERVER_REFLECTION_H
diff --git a/src/cpp/ext/proto_server_reflection_plugin.cc b/src/cpp/ext/proto_server_reflection_plugin.cc
new file mode 100644
index 0000000000..5b806ce1ae
--- /dev/null
+++ b/src/cpp/ext/proto_server_reflection_plugin.cc
@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/ext/proto_server_reflection_plugin.h>
+#include <grpc++/impl/server_builder_plugin.h>
+#include <grpc++/impl/server_initializer.h>
+#include <grpc++/server.h>
+
+#include "src/cpp/ext/proto_server_reflection.h"
+
+namespace grpc {
+namespace reflection {
+
+ProtoServerReflectionPlugin::ProtoServerReflectionPlugin()
+    : reflection_service_(new grpc::ProtoServerReflection()) {}
+
+grpc::string ProtoServerReflectionPlugin::name() {
+  return "proto_server_reflection";
+}
+
+void ProtoServerReflectionPlugin::InitServer(grpc::ServerInitializer* si) {
+  si->RegisterService(reflection_service_);
+}
+
+void ProtoServerReflectionPlugin::Finish(grpc::ServerInitializer* si) {
+  reflection_service_->SetServiceList(si->GetServiceList());
+}
+
+void ProtoServerReflectionPlugin::ChangeArguments(const grpc::string& name,
+                                                  void* value) {}
+
+bool ProtoServerReflectionPlugin::has_sync_methods() const {
+  if (reflection_service_) {
+    return reflection_service_->has_synchronous_methods();
+  }
+  return false;
+}
+
+bool ProtoServerReflectionPlugin::has_async_methods() const {
+  if (reflection_service_) {
+    return reflection_service_->has_async_methods();
+  }
+  return false;
+}
+
+static std::unique_ptr< ::grpc::ServerBuilderPlugin> CreateProtoReflection() {
+  return std::unique_ptr< ::grpc::ServerBuilderPlugin>(
+      new ProtoServerReflectionPlugin());
+}
+
+void InitProtoReflectionServerBuilderPlugin() {
+  static bool already_here = false;
+  if (already_here) return;
+  already_here = true;
+  ::grpc::ServerBuilder::InternalAddPluginFactory(&CreateProtoReflection);
+}
+
+// Force InitProtoReflectionServerBuilderPlugin() to be called at static
+// initialization time.
+struct StaticProtoReflectionPluginInitializer {
+  StaticProtoReflectionPluginInitializer() {
+    InitProtoReflectionServerBuilderPlugin();
+  }
+} static_proto_reflection_plugin_initializer;
+
+}  // namespace reflection
+}  // namespace grpc
diff --git a/src/cpp/ext/reflection.grpc.pb.cc b/src/cpp/ext/reflection.grpc.pb.cc
new file mode 100644
index 0000000000..b046dfc1b8
--- /dev/null
+++ b/src/cpp/ext/reflection.grpc.pb.cc
@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+
+// Generated by the gRPC protobuf plugin.
+// If you make any local change, they will be lost.
+// source: reflection.proto
+
+#include <grpc++/ext/reflection.pb.h>
+#include <grpc++/ext/reflection.grpc.pb.h>
+
+#include <grpc++/impl/codegen/async_stream.h>
+#include <grpc++/impl/codegen/async_unary_call.h>
+#include <grpc++/impl/codegen/channel_interface.h>
+#include <grpc++/impl/codegen/client_unary_call.h>
+#include <grpc++/impl/codegen/method_handler_impl.h>
+#include <grpc++/impl/codegen/rpc_service_method.h>
+#include <grpc++/impl/codegen/service_type.h>
+#include <grpc++/impl/codegen/sync_stream.h>
+namespace grpc {
+namespace reflection {
+namespace v1alpha {
+
+static const char* ServerReflection_method_names[] = {
+  "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo",
+};
+
+std::unique_ptr< ServerReflection::Stub> ServerReflection::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) {
+  std::unique_ptr< ServerReflection::Stub> stub(new ServerReflection::Stub(channel));
+  return stub;
+}
+
+ServerReflection::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel)
+  : channel_(channel), rpcmethod_ServerReflectionInfo_(ServerReflection_method_names[0], ::grpc::RpcMethod::BIDI_STREAMING, channel)
+  {}
+
+::grpc::ClientReaderWriter< ::grpc::reflection::v1alpha::ServerReflectionRequest, ::grpc::reflection::v1alpha::ServerReflectionResponse>* ServerReflection::Stub::ServerReflectionInfoRaw(::grpc::ClientContext* context) {
+  return new ::grpc::ClientReaderWriter< ::grpc::reflection::v1alpha::ServerReflectionRequest, ::grpc::reflection::v1alpha::ServerReflectionResponse>(channel_.get(), rpcmethod_ServerReflectionInfo_, context);
+}
+
+::grpc::ClientAsyncReaderWriter< ::grpc::reflection::v1alpha::ServerReflectionRequest, ::grpc::reflection::v1alpha::ServerReflectionResponse>* ServerReflection::Stub::AsyncServerReflectionInfoRaw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) {
+  return new ::grpc::ClientAsyncReaderWriter< ::grpc::reflection::v1alpha::ServerReflectionRequest, ::grpc::reflection::v1alpha::ServerReflectionResponse>(channel_.get(), cq, rpcmethod_ServerReflectionInfo_, context, tag);
+}
+
+ServerReflection::Service::Service() {
+  (void)ServerReflection_method_names;
+  AddMethod(new ::grpc::RpcServiceMethod(
+      ServerReflection_method_names[0],
+      ::grpc::RpcMethod::BIDI_STREAMING,
+      new ::grpc::BidiStreamingHandler< ServerReflection::Service, ::grpc::reflection::v1alpha::ServerReflectionRequest, ::grpc::reflection::v1alpha::ServerReflectionResponse>(
+          std::mem_fn(&ServerReflection::Service::ServerReflectionInfo), this)));
+}
+
+ServerReflection::Service::~Service() {
+}
+
+::grpc::Status ServerReflection::Service::ServerReflectionInfo(::grpc::ServerContext* context, ::grpc::ServerReaderWriter< ::grpc::reflection::v1alpha::ServerReflectionResponse, ::grpc::reflection::v1alpha::ServerReflectionRequest>* stream) {
+  (void) context;
+  (void) stream;
+  return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
+}
+
+
+}  // namespace grpc
+}  // namespace reflection
+}  // namespace v1alpha
+
diff --git a/src/cpp/ext/reflection.pb.cc b/src/cpp/ext/reflection.pb.cc
new file mode 100644
index 0000000000..b73a65d0a0
--- /dev/null
+++ b/src/cpp/ext/reflection.pb.cc
@@ -0,0 +1,3946 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: reflection.proto
+
+#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION
+#include <grpc++/ext/reflection.pb.h>
+
+#include <algorithm>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/port.h>
+#include <google/protobuf/stubs/once.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/wire_format_lite_inl.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/generated_message_reflection.h>
+#include <google/protobuf/reflection_ops.h>
+#include <google/protobuf/wire_format.h>
+// @@protoc_insertion_point(includes)
+
+namespace grpc {
+namespace reflection {
+namespace v1alpha {
+
+namespace {
+
+const ::google::protobuf::Descriptor* ServerReflectionRequest_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ServerReflectionRequest_reflection_ = NULL;
+struct ServerReflectionRequestOneofInstance {
+  ::google::protobuf::internal::ArenaStringPtr file_by_filename_;
+  ::google::protobuf::internal::ArenaStringPtr file_containing_symbol_;
+  const ::grpc::reflection::v1alpha::ExtensionRequest* file_containing_extension_;
+  ::google::protobuf::internal::ArenaStringPtr all_extension_numbers_of_type_;
+  ::google::protobuf::internal::ArenaStringPtr list_services_;
+}* ServerReflectionRequest_default_oneof_instance_ = NULL;
+const ::google::protobuf::Descriptor* ExtensionRequest_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ExtensionRequest_reflection_ = NULL;
+const ::google::protobuf::Descriptor* ServerReflectionResponse_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ServerReflectionResponse_reflection_ = NULL;
+struct ServerReflectionResponseOneofInstance {
+  const ::grpc::reflection::v1alpha::FileDescriptorResponse* file_descriptor_response_;
+  const ::grpc::reflection::v1alpha::ExtensionNumberResponse* all_extension_numbers_response_;
+  const ::grpc::reflection::v1alpha::ListServiceResponse* list_services_response_;
+  const ::grpc::reflection::v1alpha::ErrorResponse* error_response_;
+}* ServerReflectionResponse_default_oneof_instance_ = NULL;
+const ::google::protobuf::Descriptor* FileDescriptorResponse_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  FileDescriptorResponse_reflection_ = NULL;
+const ::google::protobuf::Descriptor* ExtensionNumberResponse_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ExtensionNumberResponse_reflection_ = NULL;
+const ::google::protobuf::Descriptor* ListServiceResponse_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ListServiceResponse_reflection_ = NULL;
+const ::google::protobuf::Descriptor* ServiceResponse_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ServiceResponse_reflection_ = NULL;
+const ::google::protobuf::Descriptor* ErrorResponse_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+  ErrorResponse_reflection_ = NULL;
+
+}  // namespace
+
+
+void protobuf_AssignDesc_reflection_2eproto() {
+  protobuf_AddDesc_reflection_2eproto();
+  const ::google::protobuf::FileDescriptor* file =
+    ::google::protobuf::DescriptorPool::generated_pool()->FindFileByName(
+      "reflection.proto");
+  GOOGLE_CHECK(file != NULL);
+  ServerReflectionRequest_descriptor_ = file->message_type(0);
+  static const int ServerReflectionRequest_offsets_[7] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionRequest, host_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionRequest_default_oneof_instance_, file_by_filename_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionRequest_default_oneof_instance_, file_containing_symbol_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionRequest_default_oneof_instance_, file_containing_extension_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionRequest_default_oneof_instance_, all_extension_numbers_of_type_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionRequest_default_oneof_instance_, list_services_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionRequest, message_request_),
+  };
+  ServerReflectionRequest_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ServerReflectionRequest_descriptor_,
+      ServerReflectionRequest::default_instance_,
+      ServerReflectionRequest_offsets_,
+      -1,
+      -1,
+      -1,
+      ServerReflectionRequest_default_oneof_instance_,
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionRequest, _oneof_case_[0]),
+      sizeof(ServerReflectionRequest),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionRequest, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionRequest, _is_default_instance_));
+  ExtensionRequest_descriptor_ = file->message_type(1);
+  static const int ExtensionRequest_offsets_[2] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionRequest, containing_type_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionRequest, extension_number_),
+  };
+  ExtensionRequest_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ExtensionRequest_descriptor_,
+      ExtensionRequest::default_instance_,
+      ExtensionRequest_offsets_,
+      -1,
+      -1,
+      -1,
+      sizeof(ExtensionRequest),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionRequest, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionRequest, _is_default_instance_));
+  ServerReflectionResponse_descriptor_ = file->message_type(2);
+  static const int ServerReflectionResponse_offsets_[7] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionResponse, valid_host_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionResponse, original_request_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionResponse_default_oneof_instance_, file_descriptor_response_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionResponse_default_oneof_instance_, all_extension_numbers_response_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionResponse_default_oneof_instance_, list_services_response_),
+    PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET(ServerReflectionResponse_default_oneof_instance_, error_response_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionResponse, message_response_),
+  };
+  ServerReflectionResponse_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ServerReflectionResponse_descriptor_,
+      ServerReflectionResponse::default_instance_,
+      ServerReflectionResponse_offsets_,
+      -1,
+      -1,
+      -1,
+      ServerReflectionResponse_default_oneof_instance_,
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionResponse, _oneof_case_[0]),
+      sizeof(ServerReflectionResponse),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionResponse, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServerReflectionResponse, _is_default_instance_));
+  FileDescriptorResponse_descriptor_ = file->message_type(3);
+  static const int FileDescriptorResponse_offsets_[1] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FileDescriptorResponse, file_descriptor_proto_),
+  };
+  FileDescriptorResponse_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      FileDescriptorResponse_descriptor_,
+      FileDescriptorResponse::default_instance_,
+      FileDescriptorResponse_offsets_,
+      -1,
+      -1,
+      -1,
+      sizeof(FileDescriptorResponse),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FileDescriptorResponse, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FileDescriptorResponse, _is_default_instance_));
+  ExtensionNumberResponse_descriptor_ = file->message_type(4);
+  static const int ExtensionNumberResponse_offsets_[2] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionNumberResponse, base_type_name_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionNumberResponse, extension_number_),
+  };
+  ExtensionNumberResponse_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ExtensionNumberResponse_descriptor_,
+      ExtensionNumberResponse::default_instance_,
+      ExtensionNumberResponse_offsets_,
+      -1,
+      -1,
+      -1,
+      sizeof(ExtensionNumberResponse),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionNumberResponse, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ExtensionNumberResponse, _is_default_instance_));
+  ListServiceResponse_descriptor_ = file->message_type(5);
+  static const int ListServiceResponse_offsets_[1] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ListServiceResponse, service_),
+  };
+  ListServiceResponse_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ListServiceResponse_descriptor_,
+      ListServiceResponse::default_instance_,
+      ListServiceResponse_offsets_,
+      -1,
+      -1,
+      -1,
+      sizeof(ListServiceResponse),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ListServiceResponse, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ListServiceResponse, _is_default_instance_));
+  ServiceResponse_descriptor_ = file->message_type(6);
+  static const int ServiceResponse_offsets_[1] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServiceResponse, name_),
+  };
+  ServiceResponse_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ServiceResponse_descriptor_,
+      ServiceResponse::default_instance_,
+      ServiceResponse_offsets_,
+      -1,
+      -1,
+      -1,
+      sizeof(ServiceResponse),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServiceResponse, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ServiceResponse, _is_default_instance_));
+  ErrorResponse_descriptor_ = file->message_type(7);
+  static const int ErrorResponse_offsets_[2] = {
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ErrorResponse, error_code_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ErrorResponse, error_message_),
+  };
+  ErrorResponse_reflection_ =
+    ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection(
+      ErrorResponse_descriptor_,
+      ErrorResponse::default_instance_,
+      ErrorResponse_offsets_,
+      -1,
+      -1,
+      -1,
+      sizeof(ErrorResponse),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ErrorResponse, _internal_metadata_),
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(ErrorResponse, _is_default_instance_));
+}
+
+namespace {
+
+GOOGLE_PROTOBUF_DECLARE_ONCE(protobuf_AssignDescriptors_once_);
+inline void protobuf_AssignDescriptorsOnce() {
+  ::google::protobuf::GoogleOnceInit(&protobuf_AssignDescriptors_once_,
+                 &protobuf_AssignDesc_reflection_2eproto);
+}
+
+void protobuf_RegisterTypes(const ::std::string&) {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ServerReflectionRequest_descriptor_, &ServerReflectionRequest::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ExtensionRequest_descriptor_, &ExtensionRequest::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ServerReflectionResponse_descriptor_, &ServerReflectionResponse::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      FileDescriptorResponse_descriptor_, &FileDescriptorResponse::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ExtensionNumberResponse_descriptor_, &ExtensionNumberResponse::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ListServiceResponse_descriptor_, &ListServiceResponse::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ServiceResponse_descriptor_, &ServiceResponse::default_instance());
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+      ErrorResponse_descriptor_, &ErrorResponse::default_instance());
+}
+
+}  // namespace
+
+void protobuf_ShutdownFile_reflection_2eproto() {
+  delete ServerReflectionRequest::default_instance_;
+  delete ServerReflectionRequest_default_oneof_instance_;
+  delete ServerReflectionRequest_reflection_;
+  delete ExtensionRequest::default_instance_;
+  delete ExtensionRequest_reflection_;
+  delete ServerReflectionResponse::default_instance_;
+  delete ServerReflectionResponse_default_oneof_instance_;
+  delete ServerReflectionResponse_reflection_;
+  delete FileDescriptorResponse::default_instance_;
+  delete FileDescriptorResponse_reflection_;
+  delete ExtensionNumberResponse::default_instance_;
+  delete ExtensionNumberResponse_reflection_;
+  delete ListServiceResponse::default_instance_;
+  delete ListServiceResponse_reflection_;
+  delete ServiceResponse::default_instance_;
+  delete ServiceResponse_reflection_;
+  delete ErrorResponse::default_instance_;
+  delete ErrorResponse_reflection_;
+}
+
+void protobuf_AddDesc_reflection_2eproto() {
+  static bool already_here = false;
+  if (already_here) return;
+  already_here = true;
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
+
+  ::google::protobuf::DescriptorPool::InternalAddGeneratedFile(
+    "\n\020reflection.proto\022\027grpc.reflection.v1al"
+    "pha\"\212\002\n\027ServerReflectionRequest\022\014\n\004host\030"
+    "\001 \001(\t\022\032\n\020file_by_filename\030\003 \001(\tH\000\022 \n\026fil"
+    "e_containing_symbol\030\004 \001(\tH\000\022N\n\031file_cont"
+    "aining_extension\030\005 \001(\0132).grpc.reflection"
+    ".v1alpha.ExtensionRequestH\000\022\'\n\035all_exten"
+    "sion_numbers_of_type\030\006 \001(\tH\000\022\027\n\rlist_ser"
+    "vices\030\007 \001(\tH\000B\021\n\017message_request\"E\n\020Exte"
+    "nsionRequest\022\027\n\017containing_type\030\001 \001(\t\022\030\n"
+    "\020extension_number\030\002 \001(\005\"\321\003\n\030ServerReflec"
+    "tionResponse\022\022\n\nvalid_host\030\001 \001(\t\022J\n\020orig"
+    "inal_request\030\002 \001(\01320.grpc.reflection.v1a"
+    "lpha.ServerReflectionRequest\022S\n\030file_des"
+    "criptor_response\030\004 \001(\0132/.grpc.reflection"
+    ".v1alpha.FileDescriptorResponseH\000\022Z\n\036all"
+    "_extension_numbers_response\030\005 \001(\01320.grpc"
+    ".reflection.v1alpha.ExtensionNumberRespo"
+    "nseH\000\022N\n\026list_services_response\030\006 \001(\0132,."
+    "grpc.reflection.v1alpha.ListServiceRespo"
+    "nseH\000\022@\n\016error_response\030\007 \001(\0132&.grpc.ref"
+    "lection.v1alpha.ErrorResponseH\000B\022\n\020messa"
+    "ge_response\"7\n\026FileDescriptorResponse\022\035\n"
+    "\025file_descriptor_proto\030\001 \003(\014\"K\n\027Extensio"
+    "nNumberResponse\022\026\n\016base_type_name\030\001 \001(\t\022"
+    "\030\n\020extension_number\030\002 \003(\005\"P\n\023ListService"
+    "Response\0229\n\007service\030\001 \003(\0132(.grpc.reflect"
+    "ion.v1alpha.ServiceResponse\"\037\n\017ServiceRe"
+    "sponse\022\014\n\004name\030\001 \001(\t\":\n\rErrorResponse\022\022\n"
+    "\nerror_code\030\001 \001(\005\022\025\n\rerror_message\030\002 \001(\t"
+    "2\223\001\n\020ServerReflection\022\177\n\024ServerReflectio"
+    "nInfo\0220.grpc.reflection.v1alpha.ServerRe"
+    "flectionRequest\0321.grpc.reflection.v1alph"
+    "a.ServerReflectionResponse(\0010\001b\006proto3", 1318);
+  ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
+    "reflection.proto", &protobuf_RegisterTypes);
+  ServerReflectionRequest::default_instance_ = new ServerReflectionRequest();
+  ServerReflectionRequest_default_oneof_instance_ = new ServerReflectionRequestOneofInstance();
+  ExtensionRequest::default_instance_ = new ExtensionRequest();
+  ServerReflectionResponse::default_instance_ = new ServerReflectionResponse();
+  ServerReflectionResponse_default_oneof_instance_ = new ServerReflectionResponseOneofInstance();
+  FileDescriptorResponse::default_instance_ = new FileDescriptorResponse();
+  ExtensionNumberResponse::default_instance_ = new ExtensionNumberResponse();
+  ListServiceResponse::default_instance_ = new ListServiceResponse();
+  ServiceResponse::default_instance_ = new ServiceResponse();
+  ErrorResponse::default_instance_ = new ErrorResponse();
+  ServerReflectionRequest::default_instance_->InitAsDefaultInstance();
+  ExtensionRequest::default_instance_->InitAsDefaultInstance();
+  ServerReflectionResponse::default_instance_->InitAsDefaultInstance();
+  FileDescriptorResponse::default_instance_->InitAsDefaultInstance();
+  ExtensionNumberResponse::default_instance_->InitAsDefaultInstance();
+  ListServiceResponse::default_instance_->InitAsDefaultInstance();
+  ServiceResponse::default_instance_->InitAsDefaultInstance();
+  ErrorResponse::default_instance_->InitAsDefaultInstance();
+  ::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_reflection_2eproto);
+}
+
+// Force AddDescriptors() to be called at static initialization time.
+struct StaticDescriptorInitializer_reflection_2eproto {
+  StaticDescriptorInitializer_reflection_2eproto() {
+    protobuf_AddDesc_reflection_2eproto();
+  }
+} static_descriptor_initializer_reflection_2eproto_;
+
+namespace {
+
+static void MergeFromFail(int line) GOOGLE_ATTRIBUTE_COLD;
+static void MergeFromFail(int line) {
+  GOOGLE_CHECK(false) << __FILE__ << ":" << line;
+}
+
+}  // namespace
+
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ServerReflectionRequest::kHostFieldNumber;
+const int ServerReflectionRequest::kFileByFilenameFieldNumber;
+const int ServerReflectionRequest::kFileContainingSymbolFieldNumber;
+const int ServerReflectionRequest::kFileContainingExtensionFieldNumber;
+const int ServerReflectionRequest::kAllExtensionNumbersOfTypeFieldNumber;
+const int ServerReflectionRequest::kListServicesFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ServerReflectionRequest::ServerReflectionRequest()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ServerReflectionRequest)
+}
+
+void ServerReflectionRequest::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+  ServerReflectionRequest_default_oneof_instance_->file_by_filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  ServerReflectionRequest_default_oneof_instance_->file_containing_symbol_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  ServerReflectionRequest_default_oneof_instance_->file_containing_extension_ = const_cast< ::grpc::reflection::v1alpha::ExtensionRequest*>(&::grpc::reflection::v1alpha::ExtensionRequest::default_instance());
+  ServerReflectionRequest_default_oneof_instance_->all_extension_numbers_of_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  ServerReflectionRequest_default_oneof_instance_->list_services_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+
+ServerReflectionRequest::ServerReflectionRequest(const ServerReflectionRequest& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ServerReflectionRequest)
+}
+
+void ServerReflectionRequest::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+  host_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  clear_has_message_request();
+}
+
+ServerReflectionRequest::~ServerReflectionRequest() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ServerReflectionRequest)
+  SharedDtor();
+}
+
+void ServerReflectionRequest::SharedDtor() {
+  host_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (has_message_request()) {
+    clear_message_request();
+  }
+  if (this != default_instance_) {
+  }
+}
+
+void ServerReflectionRequest::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ServerReflectionRequest::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ServerReflectionRequest_descriptor_;
+}
+
+const ServerReflectionRequest& ServerReflectionRequest::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ServerReflectionRequest* ServerReflectionRequest::default_instance_ = NULL;
+
+ServerReflectionRequest* ServerReflectionRequest::New(::google::protobuf::Arena* arena) const {
+  ServerReflectionRequest* n = new ServerReflectionRequest;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ServerReflectionRequest::clear_message_request() {
+// @@protoc_insertion_point(one_of_clear_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  switch(message_request_case()) {
+    case kFileByFilename: {
+      message_request_.file_by_filename_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+      break;
+    }
+    case kFileContainingSymbol: {
+      message_request_.file_containing_symbol_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+      break;
+    }
+    case kFileContainingExtension: {
+      delete message_request_.file_containing_extension_;
+      break;
+    }
+    case kAllExtensionNumbersOfType: {
+      message_request_.all_extension_numbers_of_type_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+      break;
+    }
+    case kListServices: {
+      message_request_.list_services_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+      break;
+    }
+    case MESSAGE_REQUEST_NOT_SET: {
+      break;
+    }
+  }
+  _oneof_case_[0] = MESSAGE_REQUEST_NOT_SET;
+}
+
+
+void ServerReflectionRequest::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  host_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  clear_message_request();
+}
+
+bool ServerReflectionRequest::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // optional string host = 1;
+      case 1: {
+        if (tag == 10) {
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_host()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->host().data(), this->host().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServerReflectionRequest.host"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(26)) goto parse_file_by_filename;
+        break;
+      }
+
+      // optional string file_by_filename = 3;
+      case 3: {
+        if (tag == 26) {
+         parse_file_by_filename:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_file_by_filename()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->file_by_filename().data(), this->file_by_filename().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(34)) goto parse_file_containing_symbol;
+        break;
+      }
+
+      // optional string file_containing_symbol = 4;
+      case 4: {
+        if (tag == 34) {
+         parse_file_containing_symbol:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_file_containing_symbol()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->file_containing_symbol().data(), this->file_containing_symbol().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(42)) goto parse_file_containing_extension;
+        break;
+      }
+
+      // optional .grpc.reflection.v1alpha.ExtensionRequest file_containing_extension = 5;
+      case 5: {
+        if (tag == 42) {
+         parse_file_containing_extension:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+               input, mutable_file_containing_extension()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(50)) goto parse_all_extension_numbers_of_type;
+        break;
+      }
+
+      // optional string all_extension_numbers_of_type = 6;
+      case 6: {
+        if (tag == 50) {
+         parse_all_extension_numbers_of_type:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_all_extension_numbers_of_type()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->all_extension_numbers_of_type().data(), this->all_extension_numbers_of_type().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(58)) goto parse_list_services;
+        break;
+      }
+
+      // optional string list_services = 7;
+      case 7: {
+        if (tag == 58) {
+         parse_list_services:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_list_services()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->list_services().data(), this->list_services().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServerReflectionRequest.list_services"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ServerReflectionRequest)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ServerReflectionRequest)
+  return false;
+#undef DO_
+}
+
+void ServerReflectionRequest::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  // optional string host = 1;
+  if (this->host().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->host().data(), this->host().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.host");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      1, this->host(), output);
+  }
+
+  // optional string file_by_filename = 3;
+  if (has_file_by_filename()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->file_by_filename().data(), this->file_by_filename().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      3, this->file_by_filename(), output);
+  }
+
+  // optional string file_containing_symbol = 4;
+  if (has_file_containing_symbol()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->file_containing_symbol().data(), this->file_containing_symbol().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      4, this->file_containing_symbol(), output);
+  }
+
+  // optional .grpc.reflection.v1alpha.ExtensionRequest file_containing_extension = 5;
+  if (has_file_containing_extension()) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      5, *message_request_.file_containing_extension_, output);
+  }
+
+  // optional string all_extension_numbers_of_type = 6;
+  if (has_all_extension_numbers_of_type()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->all_extension_numbers_of_type().data(), this->all_extension_numbers_of_type().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      6, this->all_extension_numbers_of_type(), output);
+  }
+
+  // optional string list_services = 7;
+  if (has_list_services()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->list_services().data(), this->list_services().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.list_services");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      7, this->list_services(), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ServerReflectionRequest)
+}
+
+::google::protobuf::uint8* ServerReflectionRequest::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  // optional string host = 1;
+  if (this->host().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->host().data(), this->host().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.host");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        1, this->host(), target);
+  }
+
+  // optional string file_by_filename = 3;
+  if (has_file_by_filename()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->file_by_filename().data(), this->file_by_filename().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        3, this->file_by_filename(), target);
+  }
+
+  // optional string file_containing_symbol = 4;
+  if (has_file_containing_symbol()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->file_containing_symbol().data(), this->file_containing_symbol().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        4, this->file_containing_symbol(), target);
+  }
+
+  // optional .grpc.reflection.v1alpha.ExtensionRequest file_containing_extension = 5;
+  if (has_file_containing_extension()) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        5, *message_request_.file_containing_extension_, target);
+  }
+
+  // optional string all_extension_numbers_of_type = 6;
+  if (has_all_extension_numbers_of_type()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->all_extension_numbers_of_type().data(), this->all_extension_numbers_of_type().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        6, this->all_extension_numbers_of_type(), target);
+  }
+
+  // optional string list_services = 7;
+  if (has_list_services()) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->list_services().data(), this->list_services().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionRequest.list_services");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        7, this->list_services(), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ServerReflectionRequest)
+  return target;
+}
+
+int ServerReflectionRequest::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  int total_size = 0;
+
+  // optional string host = 1;
+  if (this->host().size() > 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::StringSize(
+        this->host());
+  }
+
+  switch (message_request_case()) {
+    // optional string file_by_filename = 3;
+    case kFileByFilename: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::StringSize(
+          this->file_by_filename());
+      break;
+    }
+    // optional string file_containing_symbol = 4;
+    case kFileContainingSymbol: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::StringSize(
+          this->file_containing_symbol());
+      break;
+    }
+    // optional .grpc.reflection.v1alpha.ExtensionRequest file_containing_extension = 5;
+    case kFileContainingExtension: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+          *message_request_.file_containing_extension_);
+      break;
+    }
+    // optional string all_extension_numbers_of_type = 6;
+    case kAllExtensionNumbersOfType: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::StringSize(
+          this->all_extension_numbers_of_type());
+      break;
+    }
+    // optional string list_services = 7;
+    case kListServices: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::StringSize(
+          this->list_services());
+      break;
+    }
+    case MESSAGE_REQUEST_NOT_SET: {
+      break;
+    }
+  }
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ServerReflectionRequest::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ServerReflectionRequest* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ServerReflectionRequest>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ServerReflectionRequest)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ServerReflectionRequest)
+    MergeFrom(*source);
+  }
+}
+
+void ServerReflectionRequest::MergeFrom(const ServerReflectionRequest& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  switch (from.message_request_case()) {
+    case kFileByFilename: {
+      set_file_by_filename(from.file_by_filename());
+      break;
+    }
+    case kFileContainingSymbol: {
+      set_file_containing_symbol(from.file_containing_symbol());
+      break;
+    }
+    case kFileContainingExtension: {
+      mutable_file_containing_extension()->::grpc::reflection::v1alpha::ExtensionRequest::MergeFrom(from.file_containing_extension());
+      break;
+    }
+    case kAllExtensionNumbersOfType: {
+      set_all_extension_numbers_of_type(from.all_extension_numbers_of_type());
+      break;
+    }
+    case kListServices: {
+      set_list_services(from.list_services());
+      break;
+    }
+    case MESSAGE_REQUEST_NOT_SET: {
+      break;
+    }
+  }
+  if (from.host().size() > 0) {
+
+    host_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.host_);
+  }
+}
+
+void ServerReflectionRequest::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ServerReflectionRequest::CopyFrom(const ServerReflectionRequest& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ServerReflectionRequest)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ServerReflectionRequest::IsInitialized() const {
+
+  return true;
+}
+
+void ServerReflectionRequest::Swap(ServerReflectionRequest* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ServerReflectionRequest::InternalSwap(ServerReflectionRequest* other) {
+  host_.Swap(&other->host_);
+  std::swap(message_request_, other->message_request_);
+  std::swap(_oneof_case_[0], other->_oneof_case_[0]);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ServerReflectionRequest::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ServerReflectionRequest_descriptor_;
+  metadata.reflection = ServerReflectionRequest_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ServerReflectionRequest
+
+// optional string host = 1;
+void ServerReflectionRequest::clear_host() {
+  host_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ const ::std::string& ServerReflectionRequest::host() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+  return host_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ServerReflectionRequest::set_host(const ::std::string& value) {
+  
+  host_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+}
+ void ServerReflectionRequest::set_host(const char* value) {
+  
+  host_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+}
+ void ServerReflectionRequest::set_host(const char* value, size_t size) {
+  
+  host_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+}
+ ::std::string* ServerReflectionRequest::mutable_host() {
+  
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+  return host_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServerReflectionRequest::release_host() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+  
+  return host_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ServerReflectionRequest::set_allocated_host(::std::string* host) {
+  if (host != NULL) {
+    
+  } else {
+    
+  }
+  host_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), host);
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionRequest.host)
+}
+
+// optional string file_by_filename = 3;
+bool ServerReflectionRequest::has_file_by_filename() const {
+  return message_request_case() == kFileByFilename;
+}
+void ServerReflectionRequest::set_has_file_by_filename() {
+  _oneof_case_[0] = kFileByFilename;
+}
+void ServerReflectionRequest::clear_file_by_filename() {
+  if (has_file_by_filename()) {
+    message_request_.file_by_filename_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+    clear_has_message_request();
+  }
+}
+ const ::std::string& ServerReflectionRequest::file_by_filename() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+  if (has_file_by_filename()) {
+    return message_request_.file_by_filename_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  return *&::google::protobuf::internal::GetEmptyStringAlreadyInited();
+}
+ void ServerReflectionRequest::set_file_by_filename(const ::std::string& value) {
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+  if (!has_file_by_filename()) {
+    clear_message_request();
+    set_has_file_by_filename();
+    message_request_.file_by_filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.file_by_filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+}
+ void ServerReflectionRequest::set_file_by_filename(const char* value) {
+  if (!has_file_by_filename()) {
+    clear_message_request();
+    set_has_file_by_filename();
+    message_request_.file_by_filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.file_by_filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+}
+ void ServerReflectionRequest::set_file_by_filename(const char* value, size_t size) {
+  if (!has_file_by_filename()) {
+    clear_message_request();
+    set_has_file_by_filename();
+    message_request_.file_by_filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.file_by_filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(
+      reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+}
+ ::std::string* ServerReflectionRequest::mutable_file_by_filename() {
+  if (!has_file_by_filename()) {
+    clear_message_request();
+    set_has_file_by_filename();
+    message_request_.file_by_filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+  return message_request_.file_by_filename_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServerReflectionRequest::release_file_by_filename() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+  if (has_file_by_filename()) {
+    clear_has_message_request();
+    return message_request_.file_by_filename_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  } else {
+    return NULL;
+  }
+}
+ void ServerReflectionRequest::set_allocated_file_by_filename(::std::string* file_by_filename) {
+  if (!has_file_by_filename()) {
+    message_request_.file_by_filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  clear_message_request();
+  if (file_by_filename != NULL) {
+    set_has_file_by_filename();
+    message_request_.file_by_filename_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+        file_by_filename);
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionRequest.file_by_filename)
+}
+
+// optional string file_containing_symbol = 4;
+bool ServerReflectionRequest::has_file_containing_symbol() const {
+  return message_request_case() == kFileContainingSymbol;
+}
+void ServerReflectionRequest::set_has_file_containing_symbol() {
+  _oneof_case_[0] = kFileContainingSymbol;
+}
+void ServerReflectionRequest::clear_file_containing_symbol() {
+  if (has_file_containing_symbol()) {
+    message_request_.file_containing_symbol_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+    clear_has_message_request();
+  }
+}
+ const ::std::string& ServerReflectionRequest::file_containing_symbol() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+  if (has_file_containing_symbol()) {
+    return message_request_.file_containing_symbol_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  return *&::google::protobuf::internal::GetEmptyStringAlreadyInited();
+}
+ void ServerReflectionRequest::set_file_containing_symbol(const ::std::string& value) {
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+  if (!has_file_containing_symbol()) {
+    clear_message_request();
+    set_has_file_containing_symbol();
+    message_request_.file_containing_symbol_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.file_containing_symbol_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+}
+ void ServerReflectionRequest::set_file_containing_symbol(const char* value) {
+  if (!has_file_containing_symbol()) {
+    clear_message_request();
+    set_has_file_containing_symbol();
+    message_request_.file_containing_symbol_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.file_containing_symbol_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+}
+ void ServerReflectionRequest::set_file_containing_symbol(const char* value, size_t size) {
+  if (!has_file_containing_symbol()) {
+    clear_message_request();
+    set_has_file_containing_symbol();
+    message_request_.file_containing_symbol_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.file_containing_symbol_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(
+      reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+}
+ ::std::string* ServerReflectionRequest::mutable_file_containing_symbol() {
+  if (!has_file_containing_symbol()) {
+    clear_message_request();
+    set_has_file_containing_symbol();
+    message_request_.file_containing_symbol_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+  return message_request_.file_containing_symbol_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServerReflectionRequest::release_file_containing_symbol() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+  if (has_file_containing_symbol()) {
+    clear_has_message_request();
+    return message_request_.file_containing_symbol_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  } else {
+    return NULL;
+  }
+}
+ void ServerReflectionRequest::set_allocated_file_containing_symbol(::std::string* file_containing_symbol) {
+  if (!has_file_containing_symbol()) {
+    message_request_.file_containing_symbol_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  clear_message_request();
+  if (file_containing_symbol != NULL) {
+    set_has_file_containing_symbol();
+    message_request_.file_containing_symbol_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+        file_containing_symbol);
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_symbol)
+}
+
+// optional .grpc.reflection.v1alpha.ExtensionRequest file_containing_extension = 5;
+bool ServerReflectionRequest::has_file_containing_extension() const {
+  return message_request_case() == kFileContainingExtension;
+}
+void ServerReflectionRequest::set_has_file_containing_extension() {
+  _oneof_case_[0] = kFileContainingExtension;
+}
+void ServerReflectionRequest::clear_file_containing_extension() {
+  if (has_file_containing_extension()) {
+    delete message_request_.file_containing_extension_;
+    clear_has_message_request();
+  }
+}
+ const ::grpc::reflection::v1alpha::ExtensionRequest& ServerReflectionRequest::file_containing_extension() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension)
+  return has_file_containing_extension()
+      ? *message_request_.file_containing_extension_
+      : ::grpc::reflection::v1alpha::ExtensionRequest::default_instance();
+}
+::grpc::reflection::v1alpha::ExtensionRequest* ServerReflectionRequest::mutable_file_containing_extension() {
+  if (!has_file_containing_extension()) {
+    clear_message_request();
+    set_has_file_containing_extension();
+    message_request_.file_containing_extension_ = new ::grpc::reflection::v1alpha::ExtensionRequest;
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension)
+  return message_request_.file_containing_extension_;
+}
+::grpc::reflection::v1alpha::ExtensionRequest* ServerReflectionRequest::release_file_containing_extension() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension)
+  if (has_file_containing_extension()) {
+    clear_has_message_request();
+    ::grpc::reflection::v1alpha::ExtensionRequest* temp = message_request_.file_containing_extension_;
+    message_request_.file_containing_extension_ = NULL;
+    return temp;
+  } else {
+    return NULL;
+  }
+}
+void ServerReflectionRequest::set_allocated_file_containing_extension(::grpc::reflection::v1alpha::ExtensionRequest* file_containing_extension) {
+  clear_message_request();
+  if (file_containing_extension) {
+    set_has_file_containing_extension();
+    message_request_.file_containing_extension_ = file_containing_extension;
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension)
+}
+
+// optional string all_extension_numbers_of_type = 6;
+bool ServerReflectionRequest::has_all_extension_numbers_of_type() const {
+  return message_request_case() == kAllExtensionNumbersOfType;
+}
+void ServerReflectionRequest::set_has_all_extension_numbers_of_type() {
+  _oneof_case_[0] = kAllExtensionNumbersOfType;
+}
+void ServerReflectionRequest::clear_all_extension_numbers_of_type() {
+  if (has_all_extension_numbers_of_type()) {
+    message_request_.all_extension_numbers_of_type_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+    clear_has_message_request();
+  }
+}
+ const ::std::string& ServerReflectionRequest::all_extension_numbers_of_type() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+  if (has_all_extension_numbers_of_type()) {
+    return message_request_.all_extension_numbers_of_type_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  return *&::google::protobuf::internal::GetEmptyStringAlreadyInited();
+}
+ void ServerReflectionRequest::set_all_extension_numbers_of_type(const ::std::string& value) {
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+  if (!has_all_extension_numbers_of_type()) {
+    clear_message_request();
+    set_has_all_extension_numbers_of_type();
+    message_request_.all_extension_numbers_of_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.all_extension_numbers_of_type_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+}
+ void ServerReflectionRequest::set_all_extension_numbers_of_type(const char* value) {
+  if (!has_all_extension_numbers_of_type()) {
+    clear_message_request();
+    set_has_all_extension_numbers_of_type();
+    message_request_.all_extension_numbers_of_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.all_extension_numbers_of_type_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+}
+ void ServerReflectionRequest::set_all_extension_numbers_of_type(const char* value, size_t size) {
+  if (!has_all_extension_numbers_of_type()) {
+    clear_message_request();
+    set_has_all_extension_numbers_of_type();
+    message_request_.all_extension_numbers_of_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.all_extension_numbers_of_type_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(
+      reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+}
+ ::std::string* ServerReflectionRequest::mutable_all_extension_numbers_of_type() {
+  if (!has_all_extension_numbers_of_type()) {
+    clear_message_request();
+    set_has_all_extension_numbers_of_type();
+    message_request_.all_extension_numbers_of_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+  return message_request_.all_extension_numbers_of_type_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServerReflectionRequest::release_all_extension_numbers_of_type() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+  if (has_all_extension_numbers_of_type()) {
+    clear_has_message_request();
+    return message_request_.all_extension_numbers_of_type_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  } else {
+    return NULL;
+  }
+}
+ void ServerReflectionRequest::set_allocated_all_extension_numbers_of_type(::std::string* all_extension_numbers_of_type) {
+  if (!has_all_extension_numbers_of_type()) {
+    message_request_.all_extension_numbers_of_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  clear_message_request();
+  if (all_extension_numbers_of_type != NULL) {
+    set_has_all_extension_numbers_of_type();
+    message_request_.all_extension_numbers_of_type_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+        all_extension_numbers_of_type);
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionRequest.all_extension_numbers_of_type)
+}
+
+// optional string list_services = 7;
+bool ServerReflectionRequest::has_list_services() const {
+  return message_request_case() == kListServices;
+}
+void ServerReflectionRequest::set_has_list_services() {
+  _oneof_case_[0] = kListServices;
+}
+void ServerReflectionRequest::clear_list_services() {
+  if (has_list_services()) {
+    message_request_.list_services_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+    clear_has_message_request();
+  }
+}
+ const ::std::string& ServerReflectionRequest::list_services() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+  if (has_list_services()) {
+    return message_request_.list_services_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  return *&::google::protobuf::internal::GetEmptyStringAlreadyInited();
+}
+ void ServerReflectionRequest::set_list_services(const ::std::string& value) {
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+  if (!has_list_services()) {
+    clear_message_request();
+    set_has_list_services();
+    message_request_.list_services_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.list_services_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+}
+ void ServerReflectionRequest::set_list_services(const char* value) {
+  if (!has_list_services()) {
+    clear_message_request();
+    set_has_list_services();
+    message_request_.list_services_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.list_services_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+}
+ void ServerReflectionRequest::set_list_services(const char* value, size_t size) {
+  if (!has_list_services()) {
+    clear_message_request();
+    set_has_list_services();
+    message_request_.list_services_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  message_request_.list_services_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(
+      reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+}
+ ::std::string* ServerReflectionRequest::mutable_list_services() {
+  if (!has_list_services()) {
+    clear_message_request();
+    set_has_list_services();
+    message_request_.list_services_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+  return message_request_.list_services_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServerReflectionRequest::release_list_services() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+  if (has_list_services()) {
+    clear_has_message_request();
+    return message_request_.list_services_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  } else {
+    return NULL;
+  }
+}
+ void ServerReflectionRequest::set_allocated_list_services(::std::string* list_services) {
+  if (!has_list_services()) {
+    message_request_.list_services_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  }
+  clear_message_request();
+  if (list_services != NULL) {
+    set_has_list_services();
+    message_request_.list_services_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+        list_services);
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionRequest.list_services)
+}
+
+bool ServerReflectionRequest::has_message_request() const {
+  return message_request_case() != MESSAGE_REQUEST_NOT_SET;
+}
+void ServerReflectionRequest::clear_has_message_request() {
+  _oneof_case_[0] = MESSAGE_REQUEST_NOT_SET;
+}
+ServerReflectionRequest::MessageRequestCase ServerReflectionRequest::message_request_case() const {
+  return ServerReflectionRequest::MessageRequestCase(_oneof_case_[0]);
+}
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ExtensionRequest::kContainingTypeFieldNumber;
+const int ExtensionRequest::kExtensionNumberFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ExtensionRequest::ExtensionRequest()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ExtensionRequest)
+}
+
+void ExtensionRequest::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+}
+
+ExtensionRequest::ExtensionRequest(const ExtensionRequest& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ExtensionRequest)
+}
+
+void ExtensionRequest::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+  containing_type_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  extension_number_ = 0;
+}
+
+ExtensionRequest::~ExtensionRequest() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ExtensionRequest)
+  SharedDtor();
+}
+
+void ExtensionRequest::SharedDtor() {
+  containing_type_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (this != default_instance_) {
+  }
+}
+
+void ExtensionRequest::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ExtensionRequest::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ExtensionRequest_descriptor_;
+}
+
+const ExtensionRequest& ExtensionRequest::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ExtensionRequest* ExtensionRequest::default_instance_ = NULL;
+
+ExtensionRequest* ExtensionRequest::New(::google::protobuf::Arena* arena) const {
+  ExtensionRequest* n = new ExtensionRequest;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ExtensionRequest::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ExtensionRequest)
+  containing_type_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  extension_number_ = 0;
+}
+
+bool ExtensionRequest::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ExtensionRequest)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // optional string containing_type = 1;
+      case 1: {
+        if (tag == 10) {
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_containing_type()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->containing_type().data(), this->containing_type().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ExtensionRequest.containing_type"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(16)) goto parse_extension_number;
+        break;
+      }
+
+      // optional int32 extension_number = 2;
+      case 2: {
+        if (tag == 16) {
+         parse_extension_number:
+          DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<
+                   ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>(
+                 input, &extension_number_)));
+
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ExtensionRequest)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ExtensionRequest)
+  return false;
+#undef DO_
+}
+
+void ExtensionRequest::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ExtensionRequest)
+  // optional string containing_type = 1;
+  if (this->containing_type().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->containing_type().data(), this->containing_type().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ExtensionRequest.containing_type");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      1, this->containing_type(), output);
+  }
+
+  // optional int32 extension_number = 2;
+  if (this->extension_number() != 0) {
+    ::google::protobuf::internal::WireFormatLite::WriteInt32(2, this->extension_number(), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ExtensionRequest)
+}
+
+::google::protobuf::uint8* ExtensionRequest::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ExtensionRequest)
+  // optional string containing_type = 1;
+  if (this->containing_type().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->containing_type().data(), this->containing_type().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ExtensionRequest.containing_type");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        1, this->containing_type(), target);
+  }
+
+  // optional int32 extension_number = 2;
+  if (this->extension_number() != 0) {
+    target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(2, this->extension_number(), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ExtensionRequest)
+  return target;
+}
+
+int ExtensionRequest::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ExtensionRequest)
+  int total_size = 0;
+
+  // optional string containing_type = 1;
+  if (this->containing_type().size() > 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::StringSize(
+        this->containing_type());
+  }
+
+  // optional int32 extension_number = 2;
+  if (this->extension_number() != 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::Int32Size(
+        this->extension_number());
+  }
+
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ExtensionRequest::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ExtensionRequest)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ExtensionRequest* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ExtensionRequest>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ExtensionRequest)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ExtensionRequest)
+    MergeFrom(*source);
+  }
+}
+
+void ExtensionRequest::MergeFrom(const ExtensionRequest& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ExtensionRequest)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  if (from.containing_type().size() > 0) {
+
+    containing_type_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.containing_type_);
+  }
+  if (from.extension_number() != 0) {
+    set_extension_number(from.extension_number());
+  }
+}
+
+void ExtensionRequest::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ExtensionRequest)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ExtensionRequest::CopyFrom(const ExtensionRequest& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ExtensionRequest)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ExtensionRequest::IsInitialized() const {
+
+  return true;
+}
+
+void ExtensionRequest::Swap(ExtensionRequest* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ExtensionRequest::InternalSwap(ExtensionRequest* other) {
+  containing_type_.Swap(&other->containing_type_);
+  std::swap(extension_number_, other->extension_number_);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ExtensionRequest::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ExtensionRequest_descriptor_;
+  metadata.reflection = ExtensionRequest_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ExtensionRequest
+
+// optional string containing_type = 1;
+void ExtensionRequest::clear_containing_type() {
+  containing_type_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ const ::std::string& ExtensionRequest::containing_type() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+  return containing_type_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ExtensionRequest::set_containing_type(const ::std::string& value) {
+  
+  containing_type_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+}
+ void ExtensionRequest::set_containing_type(const char* value) {
+  
+  containing_type_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+}
+ void ExtensionRequest::set_containing_type(const char* value, size_t size) {
+  
+  containing_type_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+}
+ ::std::string* ExtensionRequest::mutable_containing_type() {
+  
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+  return containing_type_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ExtensionRequest::release_containing_type() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+  
+  return containing_type_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ExtensionRequest::set_allocated_containing_type(::std::string* containing_type) {
+  if (containing_type != NULL) {
+    
+  } else {
+    
+  }
+  containing_type_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), containing_type);
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ExtensionRequest.containing_type)
+}
+
+// optional int32 extension_number = 2;
+void ExtensionRequest::clear_extension_number() {
+  extension_number_ = 0;
+}
+ ::google::protobuf::int32 ExtensionRequest::extension_number() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ExtensionRequest.extension_number)
+  return extension_number_;
+}
+ void ExtensionRequest::set_extension_number(::google::protobuf::int32 value) {
+  
+  extension_number_ = value;
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ExtensionRequest.extension_number)
+}
+
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ServerReflectionResponse::kValidHostFieldNumber;
+const int ServerReflectionResponse::kOriginalRequestFieldNumber;
+const int ServerReflectionResponse::kFileDescriptorResponseFieldNumber;
+const int ServerReflectionResponse::kAllExtensionNumbersResponseFieldNumber;
+const int ServerReflectionResponse::kListServicesResponseFieldNumber;
+const int ServerReflectionResponse::kErrorResponseFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ServerReflectionResponse::ServerReflectionResponse()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ServerReflectionResponse)
+}
+
+void ServerReflectionResponse::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+  original_request_ = const_cast< ::grpc::reflection::v1alpha::ServerReflectionRequest*>(&::grpc::reflection::v1alpha::ServerReflectionRequest::default_instance());
+  ServerReflectionResponse_default_oneof_instance_->file_descriptor_response_ = const_cast< ::grpc::reflection::v1alpha::FileDescriptorResponse*>(&::grpc::reflection::v1alpha::FileDescriptorResponse::default_instance());
+  ServerReflectionResponse_default_oneof_instance_->all_extension_numbers_response_ = const_cast< ::grpc::reflection::v1alpha::ExtensionNumberResponse*>(&::grpc::reflection::v1alpha::ExtensionNumberResponse::default_instance());
+  ServerReflectionResponse_default_oneof_instance_->list_services_response_ = const_cast< ::grpc::reflection::v1alpha::ListServiceResponse*>(&::grpc::reflection::v1alpha::ListServiceResponse::default_instance());
+  ServerReflectionResponse_default_oneof_instance_->error_response_ = const_cast< ::grpc::reflection::v1alpha::ErrorResponse*>(&::grpc::reflection::v1alpha::ErrorResponse::default_instance());
+}
+
+ServerReflectionResponse::ServerReflectionResponse(const ServerReflectionResponse& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ServerReflectionResponse)
+}
+
+void ServerReflectionResponse::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+  valid_host_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  original_request_ = NULL;
+  clear_has_message_response();
+}
+
+ServerReflectionResponse::~ServerReflectionResponse() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ServerReflectionResponse)
+  SharedDtor();
+}
+
+void ServerReflectionResponse::SharedDtor() {
+  valid_host_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (has_message_response()) {
+    clear_message_response();
+  }
+  if (this != default_instance_) {
+    delete original_request_;
+  }
+}
+
+void ServerReflectionResponse::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ServerReflectionResponse::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ServerReflectionResponse_descriptor_;
+}
+
+const ServerReflectionResponse& ServerReflectionResponse::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ServerReflectionResponse* ServerReflectionResponse::default_instance_ = NULL;
+
+ServerReflectionResponse* ServerReflectionResponse::New(::google::protobuf::Arena* arena) const {
+  ServerReflectionResponse* n = new ServerReflectionResponse;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ServerReflectionResponse::clear_message_response() {
+// @@protoc_insertion_point(one_of_clear_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  switch(message_response_case()) {
+    case kFileDescriptorResponse: {
+      delete message_response_.file_descriptor_response_;
+      break;
+    }
+    case kAllExtensionNumbersResponse: {
+      delete message_response_.all_extension_numbers_response_;
+      break;
+    }
+    case kListServicesResponse: {
+      delete message_response_.list_services_response_;
+      break;
+    }
+    case kErrorResponse: {
+      delete message_response_.error_response_;
+      break;
+    }
+    case MESSAGE_RESPONSE_NOT_SET: {
+      break;
+    }
+  }
+  _oneof_case_[0] = MESSAGE_RESPONSE_NOT_SET;
+}
+
+
+void ServerReflectionResponse::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  valid_host_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (GetArenaNoVirtual() == NULL && original_request_ != NULL) delete original_request_;
+  original_request_ = NULL;
+  clear_message_response();
+}
+
+bool ServerReflectionResponse::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // optional string valid_host = 1;
+      case 1: {
+        if (tag == 10) {
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_valid_host()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->valid_host().data(), this->valid_host().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServerReflectionResponse.valid_host"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(18)) goto parse_original_request;
+        break;
+      }
+
+      // optional .grpc.reflection.v1alpha.ServerReflectionRequest original_request = 2;
+      case 2: {
+        if (tag == 18) {
+         parse_original_request:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+               input, mutable_original_request()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(34)) goto parse_file_descriptor_response;
+        break;
+      }
+
+      // optional .grpc.reflection.v1alpha.FileDescriptorResponse file_descriptor_response = 4;
+      case 4: {
+        if (tag == 34) {
+         parse_file_descriptor_response:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+               input, mutable_file_descriptor_response()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(42)) goto parse_all_extension_numbers_response;
+        break;
+      }
+
+      // optional .grpc.reflection.v1alpha.ExtensionNumberResponse all_extension_numbers_response = 5;
+      case 5: {
+        if (tag == 42) {
+         parse_all_extension_numbers_response:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+               input, mutable_all_extension_numbers_response()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(50)) goto parse_list_services_response;
+        break;
+      }
+
+      // optional .grpc.reflection.v1alpha.ListServiceResponse list_services_response = 6;
+      case 6: {
+        if (tag == 50) {
+         parse_list_services_response:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+               input, mutable_list_services_response()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(58)) goto parse_error_response;
+        break;
+      }
+
+      // optional .grpc.reflection.v1alpha.ErrorResponse error_response = 7;
+      case 7: {
+        if (tag == 58) {
+         parse_error_response:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+               input, mutable_error_response()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ServerReflectionResponse)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ServerReflectionResponse)
+  return false;
+#undef DO_
+}
+
+void ServerReflectionResponse::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  // optional string valid_host = 1;
+  if (this->valid_host().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->valid_host().data(), this->valid_host().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionResponse.valid_host");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      1, this->valid_host(), output);
+  }
+
+  // optional .grpc.reflection.v1alpha.ServerReflectionRequest original_request = 2;
+  if (this->has_original_request()) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      2, *this->original_request_, output);
+  }
+
+  // optional .grpc.reflection.v1alpha.FileDescriptorResponse file_descriptor_response = 4;
+  if (has_file_descriptor_response()) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      4, *message_response_.file_descriptor_response_, output);
+  }
+
+  // optional .grpc.reflection.v1alpha.ExtensionNumberResponse all_extension_numbers_response = 5;
+  if (has_all_extension_numbers_response()) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      5, *message_response_.all_extension_numbers_response_, output);
+  }
+
+  // optional .grpc.reflection.v1alpha.ListServiceResponse list_services_response = 6;
+  if (has_list_services_response()) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      6, *message_response_.list_services_response_, output);
+  }
+
+  // optional .grpc.reflection.v1alpha.ErrorResponse error_response = 7;
+  if (has_error_response()) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      7, *message_response_.error_response_, output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ServerReflectionResponse)
+}
+
+::google::protobuf::uint8* ServerReflectionResponse::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  // optional string valid_host = 1;
+  if (this->valid_host().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->valid_host().data(), this->valid_host().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServerReflectionResponse.valid_host");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        1, this->valid_host(), target);
+  }
+
+  // optional .grpc.reflection.v1alpha.ServerReflectionRequest original_request = 2;
+  if (this->has_original_request()) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        2, *this->original_request_, target);
+  }
+
+  // optional .grpc.reflection.v1alpha.FileDescriptorResponse file_descriptor_response = 4;
+  if (has_file_descriptor_response()) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        4, *message_response_.file_descriptor_response_, target);
+  }
+
+  // optional .grpc.reflection.v1alpha.ExtensionNumberResponse all_extension_numbers_response = 5;
+  if (has_all_extension_numbers_response()) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        5, *message_response_.all_extension_numbers_response_, target);
+  }
+
+  // optional .grpc.reflection.v1alpha.ListServiceResponse list_services_response = 6;
+  if (has_list_services_response()) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        6, *message_response_.list_services_response_, target);
+  }
+
+  // optional .grpc.reflection.v1alpha.ErrorResponse error_response = 7;
+  if (has_error_response()) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        7, *message_response_.error_response_, target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ServerReflectionResponse)
+  return target;
+}
+
+int ServerReflectionResponse::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  int total_size = 0;
+
+  // optional string valid_host = 1;
+  if (this->valid_host().size() > 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::StringSize(
+        this->valid_host());
+  }
+
+  // optional .grpc.reflection.v1alpha.ServerReflectionRequest original_request = 2;
+  if (this->has_original_request()) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        *this->original_request_);
+  }
+
+  switch (message_response_case()) {
+    // optional .grpc.reflection.v1alpha.FileDescriptorResponse file_descriptor_response = 4;
+    case kFileDescriptorResponse: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+          *message_response_.file_descriptor_response_);
+      break;
+    }
+    // optional .grpc.reflection.v1alpha.ExtensionNumberResponse all_extension_numbers_response = 5;
+    case kAllExtensionNumbersResponse: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+          *message_response_.all_extension_numbers_response_);
+      break;
+    }
+    // optional .grpc.reflection.v1alpha.ListServiceResponse list_services_response = 6;
+    case kListServicesResponse: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+          *message_response_.list_services_response_);
+      break;
+    }
+    // optional .grpc.reflection.v1alpha.ErrorResponse error_response = 7;
+    case kErrorResponse: {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+          *message_response_.error_response_);
+      break;
+    }
+    case MESSAGE_RESPONSE_NOT_SET: {
+      break;
+    }
+  }
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ServerReflectionResponse::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ServerReflectionResponse* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ServerReflectionResponse>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ServerReflectionResponse)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ServerReflectionResponse)
+    MergeFrom(*source);
+  }
+}
+
+void ServerReflectionResponse::MergeFrom(const ServerReflectionResponse& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  switch (from.message_response_case()) {
+    case kFileDescriptorResponse: {
+      mutable_file_descriptor_response()->::grpc::reflection::v1alpha::FileDescriptorResponse::MergeFrom(from.file_descriptor_response());
+      break;
+    }
+    case kAllExtensionNumbersResponse: {
+      mutable_all_extension_numbers_response()->::grpc::reflection::v1alpha::ExtensionNumberResponse::MergeFrom(from.all_extension_numbers_response());
+      break;
+    }
+    case kListServicesResponse: {
+      mutable_list_services_response()->::grpc::reflection::v1alpha::ListServiceResponse::MergeFrom(from.list_services_response());
+      break;
+    }
+    case kErrorResponse: {
+      mutable_error_response()->::grpc::reflection::v1alpha::ErrorResponse::MergeFrom(from.error_response());
+      break;
+    }
+    case MESSAGE_RESPONSE_NOT_SET: {
+      break;
+    }
+  }
+  if (from.valid_host().size() > 0) {
+
+    valid_host_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.valid_host_);
+  }
+  if (from.has_original_request()) {
+    mutable_original_request()->::grpc::reflection::v1alpha::ServerReflectionRequest::MergeFrom(from.original_request());
+  }
+}
+
+void ServerReflectionResponse::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ServerReflectionResponse::CopyFrom(const ServerReflectionResponse& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ServerReflectionResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ServerReflectionResponse::IsInitialized() const {
+
+  return true;
+}
+
+void ServerReflectionResponse::Swap(ServerReflectionResponse* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ServerReflectionResponse::InternalSwap(ServerReflectionResponse* other) {
+  valid_host_.Swap(&other->valid_host_);
+  std::swap(original_request_, other->original_request_);
+  std::swap(message_response_, other->message_response_);
+  std::swap(_oneof_case_[0], other->_oneof_case_[0]);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ServerReflectionResponse::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ServerReflectionResponse_descriptor_;
+  metadata.reflection = ServerReflectionResponse_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ServerReflectionResponse
+
+// optional string valid_host = 1;
+void ServerReflectionResponse::clear_valid_host() {
+  valid_host_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ const ::std::string& ServerReflectionResponse::valid_host() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+  return valid_host_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ServerReflectionResponse::set_valid_host(const ::std::string& value) {
+  
+  valid_host_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+}
+ void ServerReflectionResponse::set_valid_host(const char* value) {
+  
+  valid_host_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+}
+ void ServerReflectionResponse::set_valid_host(const char* value, size_t size) {
+  
+  valid_host_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+}
+ ::std::string* ServerReflectionResponse::mutable_valid_host() {
+  
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+  return valid_host_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServerReflectionResponse::release_valid_host() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+  
+  return valid_host_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ServerReflectionResponse::set_allocated_valid_host(::std::string* valid_host) {
+  if (valid_host != NULL) {
+    
+  } else {
+    
+  }
+  valid_host_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), valid_host);
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionResponse.valid_host)
+}
+
+// optional .grpc.reflection.v1alpha.ServerReflectionRequest original_request = 2;
+bool ServerReflectionResponse::has_original_request() const {
+  return !_is_default_instance_ && original_request_ != NULL;
+}
+void ServerReflectionResponse::clear_original_request() {
+  if (GetArenaNoVirtual() == NULL && original_request_ != NULL) delete original_request_;
+  original_request_ = NULL;
+}
+const ::grpc::reflection::v1alpha::ServerReflectionRequest& ServerReflectionResponse::original_request() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionResponse.original_request)
+  return original_request_ != NULL ? *original_request_ : *default_instance_->original_request_;
+}
+::grpc::reflection::v1alpha::ServerReflectionRequest* ServerReflectionResponse::mutable_original_request() {
+  
+  if (original_request_ == NULL) {
+    original_request_ = new ::grpc::reflection::v1alpha::ServerReflectionRequest;
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionResponse.original_request)
+  return original_request_;
+}
+::grpc::reflection::v1alpha::ServerReflectionRequest* ServerReflectionResponse::release_original_request() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionResponse.original_request)
+  
+  ::grpc::reflection::v1alpha::ServerReflectionRequest* temp = original_request_;
+  original_request_ = NULL;
+  return temp;
+}
+void ServerReflectionResponse::set_allocated_original_request(::grpc::reflection::v1alpha::ServerReflectionRequest* original_request) {
+  delete original_request_;
+  original_request_ = original_request;
+  if (original_request) {
+    
+  } else {
+    
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionResponse.original_request)
+}
+
+// optional .grpc.reflection.v1alpha.FileDescriptorResponse file_descriptor_response = 4;
+bool ServerReflectionResponse::has_file_descriptor_response() const {
+  return message_response_case() == kFileDescriptorResponse;
+}
+void ServerReflectionResponse::set_has_file_descriptor_response() {
+  _oneof_case_[0] = kFileDescriptorResponse;
+}
+void ServerReflectionResponse::clear_file_descriptor_response() {
+  if (has_file_descriptor_response()) {
+    delete message_response_.file_descriptor_response_;
+    clear_has_message_response();
+  }
+}
+ const ::grpc::reflection::v1alpha::FileDescriptorResponse& ServerReflectionResponse::file_descriptor_response() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response)
+  return has_file_descriptor_response()
+      ? *message_response_.file_descriptor_response_
+      : ::grpc::reflection::v1alpha::FileDescriptorResponse::default_instance();
+}
+::grpc::reflection::v1alpha::FileDescriptorResponse* ServerReflectionResponse::mutable_file_descriptor_response() {
+  if (!has_file_descriptor_response()) {
+    clear_message_response();
+    set_has_file_descriptor_response();
+    message_response_.file_descriptor_response_ = new ::grpc::reflection::v1alpha::FileDescriptorResponse;
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response)
+  return message_response_.file_descriptor_response_;
+}
+::grpc::reflection::v1alpha::FileDescriptorResponse* ServerReflectionResponse::release_file_descriptor_response() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response)
+  if (has_file_descriptor_response()) {
+    clear_has_message_response();
+    ::grpc::reflection::v1alpha::FileDescriptorResponse* temp = message_response_.file_descriptor_response_;
+    message_response_.file_descriptor_response_ = NULL;
+    return temp;
+  } else {
+    return NULL;
+  }
+}
+void ServerReflectionResponse::set_allocated_file_descriptor_response(::grpc::reflection::v1alpha::FileDescriptorResponse* file_descriptor_response) {
+  clear_message_response();
+  if (file_descriptor_response) {
+    set_has_file_descriptor_response();
+    message_response_.file_descriptor_response_ = file_descriptor_response;
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response)
+}
+
+// optional .grpc.reflection.v1alpha.ExtensionNumberResponse all_extension_numbers_response = 5;
+bool ServerReflectionResponse::has_all_extension_numbers_response() const {
+  return message_response_case() == kAllExtensionNumbersResponse;
+}
+void ServerReflectionResponse::set_has_all_extension_numbers_response() {
+  _oneof_case_[0] = kAllExtensionNumbersResponse;
+}
+void ServerReflectionResponse::clear_all_extension_numbers_response() {
+  if (has_all_extension_numbers_response()) {
+    delete message_response_.all_extension_numbers_response_;
+    clear_has_message_response();
+  }
+}
+ const ::grpc::reflection::v1alpha::ExtensionNumberResponse& ServerReflectionResponse::all_extension_numbers_response() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response)
+  return has_all_extension_numbers_response()
+      ? *message_response_.all_extension_numbers_response_
+      : ::grpc::reflection::v1alpha::ExtensionNumberResponse::default_instance();
+}
+::grpc::reflection::v1alpha::ExtensionNumberResponse* ServerReflectionResponse::mutable_all_extension_numbers_response() {
+  if (!has_all_extension_numbers_response()) {
+    clear_message_response();
+    set_has_all_extension_numbers_response();
+    message_response_.all_extension_numbers_response_ = new ::grpc::reflection::v1alpha::ExtensionNumberResponse;
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response)
+  return message_response_.all_extension_numbers_response_;
+}
+::grpc::reflection::v1alpha::ExtensionNumberResponse* ServerReflectionResponse::release_all_extension_numbers_response() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response)
+  if (has_all_extension_numbers_response()) {
+    clear_has_message_response();
+    ::grpc::reflection::v1alpha::ExtensionNumberResponse* temp = message_response_.all_extension_numbers_response_;
+    message_response_.all_extension_numbers_response_ = NULL;
+    return temp;
+  } else {
+    return NULL;
+  }
+}
+void ServerReflectionResponse::set_allocated_all_extension_numbers_response(::grpc::reflection::v1alpha::ExtensionNumberResponse* all_extension_numbers_response) {
+  clear_message_response();
+  if (all_extension_numbers_response) {
+    set_has_all_extension_numbers_response();
+    message_response_.all_extension_numbers_response_ = all_extension_numbers_response;
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response)
+}
+
+// optional .grpc.reflection.v1alpha.ListServiceResponse list_services_response = 6;
+bool ServerReflectionResponse::has_list_services_response() const {
+  return message_response_case() == kListServicesResponse;
+}
+void ServerReflectionResponse::set_has_list_services_response() {
+  _oneof_case_[0] = kListServicesResponse;
+}
+void ServerReflectionResponse::clear_list_services_response() {
+  if (has_list_services_response()) {
+    delete message_response_.list_services_response_;
+    clear_has_message_response();
+  }
+}
+ const ::grpc::reflection::v1alpha::ListServiceResponse& ServerReflectionResponse::list_services_response() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response)
+  return has_list_services_response()
+      ? *message_response_.list_services_response_
+      : ::grpc::reflection::v1alpha::ListServiceResponse::default_instance();
+}
+::grpc::reflection::v1alpha::ListServiceResponse* ServerReflectionResponse::mutable_list_services_response() {
+  if (!has_list_services_response()) {
+    clear_message_response();
+    set_has_list_services_response();
+    message_response_.list_services_response_ = new ::grpc::reflection::v1alpha::ListServiceResponse;
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response)
+  return message_response_.list_services_response_;
+}
+::grpc::reflection::v1alpha::ListServiceResponse* ServerReflectionResponse::release_list_services_response() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response)
+  if (has_list_services_response()) {
+    clear_has_message_response();
+    ::grpc::reflection::v1alpha::ListServiceResponse* temp = message_response_.list_services_response_;
+    message_response_.list_services_response_ = NULL;
+    return temp;
+  } else {
+    return NULL;
+  }
+}
+void ServerReflectionResponse::set_allocated_list_services_response(::grpc::reflection::v1alpha::ListServiceResponse* list_services_response) {
+  clear_message_response();
+  if (list_services_response) {
+    set_has_list_services_response();
+    message_response_.list_services_response_ = list_services_response;
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response)
+}
+
+// optional .grpc.reflection.v1alpha.ErrorResponse error_response = 7;
+bool ServerReflectionResponse::has_error_response() const {
+  return message_response_case() == kErrorResponse;
+}
+void ServerReflectionResponse::set_has_error_response() {
+  _oneof_case_[0] = kErrorResponse;
+}
+void ServerReflectionResponse::clear_error_response() {
+  if (has_error_response()) {
+    delete message_response_.error_response_;
+    clear_has_message_response();
+  }
+}
+ const ::grpc::reflection::v1alpha::ErrorResponse& ServerReflectionResponse::error_response() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServerReflectionResponse.error_response)
+  return has_error_response()
+      ? *message_response_.error_response_
+      : ::grpc::reflection::v1alpha::ErrorResponse::default_instance();
+}
+::grpc::reflection::v1alpha::ErrorResponse* ServerReflectionResponse::mutable_error_response() {
+  if (!has_error_response()) {
+    clear_message_response();
+    set_has_error_response();
+    message_response_.error_response_ = new ::grpc::reflection::v1alpha::ErrorResponse;
+  }
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServerReflectionResponse.error_response)
+  return message_response_.error_response_;
+}
+::grpc::reflection::v1alpha::ErrorResponse* ServerReflectionResponse::release_error_response() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServerReflectionResponse.error_response)
+  if (has_error_response()) {
+    clear_has_message_response();
+    ::grpc::reflection::v1alpha::ErrorResponse* temp = message_response_.error_response_;
+    message_response_.error_response_ = NULL;
+    return temp;
+  } else {
+    return NULL;
+  }
+}
+void ServerReflectionResponse::set_allocated_error_response(::grpc::reflection::v1alpha::ErrorResponse* error_response) {
+  clear_message_response();
+  if (error_response) {
+    set_has_error_response();
+    message_response_.error_response_ = error_response;
+  }
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServerReflectionResponse.error_response)
+}
+
+bool ServerReflectionResponse::has_message_response() const {
+  return message_response_case() != MESSAGE_RESPONSE_NOT_SET;
+}
+void ServerReflectionResponse::clear_has_message_response() {
+  _oneof_case_[0] = MESSAGE_RESPONSE_NOT_SET;
+}
+ServerReflectionResponse::MessageResponseCase ServerReflectionResponse::message_response_case() const {
+  return ServerReflectionResponse::MessageResponseCase(_oneof_case_[0]);
+}
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int FileDescriptorResponse::kFileDescriptorProtoFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+FileDescriptorResponse::FileDescriptorResponse()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.FileDescriptorResponse)
+}
+
+void FileDescriptorResponse::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+}
+
+FileDescriptorResponse::FileDescriptorResponse(const FileDescriptorResponse& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.FileDescriptorResponse)
+}
+
+void FileDescriptorResponse::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+}
+
+FileDescriptorResponse::~FileDescriptorResponse() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.FileDescriptorResponse)
+  SharedDtor();
+}
+
+void FileDescriptorResponse::SharedDtor() {
+  if (this != default_instance_) {
+  }
+}
+
+void FileDescriptorResponse::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* FileDescriptorResponse::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return FileDescriptorResponse_descriptor_;
+}
+
+const FileDescriptorResponse& FileDescriptorResponse::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+FileDescriptorResponse* FileDescriptorResponse::default_instance_ = NULL;
+
+FileDescriptorResponse* FileDescriptorResponse::New(::google::protobuf::Arena* arena) const {
+  FileDescriptorResponse* n = new FileDescriptorResponse;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void FileDescriptorResponse::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  file_descriptor_proto_.Clear();
+}
+
+bool FileDescriptorResponse::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // repeated bytes file_descriptor_proto = 1;
+      case 1: {
+        if (tag == 10) {
+         parse_file_descriptor_proto:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadBytes(
+                input, this->add_file_descriptor_proto()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(10)) goto parse_file_descriptor_proto;
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.FileDescriptorResponse)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.FileDescriptorResponse)
+  return false;
+#undef DO_
+}
+
+void FileDescriptorResponse::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  // repeated bytes file_descriptor_proto = 1;
+  for (int i = 0; i < this->file_descriptor_proto_size(); i++) {
+    ::google::protobuf::internal::WireFormatLite::WriteBytes(
+      1, this->file_descriptor_proto(i), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.FileDescriptorResponse)
+}
+
+::google::protobuf::uint8* FileDescriptorResponse::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  // repeated bytes file_descriptor_proto = 1;
+  for (int i = 0; i < this->file_descriptor_proto_size(); i++) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteBytesToArray(1, this->file_descriptor_proto(i), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.FileDescriptorResponse)
+  return target;
+}
+
+int FileDescriptorResponse::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  int total_size = 0;
+
+  // repeated bytes file_descriptor_proto = 1;
+  total_size += 1 * this->file_descriptor_proto_size();
+  for (int i = 0; i < this->file_descriptor_proto_size(); i++) {
+    total_size += ::google::protobuf::internal::WireFormatLite::BytesSize(
+      this->file_descriptor_proto(i));
+  }
+
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void FileDescriptorResponse::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const FileDescriptorResponse* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const FileDescriptorResponse>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.FileDescriptorResponse)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.FileDescriptorResponse)
+    MergeFrom(*source);
+  }
+}
+
+void FileDescriptorResponse::MergeFrom(const FileDescriptorResponse& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  file_descriptor_proto_.MergeFrom(from.file_descriptor_proto_);
+}
+
+void FileDescriptorResponse::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void FileDescriptorResponse::CopyFrom(const FileDescriptorResponse& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.FileDescriptorResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool FileDescriptorResponse::IsInitialized() const {
+
+  return true;
+}
+
+void FileDescriptorResponse::Swap(FileDescriptorResponse* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void FileDescriptorResponse::InternalSwap(FileDescriptorResponse* other) {
+  file_descriptor_proto_.UnsafeArenaSwap(&other->file_descriptor_proto_);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata FileDescriptorResponse::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = FileDescriptorResponse_descriptor_;
+  metadata.reflection = FileDescriptorResponse_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// FileDescriptorResponse
+
+// repeated bytes file_descriptor_proto = 1;
+int FileDescriptorResponse::file_descriptor_proto_size() const {
+  return file_descriptor_proto_.size();
+}
+void FileDescriptorResponse::clear_file_descriptor_proto() {
+  file_descriptor_proto_.Clear();
+}
+ const ::std::string& FileDescriptorResponse::file_descriptor_proto(int index) const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+  return file_descriptor_proto_.Get(index);
+}
+ ::std::string* FileDescriptorResponse::mutable_file_descriptor_proto(int index) {
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+  return file_descriptor_proto_.Mutable(index);
+}
+ void FileDescriptorResponse::set_file_descriptor_proto(int index, const ::std::string& value) {
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+  file_descriptor_proto_.Mutable(index)->assign(value);
+}
+ void FileDescriptorResponse::set_file_descriptor_proto(int index, const char* value) {
+  file_descriptor_proto_.Mutable(index)->assign(value);
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+}
+ void FileDescriptorResponse::set_file_descriptor_proto(int index, const void* value, size_t size) {
+  file_descriptor_proto_.Mutable(index)->assign(
+    reinterpret_cast<const char*>(value), size);
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+}
+ ::std::string* FileDescriptorResponse::add_file_descriptor_proto() {
+  // @@protoc_insertion_point(field_add_mutable:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+  return file_descriptor_proto_.Add();
+}
+ void FileDescriptorResponse::add_file_descriptor_proto(const ::std::string& value) {
+  file_descriptor_proto_.Add()->assign(value);
+  // @@protoc_insertion_point(field_add:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+}
+ void FileDescriptorResponse::add_file_descriptor_proto(const char* value) {
+  file_descriptor_proto_.Add()->assign(value);
+  // @@protoc_insertion_point(field_add_char:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+}
+ void FileDescriptorResponse::add_file_descriptor_proto(const void* value, size_t size) {
+  file_descriptor_proto_.Add()->assign(reinterpret_cast<const char*>(value), size);
+  // @@protoc_insertion_point(field_add_pointer:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+}
+ const ::google::protobuf::RepeatedPtrField< ::std::string>&
+FileDescriptorResponse::file_descriptor_proto() const {
+  // @@protoc_insertion_point(field_list:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+  return file_descriptor_proto_;
+}
+ ::google::protobuf::RepeatedPtrField< ::std::string>*
+FileDescriptorResponse::mutable_file_descriptor_proto() {
+  // @@protoc_insertion_point(field_mutable_list:grpc.reflection.v1alpha.FileDescriptorResponse.file_descriptor_proto)
+  return &file_descriptor_proto_;
+}
+
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ExtensionNumberResponse::kBaseTypeNameFieldNumber;
+const int ExtensionNumberResponse::kExtensionNumberFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ExtensionNumberResponse::ExtensionNumberResponse()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ExtensionNumberResponse)
+}
+
+void ExtensionNumberResponse::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+}
+
+ExtensionNumberResponse::ExtensionNumberResponse(const ExtensionNumberResponse& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ExtensionNumberResponse)
+}
+
+void ExtensionNumberResponse::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+  base_type_name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+
+ExtensionNumberResponse::~ExtensionNumberResponse() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  SharedDtor();
+}
+
+void ExtensionNumberResponse::SharedDtor() {
+  base_type_name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (this != default_instance_) {
+  }
+}
+
+void ExtensionNumberResponse::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ExtensionNumberResponse::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ExtensionNumberResponse_descriptor_;
+}
+
+const ExtensionNumberResponse& ExtensionNumberResponse::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ExtensionNumberResponse* ExtensionNumberResponse::default_instance_ = NULL;
+
+ExtensionNumberResponse* ExtensionNumberResponse::New(::google::protobuf::Arena* arena) const {
+  ExtensionNumberResponse* n = new ExtensionNumberResponse;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ExtensionNumberResponse::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  base_type_name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  extension_number_.Clear();
+}
+
+bool ExtensionNumberResponse::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // optional string base_type_name = 1;
+      case 1: {
+        if (tag == 10) {
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_base_type_name()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->base_type_name().data(), this->base_type_name().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(18)) goto parse_extension_number;
+        break;
+      }
+
+      // repeated int32 extension_number = 2;
+      case 2: {
+        if (tag == 18) {
+         parse_extension_number:
+          DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitive<
+                   ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>(
+                 input, this->mutable_extension_number())));
+        } else if (tag == 16) {
+          DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline<
+                   ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>(
+                 1, 18, input, this->mutable_extension_number())));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  return false;
+#undef DO_
+}
+
+void ExtensionNumberResponse::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  // optional string base_type_name = 1;
+  if (this->base_type_name().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->base_type_name().data(), this->base_type_name().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      1, this->base_type_name(), output);
+  }
+
+  // repeated int32 extension_number = 2;
+  if (this->extension_number_size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::WriteTag(2, ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
+    output->WriteVarint32(_extension_number_cached_byte_size_);
+  }
+  for (int i = 0; i < this->extension_number_size(); i++) {
+    ::google::protobuf::internal::WireFormatLite::WriteInt32NoTag(
+      this->extension_number(i), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ExtensionNumberResponse)
+}
+
+::google::protobuf::uint8* ExtensionNumberResponse::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  // optional string base_type_name = 1;
+  if (this->base_type_name().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->base_type_name().data(), this->base_type_name().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        1, this->base_type_name(), target);
+  }
+
+  // repeated int32 extension_number = 2;
+  if (this->extension_number_size() > 0) {
+    target = ::google::protobuf::internal::WireFormatLite::WriteTagToArray(
+      2,
+      ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
+      target);
+    target = ::google::protobuf::io::CodedOutputStream::WriteVarint32ToArray(
+      _extension_number_cached_byte_size_, target);
+  }
+  for (int i = 0; i < this->extension_number_size(); i++) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteInt32NoTagToArray(this->extension_number(i), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  return target;
+}
+
+int ExtensionNumberResponse::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  int total_size = 0;
+
+  // optional string base_type_name = 1;
+  if (this->base_type_name().size() > 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::StringSize(
+        this->base_type_name());
+  }
+
+  // repeated int32 extension_number = 2;
+  {
+    int data_size = 0;
+    for (int i = 0; i < this->extension_number_size(); i++) {
+      data_size += ::google::protobuf::internal::WireFormatLite::
+        Int32Size(this->extension_number(i));
+    }
+    if (data_size > 0) {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::Int32Size(data_size);
+    }
+    GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+    _extension_number_cached_byte_size_ = data_size;
+    GOOGLE_SAFE_CONCURRENT_WRITES_END();
+    total_size += data_size;
+  }
+
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ExtensionNumberResponse::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ExtensionNumberResponse* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ExtensionNumberResponse>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ExtensionNumberResponse)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ExtensionNumberResponse)
+    MergeFrom(*source);
+  }
+}
+
+void ExtensionNumberResponse::MergeFrom(const ExtensionNumberResponse& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  extension_number_.MergeFrom(from.extension_number_);
+  if (from.base_type_name().size() > 0) {
+
+    base_type_name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.base_type_name_);
+  }
+}
+
+void ExtensionNumberResponse::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ExtensionNumberResponse::CopyFrom(const ExtensionNumberResponse& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ExtensionNumberResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ExtensionNumberResponse::IsInitialized() const {
+
+  return true;
+}
+
+void ExtensionNumberResponse::Swap(ExtensionNumberResponse* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ExtensionNumberResponse::InternalSwap(ExtensionNumberResponse* other) {
+  base_type_name_.Swap(&other->base_type_name_);
+  extension_number_.UnsafeArenaSwap(&other->extension_number_);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ExtensionNumberResponse::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ExtensionNumberResponse_descriptor_;
+  metadata.reflection = ExtensionNumberResponse_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ExtensionNumberResponse
+
+// optional string base_type_name = 1;
+void ExtensionNumberResponse::clear_base_type_name() {
+  base_type_name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ const ::std::string& ExtensionNumberResponse::base_type_name() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+  return base_type_name_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ExtensionNumberResponse::set_base_type_name(const ::std::string& value) {
+  
+  base_type_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+}
+ void ExtensionNumberResponse::set_base_type_name(const char* value) {
+  
+  base_type_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+}
+ void ExtensionNumberResponse::set_base_type_name(const char* value, size_t size) {
+  
+  base_type_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+}
+ ::std::string* ExtensionNumberResponse::mutable_base_type_name() {
+  
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+  return base_type_name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ExtensionNumberResponse::release_base_type_name() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+  
+  return base_type_name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ExtensionNumberResponse::set_allocated_base_type_name(::std::string* base_type_name) {
+  if (base_type_name != NULL) {
+    
+  } else {
+    
+  }
+  base_type_name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), base_type_name);
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ExtensionNumberResponse.base_type_name)
+}
+
+// repeated int32 extension_number = 2;
+int ExtensionNumberResponse::extension_number_size() const {
+  return extension_number_.size();
+}
+void ExtensionNumberResponse::clear_extension_number() {
+  extension_number_.Clear();
+}
+ ::google::protobuf::int32 ExtensionNumberResponse::extension_number(int index) const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ExtensionNumberResponse.extension_number)
+  return extension_number_.Get(index);
+}
+ void ExtensionNumberResponse::set_extension_number(int index, ::google::protobuf::int32 value) {
+  extension_number_.Set(index, value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ExtensionNumberResponse.extension_number)
+}
+ void ExtensionNumberResponse::add_extension_number(::google::protobuf::int32 value) {
+  extension_number_.Add(value);
+  // @@protoc_insertion_point(field_add:grpc.reflection.v1alpha.ExtensionNumberResponse.extension_number)
+}
+ const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >&
+ExtensionNumberResponse::extension_number() const {
+  // @@protoc_insertion_point(field_list:grpc.reflection.v1alpha.ExtensionNumberResponse.extension_number)
+  return extension_number_;
+}
+ ::google::protobuf::RepeatedField< ::google::protobuf::int32 >*
+ExtensionNumberResponse::mutable_extension_number() {
+  // @@protoc_insertion_point(field_mutable_list:grpc.reflection.v1alpha.ExtensionNumberResponse.extension_number)
+  return &extension_number_;
+}
+
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ListServiceResponse::kServiceFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ListServiceResponse::ListServiceResponse()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ListServiceResponse)
+}
+
+void ListServiceResponse::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+}
+
+ListServiceResponse::ListServiceResponse(const ListServiceResponse& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ListServiceResponse)
+}
+
+void ListServiceResponse::SharedCtor() {
+    _is_default_instance_ = false;
+  _cached_size_ = 0;
+}
+
+ListServiceResponse::~ListServiceResponse() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ListServiceResponse)
+  SharedDtor();
+}
+
+void ListServiceResponse::SharedDtor() {
+  if (this != default_instance_) {
+  }
+}
+
+void ListServiceResponse::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ListServiceResponse::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ListServiceResponse_descriptor_;
+}
+
+const ListServiceResponse& ListServiceResponse::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ListServiceResponse* ListServiceResponse::default_instance_ = NULL;
+
+ListServiceResponse* ListServiceResponse::New(::google::protobuf::Arena* arena) const {
+  ListServiceResponse* n = new ListServiceResponse;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ListServiceResponse::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ListServiceResponse)
+  service_.Clear();
+}
+
+bool ListServiceResponse::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ListServiceResponse)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // repeated .grpc.reflection.v1alpha.ServiceResponse service = 1;
+      case 1: {
+        if (tag == 10) {
+          DO_(input->IncrementRecursionDepth());
+         parse_loop_service:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtualNoRecursionDepth(
+                input, add_service()));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(10)) goto parse_loop_service;
+        input->UnsafeDecrementRecursionDepth();
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ListServiceResponse)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ListServiceResponse)
+  return false;
+#undef DO_
+}
+
+void ListServiceResponse::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ListServiceResponse)
+  // repeated .grpc.reflection.v1alpha.ServiceResponse service = 1;
+  for (unsigned int i = 0, n = this->service_size(); i < n; i++) {
+    ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+      1, this->service(i), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ListServiceResponse)
+}
+
+::google::protobuf::uint8* ListServiceResponse::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ListServiceResponse)
+  // repeated .grpc.reflection.v1alpha.ServiceResponse service = 1;
+  for (unsigned int i = 0, n = this->service_size(); i < n; i++) {
+    target = ::google::protobuf::internal::WireFormatLite::
+      WriteMessageNoVirtualToArray(
+        1, this->service(i), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ListServiceResponse)
+  return target;
+}
+
+int ListServiceResponse::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ListServiceResponse)
+  int total_size = 0;
+
+  // repeated .grpc.reflection.v1alpha.ServiceResponse service = 1;
+  total_size += 1 * this->service_size();
+  for (int i = 0; i < this->service_size(); i++) {
+    total_size +=
+      ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        this->service(i));
+  }
+
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ListServiceResponse::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ListServiceResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ListServiceResponse* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ListServiceResponse>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ListServiceResponse)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ListServiceResponse)
+    MergeFrom(*source);
+  }
+}
+
+void ListServiceResponse::MergeFrom(const ListServiceResponse& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ListServiceResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  service_.MergeFrom(from.service_);
+}
+
+void ListServiceResponse::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ListServiceResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ListServiceResponse::CopyFrom(const ListServiceResponse& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ListServiceResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ListServiceResponse::IsInitialized() const {
+
+  return true;
+}
+
+void ListServiceResponse::Swap(ListServiceResponse* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ListServiceResponse::InternalSwap(ListServiceResponse* other) {
+  service_.UnsafeArenaSwap(&other->service_);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ListServiceResponse::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ListServiceResponse_descriptor_;
+  metadata.reflection = ListServiceResponse_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ListServiceResponse
+
+// repeated .grpc.reflection.v1alpha.ServiceResponse service = 1;
+int ListServiceResponse::service_size() const {
+  return service_.size();
+}
+void ListServiceResponse::clear_service() {
+  service_.Clear();
+}
+const ::grpc::reflection::v1alpha::ServiceResponse& ListServiceResponse::service(int index) const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ListServiceResponse.service)
+  return service_.Get(index);
+}
+::grpc::reflection::v1alpha::ServiceResponse* ListServiceResponse::mutable_service(int index) {
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ListServiceResponse.service)
+  return service_.Mutable(index);
+}
+::grpc::reflection::v1alpha::ServiceResponse* ListServiceResponse::add_service() {
+  // @@protoc_insertion_point(field_add:grpc.reflection.v1alpha.ListServiceResponse.service)
+  return service_.Add();
+}
+::google::protobuf::RepeatedPtrField< ::grpc::reflection::v1alpha::ServiceResponse >*
+ListServiceResponse::mutable_service() {
+  // @@protoc_insertion_point(field_mutable_list:grpc.reflection.v1alpha.ListServiceResponse.service)
+  return &service_;
+}
+const ::google::protobuf::RepeatedPtrField< ::grpc::reflection::v1alpha::ServiceResponse >&
+ListServiceResponse::service() const {
+  // @@protoc_insertion_point(field_list:grpc.reflection.v1alpha.ListServiceResponse.service)
+  return service_;
+}
+
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ServiceResponse::kNameFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ServiceResponse::ServiceResponse()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ServiceResponse)
+}
+
+void ServiceResponse::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+}
+
+ServiceResponse::ServiceResponse(const ServiceResponse& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ServiceResponse)
+}
+
+void ServiceResponse::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+  name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+
+ServiceResponse::~ServiceResponse() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ServiceResponse)
+  SharedDtor();
+}
+
+void ServiceResponse::SharedDtor() {
+  name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (this != default_instance_) {
+  }
+}
+
+void ServiceResponse::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ServiceResponse::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ServiceResponse_descriptor_;
+}
+
+const ServiceResponse& ServiceResponse::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ServiceResponse* ServiceResponse::default_instance_ = NULL;
+
+ServiceResponse* ServiceResponse::New(::google::protobuf::Arena* arena) const {
+  ServiceResponse* n = new ServiceResponse;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ServiceResponse::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ServiceResponse)
+  name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+
+bool ServiceResponse::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ServiceResponse)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // optional string name = 1;
+      case 1: {
+        if (tag == 10) {
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_name()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->name().data(), this->name().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ServiceResponse.name"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ServiceResponse)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ServiceResponse)
+  return false;
+#undef DO_
+}
+
+void ServiceResponse::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ServiceResponse)
+  // optional string name = 1;
+  if (this->name().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->name().data(), this->name().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServiceResponse.name");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      1, this->name(), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ServiceResponse)
+}
+
+::google::protobuf::uint8* ServiceResponse::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ServiceResponse)
+  // optional string name = 1;
+  if (this->name().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->name().data(), this->name().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ServiceResponse.name");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        1, this->name(), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ServiceResponse)
+  return target;
+}
+
+int ServiceResponse::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ServiceResponse)
+  int total_size = 0;
+
+  // optional string name = 1;
+  if (this->name().size() > 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::StringSize(
+        this->name());
+  }
+
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ServiceResponse::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ServiceResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ServiceResponse* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ServiceResponse>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ServiceResponse)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ServiceResponse)
+    MergeFrom(*source);
+  }
+}
+
+void ServiceResponse::MergeFrom(const ServiceResponse& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ServiceResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  if (from.name().size() > 0) {
+
+    name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_);
+  }
+}
+
+void ServiceResponse::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ServiceResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ServiceResponse::CopyFrom(const ServiceResponse& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ServiceResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ServiceResponse::IsInitialized() const {
+
+  return true;
+}
+
+void ServiceResponse::Swap(ServiceResponse* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ServiceResponse::InternalSwap(ServiceResponse* other) {
+  name_.Swap(&other->name_);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ServiceResponse::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ServiceResponse_descriptor_;
+  metadata.reflection = ServiceResponse_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ServiceResponse
+
+// optional string name = 1;
+void ServiceResponse::clear_name() {
+  name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ const ::std::string& ServiceResponse::name() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ServiceResponse.name)
+  return name_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ServiceResponse::set_name(const ::std::string& value) {
+  
+  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ServiceResponse.name)
+}
+ void ServiceResponse::set_name(const char* value) {
+  
+  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ServiceResponse.name)
+}
+ void ServiceResponse::set_name(const char* value, size_t size) {
+  
+  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ServiceResponse.name)
+}
+ ::std::string* ServiceResponse::mutable_name() {
+  
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ServiceResponse.name)
+  return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ServiceResponse::release_name() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ServiceResponse.name)
+  
+  return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ServiceResponse::set_allocated_name(::std::string* name) {
+  if (name != NULL) {
+    
+  } else {
+    
+  }
+  name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name);
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ServiceResponse.name)
+}
+
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// ===================================================================
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1900
+const int ErrorResponse::kErrorCodeFieldNumber;
+const int ErrorResponse::kErrorMessageFieldNumber;
+#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
+
+ErrorResponse::ErrorResponse()
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
+  SharedCtor();
+  // @@protoc_insertion_point(constructor:grpc.reflection.v1alpha.ErrorResponse)
+}
+
+void ErrorResponse::InitAsDefaultInstance() {
+  _is_default_instance_ = true;
+}
+
+ErrorResponse::ErrorResponse(const ErrorResponse& from)
+  : ::google::protobuf::Message(),
+    _internal_metadata_(NULL) {
+  SharedCtor();
+  MergeFrom(from);
+  // @@protoc_insertion_point(copy_constructor:grpc.reflection.v1alpha.ErrorResponse)
+}
+
+void ErrorResponse::SharedCtor() {
+    _is_default_instance_ = false;
+  ::google::protobuf::internal::GetEmptyString();
+  _cached_size_ = 0;
+  error_code_ = 0;
+  error_message_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+
+ErrorResponse::~ErrorResponse() {
+  // @@protoc_insertion_point(destructor:grpc.reflection.v1alpha.ErrorResponse)
+  SharedDtor();
+}
+
+void ErrorResponse::SharedDtor() {
+  error_message_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+  if (this != default_instance_) {
+  }
+}
+
+void ErrorResponse::SetCachedSize(int size) const {
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* ErrorResponse::descriptor() {
+  protobuf_AssignDescriptorsOnce();
+  return ErrorResponse_descriptor_;
+}
+
+const ErrorResponse& ErrorResponse::default_instance() {
+  if (default_instance_ == NULL) protobuf_AddDesc_reflection_2eproto();
+  return *default_instance_;
+}
+
+ErrorResponse* ErrorResponse::default_instance_ = NULL;
+
+ErrorResponse* ErrorResponse::New(::google::protobuf::Arena* arena) const {
+  ErrorResponse* n = new ErrorResponse;
+  if (arena != NULL) {
+    arena->Own(n);
+  }
+  return n;
+}
+
+void ErrorResponse::Clear() {
+// @@protoc_insertion_point(message_clear_start:grpc.reflection.v1alpha.ErrorResponse)
+  error_code_ = 0;
+  error_message_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+
+bool ErrorResponse::MergePartialFromCodedStream(
+    ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
+  ::google::protobuf::uint32 tag;
+  // @@protoc_insertion_point(parse_start:grpc.reflection.v1alpha.ErrorResponse)
+  for (;;) {
+    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127);
+    tag = p.first;
+    if (!p.second) goto handle_unusual;
+    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+      // optional int32 error_code = 1;
+      case 1: {
+        if (tag == 8) {
+          DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<
+                   ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>(
+                 input, &error_code_)));
+
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectTag(18)) goto parse_error_message;
+        break;
+      }
+
+      // optional string error_message = 2;
+      case 2: {
+        if (tag == 18) {
+         parse_error_message:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
+                input, this->mutable_error_message()));
+          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+            this->error_message().data(), this->error_message().length(),
+            ::google::protobuf::internal::WireFormatLite::PARSE,
+            "grpc.reflection.v1alpha.ErrorResponse.error_message"));
+        } else {
+          goto handle_unusual;
+        }
+        if (input->ExpectAtEnd()) goto success;
+        break;
+      }
+
+      default: {
+      handle_unusual:
+        if (tag == 0 ||
+            ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+          goto success;
+        }
+        DO_(::google::protobuf::internal::WireFormatLite::SkipField(input, tag));
+        break;
+      }
+    }
+  }
+success:
+  // @@protoc_insertion_point(parse_success:grpc.reflection.v1alpha.ErrorResponse)
+  return true;
+failure:
+  // @@protoc_insertion_point(parse_failure:grpc.reflection.v1alpha.ErrorResponse)
+  return false;
+#undef DO_
+}
+
+void ErrorResponse::SerializeWithCachedSizes(
+    ::google::protobuf::io::CodedOutputStream* output) const {
+  // @@protoc_insertion_point(serialize_start:grpc.reflection.v1alpha.ErrorResponse)
+  // optional int32 error_code = 1;
+  if (this->error_code() != 0) {
+    ::google::protobuf::internal::WireFormatLite::WriteInt32(1, this->error_code(), output);
+  }
+
+  // optional string error_message = 2;
+  if (this->error_message().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->error_message().data(), this->error_message().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ErrorResponse.error_message");
+    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
+      2, this->error_message(), output);
+  }
+
+  // @@protoc_insertion_point(serialize_end:grpc.reflection.v1alpha.ErrorResponse)
+}
+
+::google::protobuf::uint8* ErrorResponse::SerializeWithCachedSizesToArray(
+    ::google::protobuf::uint8* target) const {
+  // @@protoc_insertion_point(serialize_to_array_start:grpc.reflection.v1alpha.ErrorResponse)
+  // optional int32 error_code = 1;
+  if (this->error_code() != 0) {
+    target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(1, this->error_code(), target);
+  }
+
+  // optional string error_message = 2;
+  if (this->error_message().size() > 0) {
+    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
+      this->error_message().data(), this->error_message().length(),
+      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
+      "grpc.reflection.v1alpha.ErrorResponse.error_message");
+    target =
+      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
+        2, this->error_message(), target);
+  }
+
+  // @@protoc_insertion_point(serialize_to_array_end:grpc.reflection.v1alpha.ErrorResponse)
+  return target;
+}
+
+int ErrorResponse::ByteSize() const {
+// @@protoc_insertion_point(message_byte_size_start:grpc.reflection.v1alpha.ErrorResponse)
+  int total_size = 0;
+
+  // optional int32 error_code = 1;
+  if (this->error_code() != 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::Int32Size(
+        this->error_code());
+  }
+
+  // optional string error_message = 2;
+  if (this->error_message().size() > 0) {
+    total_size += 1 +
+      ::google::protobuf::internal::WireFormatLite::StringSize(
+        this->error_message());
+  }
+
+  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+  _cached_size_ = total_size;
+  GOOGLE_SAFE_CONCURRENT_WRITES_END();
+  return total_size;
+}
+
+void ErrorResponse::MergeFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_merge_from_start:grpc.reflection.v1alpha.ErrorResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  const ErrorResponse* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const ErrorResponse>(
+          &from);
+  if (source == NULL) {
+  // @@protoc_insertion_point(generalized_merge_from_cast_fail:grpc.reflection.v1alpha.ErrorResponse)
+    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+  } else {
+  // @@protoc_insertion_point(generalized_merge_from_cast_success:grpc.reflection.v1alpha.ErrorResponse)
+    MergeFrom(*source);
+  }
+}
+
+void ErrorResponse::MergeFrom(const ErrorResponse& from) {
+// @@protoc_insertion_point(class_specific_merge_from_start:grpc.reflection.v1alpha.ErrorResponse)
+  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
+  if (from.error_code() != 0) {
+    set_error_code(from.error_code());
+  }
+  if (from.error_message().size() > 0) {
+
+    error_message_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.error_message_);
+  }
+}
+
+void ErrorResponse::CopyFrom(const ::google::protobuf::Message& from) {
+// @@protoc_insertion_point(generalized_copy_from_start:grpc.reflection.v1alpha.ErrorResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+void ErrorResponse::CopyFrom(const ErrorResponse& from) {
+// @@protoc_insertion_point(class_specific_copy_from_start:grpc.reflection.v1alpha.ErrorResponse)
+  if (&from == this) return;
+  Clear();
+  MergeFrom(from);
+}
+
+bool ErrorResponse::IsInitialized() const {
+
+  return true;
+}
+
+void ErrorResponse::Swap(ErrorResponse* other) {
+  if (other == this) return;
+  InternalSwap(other);
+}
+void ErrorResponse::InternalSwap(ErrorResponse* other) {
+  std::swap(error_code_, other->error_code_);
+  error_message_.Swap(&other->error_message_);
+  _internal_metadata_.Swap(&other->_internal_metadata_);
+  std::swap(_cached_size_, other->_cached_size_);
+}
+
+::google::protobuf::Metadata ErrorResponse::GetMetadata() const {
+  protobuf_AssignDescriptorsOnce();
+  ::google::protobuf::Metadata metadata;
+  metadata.descriptor = ErrorResponse_descriptor_;
+  metadata.reflection = ErrorResponse_reflection_;
+  return metadata;
+}
+
+#if PROTOBUF_INLINE_NOT_IN_HEADERS
+// ErrorResponse
+
+// optional int32 error_code = 1;
+void ErrorResponse::clear_error_code() {
+  error_code_ = 0;
+}
+ ::google::protobuf::int32 ErrorResponse::error_code() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ErrorResponse.error_code)
+  return error_code_;
+}
+ void ErrorResponse::set_error_code(::google::protobuf::int32 value) {
+  
+  error_code_ = value;
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ErrorResponse.error_code)
+}
+
+// optional string error_message = 2;
+void ErrorResponse::clear_error_message() {
+  error_message_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ const ::std::string& ErrorResponse::error_message() const {
+  // @@protoc_insertion_point(field_get:grpc.reflection.v1alpha.ErrorResponse.error_message)
+  return error_message_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ErrorResponse::set_error_message(const ::std::string& value) {
+  
+  error_message_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
+  // @@protoc_insertion_point(field_set:grpc.reflection.v1alpha.ErrorResponse.error_message)
+}
+ void ErrorResponse::set_error_message(const char* value) {
+  
+  error_message_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
+  // @@protoc_insertion_point(field_set_char:grpc.reflection.v1alpha.ErrorResponse.error_message)
+}
+ void ErrorResponse::set_error_message(const char* value, size_t size) {
+  
+  error_message_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
+      ::std::string(reinterpret_cast<const char*>(value), size));
+  // @@protoc_insertion_point(field_set_pointer:grpc.reflection.v1alpha.ErrorResponse.error_message)
+}
+ ::std::string* ErrorResponse::mutable_error_message() {
+  
+  // @@protoc_insertion_point(field_mutable:grpc.reflection.v1alpha.ErrorResponse.error_message)
+  return error_message_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ ::std::string* ErrorResponse::release_error_message() {
+  // @@protoc_insertion_point(field_release:grpc.reflection.v1alpha.ErrorResponse.error_message)
+  
+  return error_message_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
+}
+ void ErrorResponse::set_allocated_error_message(::std::string* error_message) {
+  if (error_message != NULL) {
+    
+  } else {
+    
+  }
+  error_message_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), error_message);
+  // @@protoc_insertion_point(field_set_allocated:grpc.reflection.v1alpha.ErrorResponse.error_message)
+}
+
+#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
+
+// @@protoc_insertion_point(namespace_scope)
+
+}  // namespace v1alpha
+}  // namespace reflection
+}  // namespace grpc
+
+// @@protoc_insertion_point(global_scope)
diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc
index fafe31e84c..fb4c68ebe4 100644
--- a/src/cpp/server/server.cc
+++ b/src/cpp/server/server.cc
@@ -33,6 +33,7 @@
 
 #include <grpc++/server.h>
 
+#include <sstream>
 #include <utility>
 
 #include <grpc++/completion_queue.h>
@@ -41,6 +42,7 @@
 #include <grpc++/impl/grpc_library.h>
 #include <grpc++/impl/method_handler_impl.h>
 #include <grpc++/impl/rpc_service_method.h>
+#include <grpc++/impl/server_initializer.h>
 #include <grpc++/impl/service_type.h>
 #include <grpc++/security/server_credentials.h>
 #include <grpc++/server_context.h>
@@ -65,7 +67,7 @@ static std::shared_ptr<Server::GlobalCallbacks> g_callbacks = nullptr;
 static gpr_once g_once_init_callbacks = GPR_ONCE_INIT;
 
 static void InitGlobalCallbacks() {
-  if (g_callbacks == nullptr) {
+  if (!g_callbacks) {
     g_callbacks.reset(new DefaultGlobalCallbacks());
   }
 }
@@ -284,7 +286,8 @@ Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
       has_generic_service_(false),
       server_(nullptr),
       thread_pool_(thread_pool),
-      thread_pool_owned_(thread_pool_owned) {
+      thread_pool_owned_(thread_pool_owned),
+      server_initializer_(new ServerInitializer(this)) {
   g_gli_initializer.summon();
   gpr_once_init(&g_once_init_callbacks, InitGlobalCallbacks);
   global_callbacks_ = g_callbacks;
@@ -292,7 +295,12 @@ Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
   grpc_channel_args channel_args;
   args->SetChannelArgs(&channel_args);
   server_ = grpc_server_create(&channel_args, nullptr);
-  grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
+  if (thread_pool_ == nullptr) {
+    grpc_server_register_non_listening_completion_queue(server_, cq_.cq(),
+                                                        nullptr);
+  } else {
+    grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
+  }
 }
 
 Server::~Server() {
@@ -316,11 +324,15 @@ Server::~Server() {
 }
 
 void Server::SetGlobalCallbacks(GlobalCallbacks* callbacks) {
-  GPR_ASSERT(g_callbacks == nullptr);
-  GPR_ASSERT(callbacks != nullptr);
+  GPR_ASSERT(!g_callbacks);
+  GPR_ASSERT(callbacks);
   g_callbacks.reset(callbacks);
 }
 
+grpc_server* Server::c_server() { return server_; }
+
+CompletionQueue* Server::completion_queue() { return &cq_; }
+
 static grpc_server_register_method_payload_handling PayloadHandlingForMethod(
     RpcServiceMethod* method) {
   switch (method->method_type()) {
@@ -341,6 +353,7 @@ bool Server::RegisterService(const grpc::string* host, Service* service) {
                "Can only register an asynchronous service against one server.");
     service->server_ = this;
   }
+  const char* method_name = nullptr;
   for (auto it = service->methods_.begin(); it != service->methods_.end();
        ++it) {
     if (it->get() == nullptr) {  // Handled by generic service if any.
@@ -360,6 +373,17 @@ bool Server::RegisterService(const grpc::string* host, Service* service) {
     } else {
       sync_methods_->emplace_back(method, tag);
     }
+    method_name = method->name();
+  }
+
+  // Parse service name.
+  if (method_name != nullptr) {
+    std::stringstream ss(method_name);
+    grpc::string service_name;
+    if (std::getline(ss, service_name, '/') &&
+        std::getline(ss, service_name, '/')) {
+      services_.push_back(service_name);
+    }
   }
   return true;
 }
@@ -392,7 +416,9 @@ bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) {
       sync_methods_->push_back(SyncRequest(unknown_method_.get(), nullptr));
     }
     for (size_t i = 0; i < num_cqs; i++) {
-      new UnimplementedAsyncRequest(this, cqs[i]);
+      if (cqs[i]->IsFrequentlyPolled()) {
+        new UnimplementedAsyncRequest(this, cqs[i]);
+      }
     }
   }
   // Start processing rpcs.
@@ -598,4 +624,6 @@ void Server::RunRpc() {
   }
 }
 
+ServerInitializer* Server::initializer() { return server_initializer_.get(); }
+
 }  // namespace grpc
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 68cc38258c..45bb858e2e 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,58 +37,114 @@
 #include <grpc++/server.h>
 #include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
+
+#include "include/grpc/support/useful.h"
 #include "src/cpp/server/thread_pool_interface.h"
 
 namespace grpc {
 
+static std::vector<std::unique_ptr<ServerBuilderPlugin> (*)()>*
+    g_plugin_factory_list;
+static gpr_once once_init_plugin_list = GPR_ONCE_INIT;
+
+static void do_plugin_list_init(void) {
+  g_plugin_factory_list =
+      new std::vector<std::unique_ptr<ServerBuilderPlugin> (*)()>();
+}
+
 ServerBuilder::ServerBuilder()
     : max_message_size_(-1), generic_service_(nullptr) {
-  grpc_compression_options_init(&compression_options_);
+  gpr_once_init(&once_init_plugin_list, do_plugin_list_init);
+  for (auto it = g_plugin_factory_list->begin();
+       it != g_plugin_factory_list->end(); it++) {
+    auto& factory = *it;
+    plugins_.emplace_back(factory());
+  }
+  // all compression algorithms enabled by default.
+  enabled_compression_algorithms_bitset_ =
+      (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
+  memset(&maybe_default_compression_level_, 0,
+         sizeof(maybe_default_compression_level_));
+  memset(&maybe_default_compression_algorithm_, 0,
+         sizeof(maybe_default_compression_algorithm_));
 }
 
-std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue() {
-  ServerCompletionQueue* cq = new ServerCompletionQueue();
+std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue(
+    bool is_frequently_polled) {
+  ServerCompletionQueue* cq = new ServerCompletionQueue(is_frequently_polled);
   cqs_.push_back(cq);
   return std::unique_ptr<ServerCompletionQueue>(cq);
 }
 
-void ServerBuilder::RegisterService(Service* service) {
+ServerBuilder& ServerBuilder::RegisterService(Service* service) {
   services_.emplace_back(new NamedService(service));
+  return *this;
 }
 
-void ServerBuilder::RegisterService(const grpc::string& addr,
-                                    Service* service) {
+ServerBuilder& ServerBuilder::RegisterService(const grpc::string& addr,
+                                              Service* service) {
   services_.emplace_back(new NamedService(addr, service));
+  return *this;
 }
 
-void ServerBuilder::RegisterAsyncGenericService(AsyncGenericService* service) {
+ServerBuilder& ServerBuilder::RegisterAsyncGenericService(
+    AsyncGenericService* service) {
   if (generic_service_) {
     gpr_log(GPR_ERROR,
             "Adding multiple AsyncGenericService is unsupported for now. "
             "Dropping the service %p",
             service);
-    return;
+  } else {
+    generic_service_ = service;
   }
-  generic_service_ = service;
+  return *this;
 }
 
-void ServerBuilder::SetOption(std::unique_ptr<ServerBuilderOption> option) {
+ServerBuilder& ServerBuilder::SetOption(
+    std::unique_ptr<ServerBuilderOption> option) {
   options_.push_back(std::move(option));
+  return *this;
 }
 
-void ServerBuilder::AddListeningPort(const grpc::string& addr,
-                                     std::shared_ptr<ServerCredentials> creds,
-                                     int* selected_port) {
+ServerBuilder& ServerBuilder::SetCompressionAlgorithmSupportStatus(
+    grpc_compression_algorithm algorithm, bool enabled) {
+  if (enabled) {
+    GPR_BITSET(&enabled_compression_algorithms_bitset_, algorithm);
+  } else {
+    GPR_BITCLEAR(&enabled_compression_algorithms_bitset_, algorithm);
+  }
+  return *this;
+}
+
+ServerBuilder& ServerBuilder::SetDefaultCompressionLevel(
+    grpc_compression_level level) {
+  maybe_default_compression_level_.level = level;
+  return *this;
+}
+
+ServerBuilder& ServerBuilder::SetDefaultCompressionAlgorithm(
+    grpc_compression_algorithm algorithm) {
+  maybe_default_compression_algorithm_.is_set = true;
+  maybe_default_compression_algorithm_.algorithm = algorithm;
+  return *this;
+}
+
+ServerBuilder& ServerBuilder::AddListeningPort(
+    const grpc::string& addr, std::shared_ptr<ServerCredentials> creds,
+    int* selected_port) {
   Port port = {addr, creds, selected_port};
   ports_.push_back(port);
+  return *this;
 }
 
 std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<ThreadPoolInterface> thread_pool;
+  bool has_sync_methods = false;
   for (auto it = services_.begin(); it != services_.end(); ++it) {
     if ((*it)->service->has_synchronous_methods()) {
-      if (thread_pool == nullptr) {
+      if (!thread_pool) {
         thread_pool.reset(CreateDefaultThreadPool());
+        has_sync_methods = true;
         break;
       }
     }
@@ -96,24 +152,69 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   ChannelArguments args;
   for (auto option = options_.begin(); option != options_.end(); ++option) {
     (*option)->UpdateArguments(&args);
+    (*option)->UpdatePlugins(&plugins_);
+  }
+  if (!thread_pool) {
+    for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
+      if ((*plugin)->has_sync_methods()) {
+        thread_pool.reset(CreateDefaultThreadPool());
+        has_sync_methods = true;
+        break;
+      }
+    }
   }
   if (max_message_size_ > 0) {
     args.SetInt(GRPC_ARG_MAX_MESSAGE_LENGTH, max_message_size_);
   }
-  args.SetInt(GRPC_COMPRESSION_ALGORITHM_STATE_ARG,
-              compression_options_.enabled_algorithms_bitset);
+  args.SetInt(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET,
+              enabled_compression_algorithms_bitset_);
+  if (maybe_default_compression_level_.is_set) {
+    args.SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL,
+                maybe_default_compression_level_.level);
+  }
+  if (maybe_default_compression_algorithm_.is_set) {
+    args.SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM,
+                maybe_default_compression_algorithm_.algorithm);
+  }
   std::unique_ptr<Server> server(
       new Server(thread_pool.release(), true, max_message_size_, &args));
+  ServerInitializer* initializer = server->initializer();
+
+  // If the server has atleast one sync methods, we know that this is a Sync
+  // server or a Hybrid server and the completion queue (server->cq_) would be
+  // frequently polled.
+  int num_frequently_polled_cqs = has_sync_methods ? 1 : 0;
+
   for (auto cq = cqs_.begin(); cq != cqs_.end(); ++cq) {
-    grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
-                                          nullptr);
+    // A completion queue that is not polled frequently (by calling Next() or
+    // AsyncNext()) is not safe to use for listening to incoming channels.
+    // Register all such completion queues as non-listening completion queues
+    // with the GRPC core library.
+    if ((*cq)->IsFrequentlyPolled()) {
+      grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
+                                            nullptr);
+      num_frequently_polled_cqs++;
+    } else {
+      grpc_server_register_non_listening_completion_queue(server->server_,
+                                                          (*cq)->cq(), nullptr);
+    }
+  }
+
+  if (num_frequently_polled_cqs == 0) {
+    gpr_log(GPR_ERROR,
+            "At least one of the completion queues must be frequently polled");
+    return nullptr;
   }
+
   for (auto service = services_.begin(); service != services_.end();
        service++) {
     if (!server->RegisterService((*service)->host.get(), (*service)->service)) {
       return nullptr;
     }
   }
+  for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
+    (*plugin)->InitServer(initializer);
+  }
   if (generic_service_) {
     server->RegisterAsyncGenericService(generic_service_);
   } else {
@@ -137,7 +238,16 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   if (!server->Start(cqs_data, cqs_.size())) {
     return nullptr;
   }
+  for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
+    (*plugin)->Finish(initializer);
+  }
   return server;
 }
 
+void ServerBuilder::InternalAddPluginFactory(
+    std::unique_ptr<ServerBuilderPlugin> (*CreatePlugin)()) {
+  gpr_once_init(&once_init_plugin_list, do_plugin_list_init);
+  (*g_plugin_factory_list).push_back(CreatePlugin);
+}
+
 }  // namespace grpc
diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc
index 204fef1b09..43117fd1e9 100644
--- a/src/cpp/server/server_context.cc
+++ b/src/cpp/server/server_context.cc
@@ -42,7 +42,6 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/channel/compress_filter.h"
 #include "src/core/lib/surface/call.h"
 
 namespace grpc {
@@ -196,6 +195,9 @@ bool ServerContext::IsCancelled() const {
 }
 
 void ServerContext::set_compression_level(grpc_compression_level level) {
+  // TODO(dgq): get rid of grpc_call_compression_for_level and propagate the
+  // compression level by adding a new argument to
+  // CallOpSendInitialMetadata::SendInitialMetadata.
   const grpc_compression_algorithm algorithm_for_level =
       grpc_call_compression_for_level(call_, level);
   set_compression_algorithm(algorithm_for_level);
@@ -210,7 +212,7 @@ void ServerContext::set_compression_algorithm(
     abort();
   }
   GPR_ASSERT(algorithm_name != NULL);
-  AddInitialMetadata(GRPC_COMPRESS_REQUEST_ALGORITHM_KEY, algorithm_name);
+  AddInitialMetadata(GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, algorithm_name);
 }
 
 grpc::string ServerContext::peer() const {
diff --git a/src/cpp/server/server_posix.cc b/src/cpp/server/server_posix.cc
new file mode 100644
index 0000000000..c3aa2adc60
--- /dev/null
+++ b/src/cpp/server/server_posix.cc
@@ -0,0 +1,49 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/server_posix.h>
+
+#include <grpc/grpc_posix.h>
+
+namespace grpc {
+
+#ifdef GPR_SUPPORT_CHANNELS_FROM_FD
+
+void AddInsecureChannelFromFd(Server* server, int fd) {
+  grpc_server_add_insecure_channel_from_fd(
+      server->c_server(), server->completion_queue()->cq(), fd);
+}
+
+#endif  // GPR_SUPPORT_CHANNELS_FROM_FD
+
+}  // namespace grpc
diff --git a/src/csharp/.gitignore b/src/csharp/.gitignore
index 0f96a48221..fc2875a1dd 100644
--- a/src/csharp/.gitignore
+++ b/src/csharp/.gitignore
@@ -1,5 +1,7 @@
+*.xproj.user
 *.userprefs
 *.csproj.user
+*.lock.json
 StyleCop.Cache
 test-results
 packages
diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj
index 3acea7d2f8..1fa14fc3df 100644
--- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj
+++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj
@@ -81,6 +81,7 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="Grpc.Auth.nuspec" />
+    <None Include="Grpc.Auth.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.project.json b/src/csharp/Grpc.Auth/Grpc.Auth.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Auth/Grpc.Auth.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.xproj b/src/csharp/Grpc.Auth/Grpc.Auth.xproj
new file mode 100644
index 0000000000..dd3d94c574
--- /dev/null
+++ b/src/csharp/Grpc.Auth/Grpc.Auth.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>c82631ed-06d1-4458-87bc-8257d12307a8</ProjectGuid>
+    <RootNamespace>Grpc.Auth</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\Grpc.Core\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json
new file mode 100644
index 0000000000..72c258a91a
--- /dev/null
+++ b/src/csharp/Grpc.Auth/project.json
@@ -0,0 +1,41 @@
+{
+  "version": "0.16.0-dev",
+  "title": "gRPC C# Auth",
+  "authors": [ "Google Inc." ],
+  "copyright": "Copyright 2015, Google Inc.",
+  "packOptions": {
+    "summary": "Auth library for C# implementation of gRPC - an RPC library and framework",
+    "description": "Auth library for C# implementation of gRPC - an RPC library and framework. See project site for more info.",
+    "owners": [ "grpc-packages" ],
+    "licenseUrl": "https://github.com/grpc/grpc/blob/master/LICENSE",
+    "projectUrl": "https://github.com/grpc/grpc",
+    "requireLicenseAcceptance": false,
+    "tags": [ "gRPC RPC Protocol HTTP/2 Auth OAuth2" ],
+  },
+  "buildOptions": {
+    "define": [ "SIGNED" ],
+    "keyFile": "../keys/Grpc.snk",
+    "publicSign": true,
+    "xmlDoc": true,
+    "compile": {
+      "includeFiles": [ "../Grpc.Core/Version.cs" ]
+    }
+  },
+  "dependencies": {
+    "Grpc.Core": "0.16.0-dev",
+    "Google.Apis.Auth": "1.11.1"
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "net45"
+      ],
+      "dependencies": {
+        "Microsoft.NETCore.Portable.Compatibility": "1.0.1-rc2-24027",
+        "NETStandard.Library": "1.5.0-rc2-24027",
+        "System.Threading.Tasks": "4.0.11-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs b/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs
new file mode 100644
index 0000000000..064bc13cab
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs
@@ -0,0 +1,92 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Threading.Tasks;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class AppDomainUnloadTest
+    {
+#if NETSTANDARD1_5
+        [Test]
+        [Ignore("Not supported for CoreCLR")]
+        public void AppDomainUnloadHookCanCleanupAbandonedCall()
+        {
+        }
+#else
+        [Test]
+        public void AppDomainUnloadHookCanCleanupAbandonedCall()
+        {
+            var setup = new AppDomainSetup
+            {
+                ApplicationBase = AppDomain.CurrentDomain.BaseDirectory
+            };
+            var childDomain = AppDomain.CreateDomain("test", null, setup);
+            var remoteObj = childDomain.CreateInstance(typeof(AppDomainTestClass).Assembly.GetName().Name, typeof(AppDomainTestClass).FullName);
+
+            // Try to unload the appdomain once we've created a server and a channel inside the appdomain.
+            AppDomain.Unload(childDomain);
+        }
+
+        public class AppDomainTestClass
+        {
+            const string Host = "127.0.0.1";
+
+            /// <summary>
+            /// Creates a server and a channel and initiates a call. The code is invoked from inside of an AppDomain
+            /// to test if AppDomain.Unload() work if Grpc is being used.
+            /// </summary>
+            public AppDomainTestClass()
+            {
+                var helper = new MockServiceHelper(Host);
+                var server = helper.GetServer();
+                server.Start();
+                var channel = helper.GetChannel();
+
+                var readyToShutdown = new TaskCompletionSource<object>();
+                helper.DuplexStreamingHandler = new DuplexStreamingServerMethod<string, string>(async (requestStream, responseStream, context) =>
+                {
+                    readyToShutdown.SetResult(null);
+                    await requestStream.ToListAsync();
+                });
+
+                var call = Calls.AsyncDuplexStreamingCall(helper.CreateDuplexStreamingCall());
+                readyToShutdown.Task.Wait();  // make sure handler is running
+            }
+        }
+#endif
+    }
+}
diff --git a/src/csharp/Grpc.Core.Tests/ChannelTest.cs b/src/csharp/Grpc.Core.Tests/ChannelTest.cs
index 850d70ce92..db0ef3a4cd 100644
--- a/src/csharp/Grpc.Core.Tests/ChannelTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ChannelTest.cs
@@ -71,7 +71,7 @@ namespace Grpc.Core.Tests
         public void WaitForStateChangedAsync_InvalidArgument()
         {
             var channel = new Channel("localhost", ChannelCredentials.Insecure);
-            Assert.ThrowsAsync(typeof(ArgumentException), async () => await channel.WaitForStateChangedAsync(ChannelState.FatalFailure));
+            Assert.ThrowsAsync(typeof(ArgumentException), async () => await channel.WaitForStateChangedAsync(ChannelState.Shutdown));
             channel.ShutdownAsync().Wait();
         }
 
@@ -102,11 +102,11 @@ namespace Grpc.Core.Tests
         }
 
         [Test]
-        public async Task StateIsFatalFailureAfterShutdown()
+        public async Task StateIsShutdownAfterShutdown()
         {
             var channel = new Channel("localhost", ChannelCredentials.Insecure);
             await channel.ShutdownAsync();
-            Assert.AreEqual(ChannelState.FatalFailure, channel.State);
+            Assert.AreEqual(ChannelState.Shutdown, channel.State);
         }
 
         [Test]
diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
index d92addbf54..dcdddc769e 100644
--- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
@@ -235,8 +235,16 @@ namespace Grpc.Core.Tests
             await barrier.Task;  // make sure the handler has started.
             cts.Cancel();
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseAsync);
-            Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            try
+            {
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await call.ResponseAsync;
+                Assert.Fail();
+            }
+            catch (RpcException ex)
+            {
+                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            }
         }
 
         [Test]
@@ -265,9 +273,15 @@ namespace Grpc.Core.Tests
             await handlerStartedBarrier.Task;
             cts.Cancel();
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseAsync);
-            Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
-
+            try
+            {
+                await call.ResponseAsync;
+                Assert.Fail();
+            }
+            catch (RpcException ex)
+            {
+                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            }
             Assert.AreEqual("SUCCESS", await successTcs.Task);
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/CompressionTest.cs b/src/csharp/Grpc.Core.Tests/CompressionTest.cs
index 378c81851c..16be210508 100644
--- a/src/csharp/Grpc.Core.Tests/CompressionTest.cs
+++ b/src/csharp/Grpc.Core.Tests/CompressionTest.cs
@@ -34,6 +34,7 @@
 using System;
 using System.Diagnostics;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Grpc.Core;
@@ -118,5 +119,30 @@ namespace Grpc.Core.Tests
 
             await call.ResponseStream.ToListAsync();
         }
+
+        [Test]
+        public void CanReadCompressedMessages()
+        {
+            var compressionMetadata = new Metadata
+            {
+                { new Metadata.Entry(Metadata.CompressionRequestAlgorithmMetadataKey, "gzip") }
+            };
+
+            helper.UnaryHandler = new UnaryServerMethod<string, string>(async (req, context) =>
+            {
+                await context.WriteResponseHeadersAsync(compressionMetadata);
+                return req;
+            });
+
+            var stringBuilder = new StringBuilder();
+            for (int i = 0; i < 200000; i++)
+            {
+                stringBuilder.Append('a');
+            }
+            var request = stringBuilder.ToString();
+            var response = Calls.BlockingUnaryCall(helper.CreateUnaryCall(new CallOptions(compressionMetadata)), request);
+
+            Assert.AreEqual(request, response);
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
index cec8c7ce6b..6a156293ad 100644
--- a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
@@ -105,7 +105,15 @@ namespace Grpc.Core.Tests
             var parentCall = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall(new CallOptions(cancellationToken: cts.Token)));
             await readyToCancelTcs.Task;
             cts.Cancel();
-            Assert.ThrowsAsync(typeof(RpcException), async () => await parentCall);
+            try
+            {
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await parentCall;
+                Assert.Fail();
+            }
+            catch (RpcException)
+            {
+            }
             Assert.AreEqual("CHILD_CALL_CANCELLED", await successTcs.Task);
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
index 47131fc454..f6c226567d 100644
--- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
+++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
@@ -86,6 +86,10 @@
     <Compile Include="NUnitMain.cs" />
     <Compile Include="Internal\FakeNativeCall.cs" />
     <Compile Include="Internal\AsyncCallServerTest.cs" />
+    <Compile Include="ShutdownHookServerTest.cs" />
+    <Compile Include="ShutdownHookPendingCallTest.cs" />
+    <Compile Include="ShutdownHookClientTest.cs" />
+    <Compile Include="AppDomainUnloadTest.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>
@@ -95,6 +99,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.Core.Tests.project.json" />
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.project.json b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.xproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.xproj
new file mode 100644
index 0000000000..0582329154
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>759e23b2-fc04-4695-902d-b073cded3599</ProjectGuid>
+    <RootNamespace>Grpc.Core.Tests</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
index ab12c120cb..3ec2cf48cd 100644
--- a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
+++ b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
@@ -32,7 +32,7 @@
 #endregion
 
 using System;
-using System.Threading;
+using System.Linq;
 using Grpc.Core;
 using NUnit.Framework;
 
@@ -44,8 +44,12 @@ namespace Grpc.Core.Tests
         public void InitializeAndShutdownGrpcEnvironment()
         {
             var env = GrpcEnvironment.AddRef();
-            Assert.IsNotNull(env.CompletionQueue);
-            GrpcEnvironment.Release();
+            Assert.IsTrue(env.CompletionQueues.Count > 0);
+            for (int i = 0; i < env.CompletionQueues.Count; i++)
+            {
+                Assert.IsNotNull(env.CompletionQueues.ElementAt(i));
+            }
+            GrpcEnvironment.ReleaseAsync().Wait();
         }
 
         [Test]
@@ -54,8 +58,8 @@ namespace Grpc.Core.Tests
             var env1 = GrpcEnvironment.AddRef();
             var env2 = GrpcEnvironment.AddRef();
             Assert.AreSame(env1, env2);
-            GrpcEnvironment.Release();
-            GrpcEnvironment.Release();
+            GrpcEnvironment.ReleaseAsync().Wait();
+            GrpcEnvironment.ReleaseAsync().Wait();
         }
 
         [Test]
@@ -64,10 +68,10 @@ namespace Grpc.Core.Tests
             Assert.AreEqual(0, GrpcEnvironment.GetRefCount());
 
             var env1 = GrpcEnvironment.AddRef();
-            GrpcEnvironment.Release();
+            GrpcEnvironment.ReleaseAsync().Wait();
 
             var env2 = GrpcEnvironment.AddRef();
-            GrpcEnvironment.Release();
+            GrpcEnvironment.ReleaseAsync().Wait();
 
             Assert.AreNotSame(env1, env2);
         }
@@ -76,7 +80,7 @@ namespace Grpc.Core.Tests
         public void ReleaseWithoutAddRef()
         {
             Assert.AreEqual(0, GrpcEnvironment.GetRefCount());
-            Assert.Throws(typeof(InvalidOperationException), () => GrpcEnvironment.Release());
+            Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await GrpcEnvironment.ReleaseAsync());
         }
 
         [Test]
diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
index 0e204761f6..c35aaf680f 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
@@ -53,8 +53,6 @@ namespace Grpc.Core.Internal.Tests
         [SetUp]
         public void Init()
         {
-            var environment = GrpcEnvironment.AddRef();
-
             // Create a fake server just so we have an instance to refer to.
             // The server won't actually be used at all.
             server = new Server()
@@ -66,7 +64,6 @@ namespace Grpc.Core.Internal.Tests
             fakeCall = new FakeNativeCall();
             asyncCallServer = new AsyncCallServer<string, string>(
                 Marshallers.StringMarshaller.Serializer, Marshallers.StringMarshaller.Deserializer,
-                environment,
                 server);
             asyncCallServer.InitializeForTesting(fakeCall);
         }
@@ -75,7 +72,6 @@ namespace Grpc.Core.Internal.Tests
         public void Cleanup()
         {
             server.ShutdownAsync().Wait();
-            GrpcEnvironment.Release();
         }
 
         [Test]
@@ -136,7 +132,6 @@ namespace Grpc.Core.Internal.Tests
         public void WriteAfterCancelNotificationFails()
         {
             var finishedTask = asyncCallServer.ServerSideCallAsync();
-            var requestStream = new ServerRequestStream<string, string>(asyncCallServer);
             var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
 
             fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
@@ -181,6 +176,21 @@ namespace Grpc.Core.Internal.Tests
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
 
+        [Test]
+        public void WriteAfterWriteStatusThrowsInvalidOperationException()
+        {
+            var finishedTask = asyncCallServer.ServerSideCallAsync();
+            var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
+
+            asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata(), null);
+            Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await responseStream.WriteAsync("request1"));
+
+            fakeCall.SendStatusFromServerHandler(true);
+            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+
+            AssertFinished(asyncCallServer, fakeCall, finishedTask);
+        }
+
         static void AssertFinished(AsyncCallServer<string, string> asyncCallServer, FakeNativeCall fakeCall, Task finishedTask)
         {
             Assert.IsTrue(fakeCall.IsDisposed);
diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
index 777a1c8c50..98e27a17a1 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
@@ -33,7 +33,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 
 using Grpc.Core.Internal;
@@ -82,7 +81,7 @@ namespace Grpc.Core.Internal.Tests
             Assert.ThrowsAsync(typeof(InvalidOperationException),
                 async () => await asyncCall.ReadMessageAsync());
             Assert.Throws(typeof(InvalidOperationException),
-                () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {}));
+                () => asyncCall.SendMessageAsync("abc", new WriteFlags()));
         }
 
         [Test]
@@ -103,7 +102,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.UnaryCallAsync("request1");
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument);
@@ -148,7 +147,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.ClientStreamingCallAsync();
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument);
@@ -193,7 +192,7 @@ namespace Grpc.Core.Internal.Tests
 
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.Internal),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Internal);
@@ -211,7 +210,9 @@ namespace Grpc.Core.Internal.Tests
                 new Metadata());
 
             AssertUnaryResponseSuccess(asyncCall, fakeCall, resultTask);
-            var ex = Assert.Throws<RpcException>(() => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(Status.DefaultSuccess, ex.Status);
         }
 
@@ -227,7 +228,9 @@ namespace Grpc.Core.Internal.Tests
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.OutOfRange);
-            var ex = Assert.Throws<RpcException>(() => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(StatusCode.OutOfRange, ex.Status.StatusCode);
         }
 
@@ -267,7 +270,7 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void ClientStreaming_WriteAfterCancellationRequestThrowsOperationCancelledException()
+        public void ClientStreaming_WriteAfterCancellationRequestThrowsTaskCanceledException()
         {
             var resultTask = asyncCall.ClientStreamingCallAsync();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -275,11 +278,12 @@ namespace Grpc.Core.Internal.Tests
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
 
-            Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1"));
+            var writeTask = requestStream.WriteAsync("request1");
+            Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.Cancelled),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Cancelled);
@@ -290,7 +294,7 @@ namespace Grpc.Core.Internal.Tests
         {
             asyncCall.StartServerStreamingCall("request1");
             Assert.Throws(typeof(InvalidOperationException),
-                () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {}));
+                () => asyncCall.SendMessageAsync("abc", new WriteFlags()));
         }
 
         [Test]
@@ -390,12 +394,13 @@ namespace Grpc.Core.Internal.Tests
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await requestStream.WriteAsync("request1"));
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(Status.DefaultSuccess, ex.Status);
         }
 
         [Test]
-        public void DuplexStreaming_CompleteAfterReceivingStatusFails()
+        public void DuplexStreaming_CompleteAfterReceivingStatusSuceeds()
         {
             asyncCall.StartDuplexStreamingCall();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -411,7 +416,7 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void DuplexStreaming_WriteAfterCancellationRequestThrowsOperationCancelledException()
+        public void DuplexStreaming_WriteAfterCancellationRequestThrowsTaskCanceledException()
         {
             asyncCall.StartDuplexStreamingCall();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -419,7 +424,9 @@ namespace Grpc.Core.Internal.Tests
 
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
-            Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             var readTask = responseStream.MoveNext();
             fakeCall.ReceivedMessageHandler(true, null);
diff --git a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
index c6843f10af..e9ec59eb3d 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
@@ -48,7 +48,7 @@ namespace Grpc.Core.Internal.Tests
             GrpcEnvironment.AddRef();
             var cq = CompletionQueueSafeHandle.Create();
             cq.Dispose();
-            GrpcEnvironment.Release();
+            GrpcEnvironment.ReleaseAsync().Wait();
         }
 
         [Test]
@@ -59,8 +59,8 @@ namespace Grpc.Core.Internal.Tests
             cq.Shutdown();
             var ev = cq.Next();
             cq.Dispose();
-            GrpcEnvironment.Release();
-            Assert.AreEqual(GRPCCompletionType.Shutdown, ev.type);
+            GrpcEnvironment.ReleaseAsync().Wait();
+            Assert.AreEqual(CompletionQueueEvent.CompletionType.Shutdown, ev.type);
             Assert.AreNotEqual(IntPtr.Zero, ev.success);
             Assert.AreEqual(IntPtr.Zero, ev.tag);
         }
diff --git a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
index 74f7f2497a..c124ea29af 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
@@ -61,15 +61,15 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void InfFuture()
+        public void InfFutureMatchesNativeValue()
         {
-            var timespec = Timespec.InfFuture;
+            Assert.AreEqual(Timespec.NativeInfFuture, Timespec.InfFuture);
         }
 
         [Test]
-        public void InfPast()
+        public void InfPastMatchesNativeValue()
         {
-            var timespec = Timespec.InfPast;
+            Assert.AreEqual(Timespec.NativeInfPast, Timespec.InfPast);
         }
 
         [Test]
@@ -108,7 +108,7 @@ namespace Grpc.Core.Internal.Tests
             Assert.Throws(typeof(InvalidOperationException),
                 () => new Timespec(0, 1000 * 1000 * 1000).ToDateTime());
             Assert.Throws(typeof(InvalidOperationException),
-                () => new Timespec(0, 0, GPRClockType.Monotonic).ToDateTime());
+                () => new Timespec(0, 0, ClockType.Monotonic).ToDateTime());
         }
 
         [Test]
diff --git a/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs b/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
index 0663e77d1e..d770f82390 100644
--- a/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
+++ b/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
@@ -134,7 +134,15 @@ namespace Grpc.Core.Tests
         {
             helper.ClientStreamingHandler = new ClientStreamingServerMethod<string, string>(async (requestStream, context) =>
             {
-                Assert.ThrowsAsync<IOException>(async () => await requestStream.MoveNext());
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await requestStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (IOException)
+                {
+                }
                 return "RESPONSE";
             });
 
diff --git a/src/csharp/Grpc.Core.Tests/MockServiceHelper.cs b/src/csharp/Grpc.Core.Tests/MockServiceHelper.cs
index 3047314345..4d90470056 100644
--- a/src/csharp/Grpc.Core.Tests/MockServiceHelper.cs
+++ b/src/csharp/Grpc.Core.Tests/MockServiceHelper.cs
@@ -102,7 +102,7 @@ namespace Grpc.Core.Tests
                 marshaller,
                 marshaller);
 
-            serviceDefinition = ServerServiceDefinition.CreateBuilder(ServiceName)
+            serviceDefinition = ServerServiceDefinition.CreateBuilder()
                 .AddMethod(unaryMethod, (request, context) => unaryHandler(request, context))
                 .AddMethod(clientStreamingMethod, (requestStream, context) => clientStreamingHandler(requestStream, context))
                 .AddMethod(serverStreamingMethod, (request, responseStream, context) => serverStreamingHandler(request, responseStream, context))
diff --git a/src/csharp/Grpc.Core.Tests/NUnitMain.cs b/src/csharp/Grpc.Core.Tests/NUnitMain.cs
index 9c1d7bf3c8..24a9f846d1 100644
--- a/src/csharp/Grpc.Core.Tests/NUnitMain.cs
+++ b/src/csharp/Grpc.Core.Tests/NUnitMain.cs
@@ -49,7 +49,7 @@ namespace Grpc.Core.Tests
         {
             // Make logger immune to NUnit capturing stdout and stderr to workaround https://github.com/nunit/nunit/issues/1406.
             GrpcEnvironment.SetLogger(new TextWriterLogger(Console.Error));
-#if DOTNET5_4
+#if NETSTANDARD1_5
             return new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args, new ExtendedTextWrapper(Console.Out), Console.In);
 #else
             return new AutoRun().Execute(args);
diff --git a/src/csharp/Grpc.Core.Tests/NUnitVersionTest.cs b/src/csharp/Grpc.Core.Tests/NUnitVersionTest.cs
index 3fa6ad09c0..1a9e441611 100644
--- a/src/csharp/Grpc.Core.Tests/NUnitVersionTest.cs
+++ b/src/csharp/Grpc.Core.Tests/NUnitVersionTest.cs
@@ -56,7 +56,7 @@ namespace Grpc.Core.Tests
                 Console.Error.WriteLine("You are using and old version of NUnit that doesn't support async tests and skips them instead. " +
                 "This test has failed to indicate that.");
                 Console.Error.Flush();
-                Environment.Exit(1);
+                throw new Exception("NUnitVersionTest has failed.");
             }
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/PInvokeTest.cs b/src/csharp/Grpc.Core.Tests/PInvokeTest.cs
index d2b2fc6a66..d3735c7880 100644
--- a/src/csharp/Grpc.Core.Tests/PInvokeTest.cs
+++ b/src/csharp/Grpc.Core.Tests/PInvokeTest.cs
@@ -65,7 +65,7 @@ namespace Grpc.Core.Tests
                     cq.Dispose();
                 });
 
-            GrpcEnvironment.Release();
+            GrpcEnvironment.ReleaseAsync().Wait();
         }
 
         /// <summary>
diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs
index 3830f0cbac..501992c569 100644
--- a/src/csharp/Grpc.Core.Tests/SanityTest.cs
+++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs
@@ -45,6 +45,8 @@ namespace Grpc.Core.Tests
 {
     public class SanityTest
     {
+        // TODO: make sanity test work for CoreCLR as well
+#if !NETSTANDARD1_5
         /// <summary>
         /// Because we depend on a native library, sometimes when things go wrong, the
         /// entire NUnit test process crashes. To be able to track down problems better,
@@ -121,5 +123,6 @@ namespace Grpc.Core.Tests
             }
             return result;
         }
+#endif
     }
 }
diff --git a/src/csharp/Grpc.Core.Tests/ServerTest.cs b/src/csharp/Grpc.Core.Tests/ServerTest.cs
index b40508accc..3b51aa6330 100644
--- a/src/csharp/Grpc.Core.Tests/ServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ServerTest.cs
@@ -89,9 +89,24 @@ namespace Grpc.Core.Tests
             };
             server.Start();
             Assert.Throws(typeof(InvalidOperationException), () => server.Ports.Add("localhost", 9999, ServerCredentials.Insecure));
-            Assert.Throws(typeof(InvalidOperationException), () => server.Services.Add(ServerServiceDefinition.CreateBuilder("serviceName").Build()));
+            Assert.Throws(typeof(InvalidOperationException), () => server.Services.Add(ServerServiceDefinition.CreateBuilder().Build()));
 
             server.ShutdownAsync().Wait();
         }
+
+        [Test]
+        public void UnstartedServerCanBeShutdown()
+        {
+            var server = new Server();
+            server.ShutdownAsync().Wait();
+            Assert.Throws(typeof(InvalidOperationException), () => server.Start());
+        }
+
+        [Test]
+        public void UnstartedServerDoesNotPreventShutdown()
+        {
+            // just create a server, don't start it, and make sure it doesn't prevent shutdown.
+            var server = new Server();
+        }
     }
 }
diff --git a/src/proto/grpc/testing/perf_db.proto b/src/csharp/Grpc.Core.Tests/ShutdownHookClientTest.cs
index 0ba8596fe9..12b8452f64 100644
--- a/src/proto/grpc/testing/perf_db.proto
+++ b/src/csharp/Grpc.Core.Tests/ShutdownHookClientTest.cs
@@ -1,3 +1,5 @@
+#region Copyright notice and license
+
 // Copyright 2015, Google Inc.
 // All rights reserved.
 //
@@ -27,43 +29,29 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-syntax = "proto3";
-
-import "src/proto/grpc/testing/control.proto";
-
-package grpc.testing;
+#endregion
 
-service PerfDbTransfer {
-  // Sends client info
-  rpc RecordSingleClientData(SingleUserRecordRequest)
-      returns (SingleUserRecordReply) {}
-}
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Utils;
+using NUnit.Framework;
 
-// Metrics to be stored
-message Metrics {
-  double qps = 1;
-  double qps_per_core = 2;
-  double perc_lat_50 = 3;
-  double perc_lat_90 = 4;
-  double perc_lat_95 = 5;
-  double perc_lat_99 = 6;
-  double perc_lat_99_point_9 = 7;
-  double server_system_time = 8;
-  double server_user_time = 9;
-  double client_system_time = 10;
-  double client_user_time = 11;
-}
+namespace Grpc.Core.Tests
+{
+    public class ShutdownHookClientTest
+    {
+        const string Host = "127.0.0.1";
 
-// Request for storing a single user's data
-message SingleUserRecordRequest {
-  string hashed_id = 1;
-  string test_name = 2;
-  string sys_info = 3;
-  string tag = 4;
-  Metrics metrics = 5;
-  ClientConfig client_config = 6;
-  ServerConfig server_config = 7;
+        [Test]
+        public void ProcessExitHookCanCleanupAbandonedChannels()
+        {
+            var channel = new Channel(Host, 1000, ChannelCredentials.Insecure);
+            var channel2 = new Channel(Host, 1001, ChannelCredentials.Insecure);
+        }
+    }
 }
-
-// Reply to request for storing single user's data
-message SingleUserRecordReply {}
diff --git a/src/csharp/Grpc.Core.Tests/ShutdownHookPendingCallTest.cs b/src/csharp/Grpc.Core.Tests/ShutdownHookPendingCallTest.cs
new file mode 100644
index 0000000000..175233840d
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/ShutdownHookPendingCallTest.cs
@@ -0,0 +1,69 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class ShutdownHookPendingCallTest
+    {
+        const string Host = "127.0.0.1";
+
+        [Test]
+        public void ProcessExitHookCanCleanupAbandonedCall()
+        {
+            var helper = new MockServiceHelper(Host);
+            var server = helper.GetServer();
+            server.Start();
+            var channel = helper.GetChannel();
+
+            var readyToShutdown = new TaskCompletionSource<object>();
+            helper.DuplexStreamingHandler = new DuplexStreamingServerMethod<string, string>(async (requestStream, responseStream, context) =>
+            {
+                readyToShutdown.SetResult(null);
+                await requestStream.ToListAsync();
+            });
+
+            var call = Calls.AsyncDuplexStreamingCall(helper.CreateDuplexStreamingCall());
+            readyToShutdown.Task.Wait();  // make sure handler is running
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core.Tests/ShutdownHookServerTest.cs b/src/csharp/Grpc.Core.Tests/ShutdownHookServerTest.cs
new file mode 100644
index 0000000000..e7ea7a0bf5
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/ShutdownHookServerTest.cs
@@ -0,0 +1,58 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class ShutdownHookServerTest
+    {
+        const string Host = "127.0.0.1";
+
+        [Test]
+        public void ProcessExitHookCanCleanupAbandonedServers()
+        {
+            var helper = new MockServiceHelper(Host);
+            var server = helper.GetServer();
+            server.Start();
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core.Tests/project.json b/src/csharp/Grpc.Core.Tests/project.json
new file mode 100644
index 0000000000..f58bcbb515
--- /dev/null
+++ b/src/csharp/Grpc.Core.Tests/project.json
@@ -0,0 +1,70 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.Core": {
+      "target": "project"
+    },
+    "Newtonsoft.Json": "8.0.3",
+    "NUnit": "3.2.0",
+    "NUnitLite": "3.2.0-*"
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  },
+}
diff --git a/src/csharp/Grpc.Core/AsyncClientStreamingCall.cs b/src/csharp/Grpc.Core/AsyncClientStreamingCall.cs
index 5646fed3d9..02b08d2a10 100644
--- a/src/csharp/Grpc.Core/AsyncClientStreamingCall.cs
+++ b/src/csharp/Grpc.Core/AsyncClientStreamingCall.cs
@@ -127,6 +127,10 @@ namespace Grpc.Core
         /// Otherwise, requests cancellation of the call which should terminate all pending async operations associated with the call.
         /// As a result, all resources being used by the call should be released eventually.
         /// </summary>
+        /// <remarks>
+        /// Normally, there is no need for you to dispose the call unless you want to utilize the
+        /// "Cancel" semantics of invoking <c>Dispose</c>.
+        /// </remarks>
         public void Dispose()
         {
             disposeAction.Invoke();
diff --git a/src/csharp/Grpc.Core/AsyncDuplexStreamingCall.cs b/src/csharp/Grpc.Core/AsyncDuplexStreamingCall.cs
index e75108c7e5..68fd6d0b05 100644
--- a/src/csharp/Grpc.Core/AsyncDuplexStreamingCall.cs
+++ b/src/csharp/Grpc.Core/AsyncDuplexStreamingCall.cs
@@ -117,6 +117,10 @@ namespace Grpc.Core
         /// Otherwise, requests cancellation of the call which should terminate all pending async operations associated with the call.
         /// As a result, all resources being used by the call should be released eventually.
         /// </summary>
+        /// <remarks>
+        /// Normally, there is no need for you to dispose the call unless you want to utilize the
+        /// "Cancel" semantics of invoking <c>Dispose</c>.
+        /// </remarks>
         public void Dispose()
         {
             disposeAction.Invoke();
diff --git a/src/csharp/Grpc.Core/AsyncServerStreamingCall.cs b/src/csharp/Grpc.Core/AsyncServerStreamingCall.cs
index f953091984..5777c72615 100644
--- a/src/csharp/Grpc.Core/AsyncServerStreamingCall.cs
+++ b/src/csharp/Grpc.Core/AsyncServerStreamingCall.cs
@@ -103,6 +103,10 @@ namespace Grpc.Core
         /// Otherwise, requests cancellation of the call which should terminate all pending async operations associated with the call.
         /// As a result, all resources being used by the call should be released eventually.
         /// </summary>
+        /// <remarks>
+        /// Normally, there is no need for you to dispose the call unless you want to utilize the
+        /// "Cancel" semantics of invoking <c>Dispose</c>.
+        /// </remarks>
         public void Dispose()
         {
             disposeAction.Invoke();
diff --git a/src/csharp/Grpc.Core/AsyncUnaryCall.cs b/src/csharp/Grpc.Core/AsyncUnaryCall.cs
index 97df8f5e91..d180c27922 100644
--- a/src/csharp/Grpc.Core/AsyncUnaryCall.cs
+++ b/src/csharp/Grpc.Core/AsyncUnaryCall.cs
@@ -112,6 +112,10 @@ namespace Grpc.Core
         /// Otherwise, requests cancellation of the call which should terminate all pending async operations associated with the call.
         /// As a result, all resources being used by the call should be released eventually.
         /// </summary>
+        /// <remarks>
+        /// Normally, there is no need for you to dispose the call unless you want to utilize the
+        /// "Cancel" semantics of invoking <c>Dispose</c>.
+        /// </remarks>
         public void Dispose()
         {
             disposeAction.Invoke();
diff --git a/src/csharp/Grpc.Core/CallOptions.cs b/src/csharp/Grpc.Core/CallOptions.cs
index 9ca88849ee..35548cfc96 100644
--- a/src/csharp/Grpc.Core/CallOptions.cs
+++ b/src/csharp/Grpc.Core/CallOptions.cs
@@ -88,7 +88,13 @@ namespace Grpc.Core
         }
 
         /// <summary>
-        /// Token that can be used for cancelling the call.
+        /// Token that can be used for cancelling the call on the client side.
+        /// Cancelling the token will request cancellation
+        /// of the remote call. Best effort will be made to deliver the cancellation
+        /// notification to the server and interaction of the call with the server side
+        /// will be terminated. Unless the call finishes before the cancellation could
+        /// happen (there is an inherent race),
+        /// the call will finish with <c>StatusCode.Cancelled</c> status.
         /// </summary>
         public CancellationToken CancellationToken
         {
diff --git a/src/csharp/Grpc.Core/Channel.cs b/src/csharp/Grpc.Core/Channel.cs
index 93a6e6a3d9..4f29c35b32 100644
--- a/src/csharp/Grpc.Core/Channel.cs
+++ b/src/csharp/Grpc.Core/Channel.cs
@@ -31,7 +31,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -56,6 +55,7 @@ namespace Grpc.Core
 
         readonly string target;
         readonly GrpcEnvironment environment;
+        readonly CompletionQueueSafeHandle completionQueue;
         readonly ChannelSafeHandle handle;
         readonly Dictionary<string, ChannelOption> options;
 
@@ -67,14 +67,26 @@ namespace Grpc.Core
         /// </summary>
         /// <param name="target">Target of the channel.</param>
         /// <param name="credentials">Credentials to secure the channel.</param>
+        public Channel(string target, ChannelCredentials credentials) :
+            this(target, credentials, null)
+        {
+        }
+
+        /// <summary>
+        /// Creates a channel that connects to a specific host.
+        /// Port will default to 80 for an unsecure channel and to 443 for a secure channel.
+        /// </summary>
+        /// <param name="target">Target of the channel.</param>
+        /// <param name="credentials">Credentials to secure the channel.</param>
         /// <param name="options">Channel options.</param>
-        public Channel(string target, ChannelCredentials credentials, IEnumerable<ChannelOption> options = null)
+        public Channel(string target, ChannelCredentials credentials, IEnumerable<ChannelOption> options)
         {
             this.target = GrpcPreconditions.CheckNotNull(target, "target");
             this.options = CreateOptionsDictionary(options);
             EnsureUserAgentChannelOption(this.options);
             this.environment = GrpcEnvironment.AddRef();
 
+            this.completionQueue = this.environment.PickCompletionQueue();
             using (var nativeCredentials = credentials.ToNativeCredentials())
             using (var nativeChannelArgs = ChannelOptions.CreateChannelArgs(this.options.Values))
             {
@@ -87,6 +99,18 @@ namespace Grpc.Core
                     this.handle = ChannelSafeHandle.CreateInsecure(target, nativeChannelArgs);
                 }
             }
+            GrpcEnvironment.RegisterChannel(this);
+        }
+
+        /// <summary>
+        /// Creates a channel that connects to a specific host and port.
+        /// </summary>
+        /// <param name="host">The name or IP address of the host.</param>
+        /// <param name="port">The port.</param>
+        /// <param name="credentials">Credentials to secure the channel.</param>
+        public Channel(string host, int port, ChannelCredentials credentials) :
+            this(host, port, credentials, null)
+        {
         }
 
         /// <summary>
@@ -96,14 +120,14 @@ namespace Grpc.Core
         /// <param name="port">The port.</param>
         /// <param name="credentials">Credentials to secure the channel.</param>
         /// <param name="options">Channel options.</param>
-        public Channel(string host, int port, ChannelCredentials credentials, IEnumerable<ChannelOption> options = null) :
+        public Channel(string host, int port, ChannelCredentials credentials, IEnumerable<ChannelOption> options) :
             this(string.Format("{0}:{1}", host, port), credentials, options)
         {
         }
 
         /// <summary>
         /// Gets current connectivity state of this channel.
-        /// After channel is has been shutdown, <c>ChannelState.FatalFailure</c> will be returned.
+        /// After channel is has been shutdown, <c>ChannelState.Shutdown</c> will be returned.
         /// </summary>
         public ChannelState State
         {
@@ -120,8 +144,8 @@ namespace Grpc.Core
         /// </summary>
         public Task WaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null)
         {
-            GrpcPreconditions.CheckArgument(lastObservedState != ChannelState.FatalFailure,
-                "FatalFailure is a terminal state. No further state changes can occur.");
+            GrpcPreconditions.CheckArgument(lastObservedState != ChannelState.Shutdown,
+                "Shutdown is a terminal state. No further state changes can occur.");
             var tcs = new TaskCompletionSource<object>();
             var deadlineTimespec = deadline.HasValue ? Timespec.FromDateTime(deadline.Value) : Timespec.InfFuture;
             var handler = new BatchCompletionDelegate((success, ctx) =>
@@ -135,7 +159,7 @@ namespace Grpc.Core
                     tcs.SetCanceled();
                 }
             });
-            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, environment.CompletionQueue, environment.CompletionRegistry, handler);
+            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, completionQueue, handler);
             return tcs.Task;
         }
 
@@ -171,7 +195,7 @@ namespace Grpc.Core
         /// <summary>
         /// Allows explicitly requesting channel to connect without starting an RPC.
         /// Returned task completes once state Ready was seen. If the deadline is reached,
-        /// or channel enters the FatalFailure state, the task is cancelled.
+        /// or channel enters the Shutdown state, the task is cancelled.
         /// There is no need to call this explicitly unless your use case requires that.
         /// Starting an RPC on a new channel will request connection implicitly.
         /// </summary>
@@ -181,9 +205,9 @@ namespace Grpc.Core
             var currentState = GetConnectivityState(true);
             while (currentState != ChannelState.Ready)
             {
-                if (currentState == ChannelState.FatalFailure)
+                if (currentState == ChannelState.Shutdown)
                 {
-                    throw new OperationCanceledException("Channel has reached FatalFailure state.");
+                    throw new OperationCanceledException("Channel has reached Shutdown state.");
                 }
                 await WaitForStateChangedAsync(currentState, deadline).ConfigureAwait(false);
                 currentState = GetConnectivityState(false);
@@ -191,9 +215,16 @@ namespace Grpc.Core
         }
 
         /// <summary>
-        /// Waits until there are no more active calls for this channel and then cleans up
-        /// resources used by this channel.
+        /// Shuts down the channel cleanly. It is strongly recommended to shutdown
+        /// all previously created channels before exiting from the process.
         /// </summary>
+        /// <remarks>
+        /// This method doesn't wait for all calls on this channel to finish (nor does
+        /// it explicitly cancel all outstanding calls). It is user's responsibility to make sure
+        /// all the calls on this channel have finished (successfully or with an error)
+        /// before shutting down the channel to ensure channel shutdown won't impact
+        /// the outcome of those remote calls.
+        /// </remarks>
         public async Task ShutdownAsync()
         {
             lock (myLock)
@@ -201,6 +232,7 @@ namespace Grpc.Core
                 GrpcPreconditions.CheckState(!shutdownRequested);
                 shutdownRequested = true;
             }
+            GrpcEnvironment.UnregisterChannel(this);
 
             shutdownTokenSource.Cancel();
 
@@ -212,7 +244,7 @@ namespace Grpc.Core
 
             handle.Dispose();
 
-            await Task.Run(() => GrpcEnvironment.Release()).ConfigureAwait(false);
+            await GrpcEnvironment.ReleaseAsync().ConfigureAwait(false);
         }
 
         internal ChannelSafeHandle Handle
@@ -231,6 +263,14 @@ namespace Grpc.Core
             }
         }
 
+        internal CompletionQueueSafeHandle CompletionQueue
+        {
+            get
+            {
+                return this.completionQueue;
+            }
+        }
+
         internal void AddCallReference(object call)
         {
             activeCallCounter.Increment();
@@ -255,7 +295,7 @@ namespace Grpc.Core
             }
             catch (ObjectDisposedException)
             {
-                return ChannelState.FatalFailure;
+                return ChannelState.Shutdown;
             }
         }
 
diff --git a/src/csharp/Grpc.Core/ChannelState.cs b/src/csharp/Grpc.Core/ChannelState.cs
index d293b98f75..a6c3b2a488 100644
--- a/src/csharp/Grpc.Core/ChannelState.cs
+++ b/src/csharp/Grpc.Core/ChannelState.cs
@@ -64,6 +64,6 @@ namespace Grpc.Core
         /// <summary>
         /// Channel has seen a failure that it cannot recover from
         /// </summary>
-        FatalFailure
+        Shutdown
     }
 }
diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj
index 95077a6ca5..1952ee3712 100644
--- a/src/csharp/Grpc.Core/Grpc.Core.csproj
+++ b/src/csharp/Grpc.Core/Grpc.Core.csproj
@@ -74,7 +74,6 @@
     <Compile Include="Internal\CallSafeHandle.cs" />
     <Compile Include="Internal\ChannelSafeHandle.cs" />
     <Compile Include="Internal\CompletionQueueSafeHandle.cs" />
-    <Compile Include="Internal\Enums.cs" />
     <Compile Include="Internal\SafeHandleZeroIsInvalid.cs" />
     <Compile Include="Internal\Timespec.cs" />
     <Compile Include="Internal\GrpcThreadPool.cs" />
@@ -87,7 +86,6 @@
     <Compile Include="Utils\BenchmarkUtil.cs" />
     <Compile Include="ChannelCredentials.cs" />
     <Compile Include="Internal\ChannelArgsSafeHandle.cs" />
-    <Compile Include="Internal\AsyncCompletion.cs" />
     <Compile Include="Internal\AsyncCallBase.cs" />
     <Compile Include="Internal\AsyncCallServer.cs" />
     <Compile Include="Internal\AsyncCall.cs" />
@@ -134,9 +132,16 @@
     <Compile Include="DefaultCallInvoker.cs" />
     <Compile Include="Internal\UnimplementedCallInvoker.cs" />
     <Compile Include="Internal\InterceptingCallInvoker.cs" />
+    <Compile Include="Internal\ServerRpcNew.cs" />
+    <Compile Include="Internal\ClientSideStatus.cs" />
+    <Compile Include="Internal\ClockType.cs" />
+    <Compile Include="Internal\CallError.cs" />
+    <Compile Include="Logging\LogLevel.cs" />
+    <Compile Include="Logging\LogLevelFilterLogger.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Grpc.Core.nuspec" />
+    <None Include="Grpc.Core.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
   <Import Project="NativeDeps.targets" />
@@ -144,7 +149,7 @@
   <ItemGroup />
   <ItemGroup>
     <EmbeddedResource Include="..\..\..\etc\roots.pem">
-      <Link>Resources\roots.pem</Link>
+      <Link>roots.pem</Link>
     </EmbeddedResource>
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Core/Grpc.Core.nuspec b/src/csharp/Grpc.Core/Grpc.Core.nuspec
index 0ada0049c2..47593f787b 100644
--- a/src/csharp/Grpc.Core/Grpc.Core.nuspec
+++ b/src/csharp/Grpc.Core/Grpc.Core.nuspec
@@ -24,11 +24,12 @@
     <file src="bin/ReleaseSigned/Grpc.Core.xml" target="lib/net45" />
     <file src="**\*.cs" target="src" />
     <file src="Grpc.Core.targets" target="\build\net45\Grpc.Core.targets" />
-    <file src="windows_x86/grpc_csharp_ext.dll" target="/build/native/bin/windows_x86/grpc_csharp_ext.dll" />
-    <file src="windows_x64/grpc_csharp_ext.dll" target="/build/native/bin/windows_x64/grpc_csharp_ext.dll" />
-    <file src="linux_x86/libgrpc_csharp_ext.so" target="/build/native/bin/linux_x86/libgrpc_csharp_ext.so" />
-    <file src="linux_x64/libgrpc_csharp_ext.so" target="/build/native/bin/linux_x64/libgrpc_csharp_ext.so" />
-    <file src="macosx_x86/libgrpc_csharp_ext.dylib" target="/build/native/bin/macosx_x86/libgrpc_csharp_ext.dylib" />
-    <file src="macosx_x64/libgrpc_csharp_ext.dylib" target="/build/native/bin/macosx_x64/libgrpc_csharp_ext.dylib" />
+    <!-- without backslashes in the the source path, nuget won't copy the files -->
+    <file src="..\nativelibs\windows_x86\grpc_csharp_ext.dll" target="/build/native/bin/windows_x86/grpc_csharp_ext.dll" />
+    <file src="..\nativelibs\windows_x64\grpc_csharp_ext.dll" target="/build/native/bin/windows_x64/grpc_csharp_ext.dll" />
+    <file src="..\nativelibs\linux_x86\libgrpc_csharp_ext.so" target="/build/native/bin/linux_x86/libgrpc_csharp_ext.so" />
+    <file src="..\nativelibs\linux_x64\libgrpc_csharp_ext.so" target="/build/native/bin/linux_x64/libgrpc_csharp_ext.so" />
+    <file src="..\nativelibs\macosx_x86\libgrpc_csharp_ext.dylib" target="/build/native/bin/macosx_x86/libgrpc_csharp_ext.dylib" />
+    <file src="..\nativelibs\macosx_x64\libgrpc_csharp_ext.dylib" target="/build/native/bin/macosx_x64/libgrpc_csharp_ext.dylib" />
   </files>
 </package>
diff --git a/src/csharp/Grpc.Core/Grpc.Core.project.json b/src/csharp/Grpc.Core/Grpc.Core.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Core/Grpc.Core.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Core/Grpc.Core.xproj b/src/csharp/Grpc.Core/Grpc.Core.xproj
new file mode 100644
index 0000000000..137236ffdb
--- /dev/null
+++ b/src/csharp/Grpc.Core/Grpc.Core.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>dc9908b6-f291-4fc8-a46d-2ea2551790ec</ProjectGuid>
+    <RootNamespace>Grpc.Core</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs
index a5c78cc9d7..eeed699712 100644
--- a/src/csharp/Grpc.Core/GrpcEnvironment.cs
+++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs
@@ -32,6 +32,8 @@
 #endregion
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using Grpc.Core.Internal;
@@ -45,17 +47,24 @@ namespace Grpc.Core
     /// </summary>
     public class GrpcEnvironment
     {
-        const int THREAD_POOL_SIZE = 4;
+        const LogLevel DefaultLogLevel = LogLevel.Info;
+        const int MinDefaultThreadPoolSize = 4;
 
         static object staticLock = new object();
         static GrpcEnvironment instance;
         static int refCount;
+        static int? customThreadPoolSize;
+        static int? customCompletionQueueCount;
+        static readonly HashSet<Channel> registeredChannels = new HashSet<Channel>();
+        static readonly HashSet<Server> registeredServers = new HashSet<Server>();
 
-        static ILogger logger = new ConsoleLogger();
+        static ILogger logger = new LogLevelFilterLogger(new ConsoleLogger(), DefaultLogLevel);
 
+        readonly object myLock = new object();
         readonly GrpcThreadPool threadPool;
-        readonly CompletionRegistry completionRegistry;
         readonly DebugStats debugStats = new DebugStats();
+        readonly AtomicCounter cqPickerCounter = new AtomicCounter();
+
         bool isClosed;
 
         /// <summary>
@@ -64,6 +73,8 @@ namespace Grpc.Core
         /// </summary>
         internal static GrpcEnvironment AddRef()
         {
+            ShutdownHooks.Register();
+
             lock (staticLock)
             {
                 refCount++;
@@ -76,21 +87,26 @@ namespace Grpc.Core
         }
 
         /// <summary>
-        /// Decrements the reference count for currently active environment and shuts down the gRPC environment if reference count drops to zero.
-        /// (and blocks until the environment has been fully shutdown).
+        /// Decrements the reference count for currently active environment and asynchronously shuts down the gRPC environment if reference count drops to zero.
         /// </summary>
-        internal static void Release()
+        internal static async Task ReleaseAsync()
         {
+            GrpcEnvironment instanceToShutdown = null;
             lock (staticLock)
             {
                 GrpcPreconditions.CheckState(refCount > 0);
                 refCount--;
                 if (refCount == 0)
                 {
-                    instance.Close();
+                    instanceToShutdown = instance;
                     instance = null;
                 }
             }
+
+            if (instanceToShutdown != null)
+            {
+                await instanceToShutdown.ShutdownAsync().ConfigureAwait(false);
+            }
         }
 
         internal static int GetRefCount()
@@ -101,6 +117,68 @@ namespace Grpc.Core
             }
         }
 
+        internal static void RegisterChannel(Channel channel)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckNotNull(channel);
+                registeredChannels.Add(channel);
+            }
+        }
+
+        internal static void UnregisterChannel(Channel channel)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckNotNull(channel);
+                GrpcPreconditions.CheckArgument(registeredChannels.Remove(channel), "Channel not found in the registered channels set.");
+            }
+        }
+
+        internal static void RegisterServer(Server server)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckNotNull(server);
+                registeredServers.Add(server);
+            }
+        }
+
+        internal static void UnregisterServer(Server server)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckNotNull(server);
+                GrpcPreconditions.CheckArgument(registeredServers.Remove(server), "Server not found in the registered servers set.");
+            }
+        }
+
+        /// <summary>
+        /// Requests shutdown of all channels created by the current process.
+        /// </summary>
+        public static Task ShutdownChannelsAsync()
+        {
+            HashSet<Channel> snapshot = null;
+            lock (staticLock)
+            {
+                snapshot = new HashSet<Channel>(registeredChannels);
+            }
+            return Task.WhenAll(snapshot.Select((channel) => channel.ShutdownAsync()));
+        }
+
+        /// <summary>
+        /// Requests immediate shutdown of all servers created by the current process.
+        /// </summary>
+        public static Task KillServersAsync()
+        {
+            HashSet<Server> snapshot = null;
+            lock (staticLock)
+            {
+                snapshot = new HashSet<Server>(registeredServers);
+            }
+            return Task.WhenAll(snapshot.Select((server) => server.KillAsync()));
+        }
+
         /// <summary>
         /// Gets application-wide logger used by gRPC.
         /// </summary>
@@ -123,39 +201,79 @@ namespace Grpc.Core
         }
 
         /// <summary>
+        /// Sets the number of threads in the gRPC thread pool that polls for internal RPC events.
+        /// Can be only invoke before the <c>GrpcEnviroment</c> is started and cannot be changed afterwards.
+        /// Setting thread pool size is an advanced setting and you should only use it if you know what you are doing.
+        /// Most users should rely on the default value provided by gRPC library.
+        /// Note: this method is part of an experimental API that can change or be removed without any prior notice.
+        /// </summary>
+        public static void SetThreadPoolSize(int threadCount)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckState(instance == null, "Can only be set before GrpcEnvironment is initialized");
+                GrpcPreconditions.CheckArgument(threadCount > 0, "threadCount needs to be a positive number");
+                customThreadPoolSize = threadCount;
+            }
+        }
+
+        /// <summary>
+        /// Sets the number of completion queues in the  gRPC thread pool that polls for internal RPC events.
+        /// Can be only invoke before the <c>GrpcEnviroment</c> is started and cannot be changed afterwards.
+        /// Setting the number of completions queues is an advanced setting and you should only use it if you know what you are doing.
+        /// Most users should rely on the default value provided by gRPC library.
+        /// Note: this method is part of an experimental API that can change or be removed without any prior notice.
+        /// </summary>
+        public static void SetCompletionQueueCount(int completionQueueCount)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckState(instance == null, "Can only be set before GrpcEnvironment is initialized");
+                GrpcPreconditions.CheckArgument(completionQueueCount > 0, "threadCount needs to be a positive number");
+                customCompletionQueueCount = completionQueueCount;
+            }
+        }
+
+        /// <summary>
         /// Creates gRPC environment.
         /// </summary>
         private GrpcEnvironment()
         {
             GrpcNativeInit();
-            completionRegistry = new CompletionRegistry(this);
-            threadPool = new GrpcThreadPool(this, THREAD_POOL_SIZE);
+            threadPool = new GrpcThreadPool(this, GetThreadPoolSizeOrDefault(), GetCompletionQueueCountOrDefault());
             threadPool.Start();
         }
 
         /// <summary>
-        /// Gets the completion registry used by this gRPC environment.
+        /// Gets the completion queues used by this gRPC environment.
         /// </summary>
-        internal CompletionRegistry CompletionRegistry
+        internal IReadOnlyCollection<CompletionQueueSafeHandle> CompletionQueues
         {
             get
             {
-                return this.completionRegistry;
+                return this.threadPool.CompletionQueues;
             }
         }
 
-        /// <summary>
-        /// Gets the completion queue used by this gRPC environment.
-        /// </summary>
-        internal CompletionQueueSafeHandle CompletionQueue
+        internal bool IsAlive
         {
             get
             {
-                return this.threadPool.CompletionQueue;
+                return this.threadPool.IsAlive;
             }
         }
 
         /// <summary>
+        /// Picks a completion queue in a round-robin fashion.
+        /// Shouldn't be invoked on a per-call basis (used at per-channel basis).
+        /// </summary>
+        internal CompletionQueueSafeHandle PickCompletionQueue()
+        {
+            var cqIndex = (int) ((cqPickerCounter.Increment() - 1) % this.threadPool.CompletionQueues.Count);
+            return this.threadPool.CompletionQueues.ElementAt(cqIndex);
+        }
+
+        /// <summary>
         /// Gets the completion queue used by this gRPC environment.
         /// </summary>
         internal DebugStats DebugStats
@@ -188,17 +306,70 @@ namespace Grpc.Core
         /// <summary>
         /// Shuts down this environment.
         /// </summary>
-        private void Close()
+        private async Task ShutdownAsync()
         {
             if (isClosed)
             {
                 throw new InvalidOperationException("Close has already been called");
             }
-            threadPool.Stop();
+            await threadPool.StopAsync().ConfigureAwait(false);
             GrpcNativeShutdown();
             isClosed = true;
 
             debugStats.CheckOK();
         }
+
+        private int GetThreadPoolSizeOrDefault()
+        {
+            if (customThreadPoolSize.HasValue)
+            {
+                return customThreadPoolSize.Value;
+            }
+            // In systems with many cores, use half of the cores for GrpcThreadPool
+            // and the other half for .NET thread pool. This heuristic definitely needs
+            // more work, but seems to work reasonably well for a start.
+            return Math.Max(MinDefaultThreadPoolSize, Environment.ProcessorCount / 2);
+        }
+
+        private int GetCompletionQueueCountOrDefault()
+        {
+            if (customCompletionQueueCount.HasValue)
+            {
+                return customCompletionQueueCount.Value;
+            }
+            // by default, create a completion queue for each thread
+            return GetThreadPoolSizeOrDefault();
+        }
+
+        private static class ShutdownHooks
+        {
+            static object staticLock = new object();
+            static bool hooksRegistered;
+
+            public static void Register()
+            {
+                lock (staticLock)
+                {
+                    if (!hooksRegistered)
+                    {
+                        // TODO(jtattermusch): register shutdownhooks for CoreCLR as well
+#if !NETSTANDARD1_5
+
+                        AppDomain.CurrentDomain.ProcessExit += ShutdownHookHandler;
+                        AppDomain.CurrentDomain.DomainUnload += ShutdownHookHandler;
+#endif
+                    }
+                    hooksRegistered = true;
+                }
+            }
+
+            /// <summary>
+            /// Handler for AppDomain.DomainUnload and AppDomain.ProcessExit hooks.
+            /// </summary>
+            private static void ShutdownHookHandler(object sender, EventArgs e)
+            {
+                Task.WaitAll(GrpcEnvironment.ShutdownChannelsAsync(), GrpcEnvironment.KillServersAsync());
+            }
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/IAsyncStreamReader.cs b/src/csharp/Grpc.Core/IAsyncStreamReader.cs
index 49e1ea7832..aa3b802a50 100644
--- a/src/csharp/Grpc.Core/IAsyncStreamReader.cs
+++ b/src/csharp/Grpc.Core/IAsyncStreamReader.cs
@@ -41,10 +41,24 @@ namespace Grpc.Core
 {
     /// <summary>
     /// A stream of messages to be read.
+    /// Messages can be awaited <c>await reader.MoveNext()</c>, that returns <c>true</c>
+    /// if there is a message available and <c>false</c> if there are no more messages
+    /// (i.e. the stream has been closed).
+    /// <para>
+    /// On the client side, the last invocation of <c>MoveNext()</c> either returns <c>false</c>
+    /// if the call has finished successfully or throws <c>RpcException</c> if call finished
+    /// with an error. Once the call finishes, subsequent invocations of <c>MoveNext()</c> will
+    /// continue yielding the same result (returning <c>false</c> or throwing an exception).
+    /// </para>
+    /// <para>
+    /// On the server side, <c>MoveNext()</c> does not throw exceptions.
+    /// In case of a failure, the request stream will appear to be finished
+    /// (<c>MoveNext</c> will return <c>false</c>) and the <c>CancellationToken</c>
+    /// associated with the call will be cancelled to signal the failure.
+    /// </para>
     /// </summary>
     /// <typeparam name="T">The message type.</typeparam>
     public interface IAsyncStreamReader<T> : IAsyncEnumerator<T>
     {
-        // TODO(jtattermusch): consider just using IAsyncEnumerator instead of this interface.
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs
index 55351869b5..f549c52876 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs
@@ -32,12 +32,7 @@
 #endregion
 
 using System;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
 using System.Threading.Tasks;
-using Grpc.Core.Internal;
 using Grpc.Core.Logging;
 using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
@@ -57,9 +52,11 @@ namespace Grpc.Core.Internal
         // Completion of a pending unary response if not null.
         TaskCompletionSource<TResponse> unaryResponseTcs;
 
+        // TODO(jtattermusch): this field doesn't need to be initialized for unary response calls.
         // Indicates that response streaming call has finished.
         TaskCompletionSource<object> streamingCallFinishedTcs = new TaskCompletionSource<object>();
 
+        // TODO(jtattermusch): this field could be lazy-initialized (only if someone requests the response headers).
         // Response headers set here once received.
         TaskCompletionSource<Metadata> responseHeadersTcs = new TaskCompletionSource<Metadata>();
 
@@ -67,7 +64,7 @@ namespace Grpc.Core.Internal
         ClientSideStatus? finishedStatus;
 
         public AsyncCall(CallInvocationDetails<TRequest, TResponse> callDetails)
-            : base(callDetails.RequestMarshaller.Serializer, callDetails.ResponseMarshaller.Deserializer, callDetails.Channel.Environment)
+            : base(callDetails.RequestMarshaller.Serializer, callDetails.ResponseMarshaller.Deserializer)
         {
             this.details = callDetails.WithOptions(callDetails.Options.Normalize());
             this.initialMetadataSent = true;  // we always send metadata at the very beginning of the call.
@@ -144,7 +141,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 halfcloseRequested = true;
                 readingDone = true;
@@ -171,7 +168,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 readingDone = true;
 
@@ -195,7 +192,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 halfcloseRequested = true;
 
@@ -220,7 +217,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
@@ -232,11 +229,10 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Sends a streaming request. Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendMessage(TRequest msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendMessageAsync(TRequest msg, WriteFlags writeFlags)
         {
-            StartSendMessageInternal(msg, writeFlags, completionDelegate);
+            return SendMessageInternalAsync(msg, writeFlags);
         }
 
         /// <summary>
@@ -250,29 +246,32 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Sends halfclose, indicating client is done with streaming requests.
         /// Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendCloseFromClient(AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendCloseFromClientAsync()
         {
             lock (myLock)
             {
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
-                CheckSendingAllowed(allowFinished: true);
+                GrpcPreconditions.CheckState(started);
 
-                if (!disposed && !finished)
+                var earlyResult = CheckSendPreconditionsClientSide();
+                if (earlyResult != null)
                 {
-                    call.StartSendCloseFromClient(HandleSendCloseFromClientFinished);
+                    return earlyResult;
                 }
-                else
+
+                if (disposed || finished)
                 {
                     // In case the call has already been finished by the serverside,
-                    // the halfclose has already been done implicitly, so we only
-                    // emit the notification for the completion delegate.
-                    Task.Run(() => HandleSendCloseFromClientFinished(true));
+                    // the halfclose has already been done implicitly, so just return
+                    // completed task here.
+                    halfcloseRequested = true;
+                    return Task.FromResult<object>(null);
                 }
+                call.StartSendCloseFromClient(HandleSendFinished);
 
                 halfcloseRequested = true;
-                sendCompletionDelegate = completionDelegate;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
@@ -342,6 +341,45 @@ namespace Grpc.Core.Internal
             get { return true; }
         }
 
+        protected override Task CheckSendAllowedOrEarlyResult()
+        {
+            var earlyResult = CheckSendPreconditionsClientSide();
+            if (earlyResult != null)
+            {
+                return earlyResult;
+            }
+
+            if (finishedStatus.HasValue)
+            {
+                // throwing RpcException if we already received status on client
+                // side makes the most sense.
+                // Note that this throws even for StatusCode.OK.
+                // Writing after the call has finished is not a programming error because server can close
+                // the call anytime, so don't throw directly, but let the write task finish with an error.
+                var tcs = new TaskCompletionSource<object>();
+                tcs.SetException(new RpcException(finishedStatus.Value.Status));
+                return tcs.Task;
+            }
+
+            return null;
+        }
+
+        private Task CheckSendPreconditionsClientSide()
+        {
+            GrpcPreconditions.CheckState(!halfcloseRequested, "Request stream has already been completed.");
+            GrpcPreconditions.CheckState(streamingWriteTcs == null, "Only one write can be pending at a time.");
+
+            if (cancelRequested)
+            {
+                // Return a cancelled task.
+                var tcs = new TaskCompletionSource<object>();
+                tcs.SetCanceled();
+                return tcs.Task;
+            }
+
+            return null;
+        }
+
         private void Initialize(CompletionQueueSafeHandle cq)
         {
             using (Profilers.ForCurrentThread().NewScope("AsyncCall.Initialize"))
@@ -368,7 +406,7 @@ namespace Grpc.Core.Internal
                 var credentials = details.Options.Credentials;
                 using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null)
                 {
-                    var result = details.Channel.Handle.CreateCall(environment.CompletionRegistry,
+                    var result = details.Channel.Handle.CreateCall(
                                  parentCall, ContextPropagationToken.DefaultMask, cq,
                                  details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials);
                     return result;
@@ -400,6 +438,7 @@ namespace Grpc.Core.Internal
         /// </summary>
         private void HandleReceivedResponseHeaders(bool success, Metadata responseHeaders)
         {
+            // TODO(jtattermusch): handle success==false
             responseHeadersTcs.SetResult(responseHeaders);
         }
 
@@ -443,19 +482,6 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected override void CheckSendingAllowed(bool allowFinished)
-        {
-            base.CheckSendingAllowed(true);
-
-            // throwing RpcException if we already received status on client
-            // side makes the most sense.
-            // Note that this throws even for StatusCode.OK.
-            if (!allowFinished && finishedStatus.HasValue)
-            {
-                throw new RpcException(finishedStatus.Value.Status);
-            }
-        }
-
         /// <summary>
         /// Handles receive status completion for calls with streaming response.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
index 4de23706b2..eb9c3ea62d 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
@@ -58,7 +58,6 @@ namespace Grpc.Core.Internal
         readonly Func<TWrite, byte[]> serializer;
         readonly Func<byte[], TRead> deserializer;
 
-        protected readonly GrpcEnvironment environment;
         protected readonly object myLock = new object();
 
         protected INativeCall call;
@@ -67,8 +66,8 @@ namespace Grpc.Core.Internal
         protected bool started;
         protected bool cancelRequested;
 
-        protected AsyncCompletionDelegate<object> sendCompletionDelegate;  // Completion of a pending send or sendclose if not null.
         protected TaskCompletionSource<TRead> streamingReadTcs;  // Completion of a pending streaming read if not null.
+        protected TaskCompletionSource<object> streamingWriteTcs;  // Completion of a pending streaming write or send close from client if not null.
         protected TaskCompletionSource<object> sendStatusFromServerTcs;
 
         protected bool readingDone;  // True if last read (i.e. read with null payload) was already received.
@@ -78,11 +77,10 @@ namespace Grpc.Core.Internal
         protected bool initialMetadataSent;
         protected long streamingWritesCounter;  // Number of streaming send operations started so far.
 
-        public AsyncCallBase(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer, GrpcEnvironment environment)
+        public AsyncCallBase(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer)
         {
             this.serializer = GrpcPreconditions.CheckNotNull(serializer);
             this.deserializer = GrpcPreconditions.CheckNotNull(deserializer);
-            this.environment = GrpcPreconditions.CheckNotNull(environment);
         }
 
         /// <summary>
@@ -128,28 +126,31 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Initiates sending a message. Only one send operation can be active at a time.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
-        protected void StartSendMessageInternal(TWrite msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        protected Task SendMessageInternalAsync(TWrite msg, WriteFlags writeFlags)
         {
             byte[] payload = UnsafeSerialize(msg);
 
             lock (myLock)
             {
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
-                CheckSendingAllowed(allowFinished: false);
+                GrpcPreconditions.CheckState(started);
+                var earlyResult = CheckSendAllowedOrEarlyResult();
+                if (earlyResult != null)
+                {
+                    return earlyResult;
+                }
 
                 call.StartSendMessage(HandleSendFinished, payload, writeFlags, !initialMetadataSent);
 
-                sendCompletionDelegate = completionDelegate;
                 initialMetadataSent = true;
                 streamingWritesCounter++;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
         /// <summary>
         /// Initiates reading a message. Only one read operation can be active at a time.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
         protected Task<TRead> ReadMessageInternalAsync()
         {
@@ -159,7 +160,7 @@ namespace Grpc.Core.Internal
                 if (readingDone)
                 {
                     // the last read that returns null or throws an exception is idempotent
-                    // and maintain its state.
+                    // and maintains its state.
                     GrpcPreconditions.CheckState(streamingReadTcs != null, "Call does not support streaming reads.");
                     return streamingReadTcs.Task;
                 }
@@ -183,7 +184,7 @@ namespace Grpc.Core.Internal
             {
                 if (!disposed && call != null)
                 {
-                    bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished);
+                    bool noMoreSendCompletions = streamingWriteTcs == null && (halfcloseRequested || cancelRequested || finished);
                     if (noMoreSendCompletions && readingDone && finished)
                     {
                         ReleaseResources();
@@ -213,24 +214,11 @@ namespace Grpc.Core.Internal
         {
         }
 
-        protected virtual void CheckSendingAllowed(bool allowFinished)
-        {
-            GrpcPreconditions.CheckState(started);
-            CheckNotCancelled();
-            GrpcPreconditions.CheckState(!disposed || allowFinished);
-
-            GrpcPreconditions.CheckState(!halfcloseRequested, "Already halfclosed.");
-            GrpcPreconditions.CheckState(!finished || allowFinished, "Already finished.");
-            GrpcPreconditions.CheckState(sendCompletionDelegate == null, "Only one write can be pending at a time");
-        }
-
-        protected void CheckNotCancelled()
-        {
-            if (cancelRequested)
-            {
-                throw new OperationCanceledException("Remote call has been cancelled.");
-            }
-        }
+        /// <summary>
+        /// Checks if sending is allowed and possibly returns a Task that allows short-circuiting the send
+        /// logic by directly returning the write operation result task. Normally, null is returned.
+        /// </summary>
+        protected abstract Task CheckSendAllowedOrEarlyResult();
 
         protected byte[] UnsafeSerialize(TWrite msg)
         {
@@ -259,63 +247,27 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected void FireCompletion<T>(AsyncCompletionDelegate<T> completionDelegate, T value, Exception error)
-        {
-            try
-            {
-                completionDelegate(value, error);
-            }
-            catch (Exception e)
-            {
-                Logger.Error(e, "Exception occured while invoking completion delegate.");
-            }
-        }
-
         /// <summary>
-        /// Handles send completion.
+        /// Handles send completion (including SendCloseFromClient).
         /// </summary>
         protected void HandleSendFinished(bool success)
         {
-            AsyncCompletionDelegate<object> origCompletionDelegate = null;
-            lock (myLock)
-            {
-                origCompletionDelegate = sendCompletionDelegate;
-                sendCompletionDelegate = null;
-
-                ReleaseResourcesIfPossible();
-            }
-
-            if (!success)
-            {
-                FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Send failed"));
-            }
-            else
-            {
-                FireCompletion(origCompletionDelegate, null, null);
-            }
-        }
-
-        /// <summary>
-        /// Handles halfclose (send close from client) completion.
-        /// </summary>
-        protected void HandleSendCloseFromClientFinished(bool success)
-        {
-            AsyncCompletionDelegate<object> origCompletionDelegate = null;
+            TaskCompletionSource<object> origTcs = null;
             lock (myLock)
             {
-                origCompletionDelegate = sendCompletionDelegate;
-                sendCompletionDelegate = null;
+                origTcs = streamingWriteTcs;
+                streamingWriteTcs = null;
 
                 ReleaseResourcesIfPossible();
             }
 
             if (!success)
             {
-                FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Sending close from client has failed."));
+                origTcs.SetException(new InvalidOperationException("Send failed"));
             }
             else
             {
-                FireCompletion(origCompletionDelegate, null, null);
+                origTcs.SetResult(null);
             }
         }
 
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
index b1566b44a7..56c23ba3ef 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
@@ -51,14 +51,14 @@ namespace Grpc.Core.Internal
         readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
         readonly Server server;
 
-        public AsyncCallServer(Func<TResponse, byte[]> serializer, Func<byte[], TRequest> deserializer, GrpcEnvironment environment, Server server) : base(serializer, deserializer, environment)
+        public AsyncCallServer(Func<TResponse, byte[]> serializer, Func<byte[], TRequest> deserializer, Server server) : base(serializer, deserializer)
         {
             this.server = GrpcPreconditions.CheckNotNull(server);
         }
 
-        public void Initialize(CallSafeHandle call)
+        public void Initialize(CallSafeHandle call, CompletionQueueSafeHandle completionQueue)
         {
-            call.Initialize(environment.CompletionRegistry, environment.CompletionQueue);
+            call.Initialize(completionQueue);
 
             server.AddCallReference(this);
             InitializeInternal(call);
@@ -91,11 +91,10 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Sends a streaming response. Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendMessage(TResponse msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendMessageAsync(TResponse msg, WriteFlags writeFlags)
         {
-            StartSendMessageInternal(msg, writeFlags, completionDelegate);
+            return SendMessageInternalAsync(msg, writeFlags);
         }
 
         /// <summary>
@@ -110,20 +109,22 @@ namespace Grpc.Core.Internal
         /// Initiates sending a initial metadata. 
         /// Even though C-core allows sending metadata in parallel to sending messages, we will treat sending metadata as a send message operation
         /// to make things simpler.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
-        public void StartSendInitialMetadata(Metadata headers, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendInitialMetadataAsync(Metadata headers)
         {
             lock (myLock)
             {
                 GrpcPreconditions.CheckNotNull(headers, "metadata");
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
 
+                GrpcPreconditions.CheckState(started);
                 GrpcPreconditions.CheckState(!initialMetadataSent, "Response headers can only be sent once per call.");
                 GrpcPreconditions.CheckState(streamingWritesCounter == 0, "Response headers can only be sent before the first write starts.");
-                CheckSendingAllowed(allowFinished: false);
 
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
+                var earlyResult = CheckSendAllowedOrEarlyResult();
+                if (earlyResult != null)
+                {
+                    return earlyResult;
+                }
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(headers))
                 {
@@ -131,7 +132,8 @@ namespace Grpc.Core.Internal
                 }
 
                 this.initialMetadataSent = true;
-                sendCompletionDelegate = completionDelegate;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
@@ -196,6 +198,16 @@ namespace Grpc.Core.Internal
             server.RemoveCallReference(this);
         }
 
+        protected override Task CheckSendAllowedOrEarlyResult()
+        {
+            GrpcPreconditions.CheckState(!halfcloseRequested, "Response stream has already been completed.");
+            GrpcPreconditions.CheckState(!finished, "Already finished.");
+            GrpcPreconditions.CheckState(streamingWriteTcs == null, "Only one write can be pending at a time");
+            GrpcPreconditions.CheckState(!disposed);
+
+            return null;
+        }
+
         /// <summary>
         /// Handles the server side close completion.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs b/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
index 66d2a66f99..c28a6f64d3 100644
--- a/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
@@ -120,107 +120,4 @@ namespace Grpc.Core.Internal
             return true;
         }
     }
-
-    /// <summary>
-    /// Status + metadata received on client side when call finishes.
-    /// (when receive_status_on_client operation finishes).
-    /// </summary>
-    internal struct ClientSideStatus
-    {
-        readonly Status status;
-        readonly Metadata trailers;
-
-        public ClientSideStatus(Status status, Metadata trailers)
-        {
-            this.status = status;
-            this.trailers = trailers;
-        }
-
-        public Status Status
-        {
-            get
-            {
-                return this.status;
-            }    
-        }
-
-        public Metadata Trailers
-        {
-            get
-            {
-                return this.trailers;
-            }
-        }
-    }
-
-    /// <summary>
-    /// Details of a newly received RPC.
-    /// </summary>
-    internal struct ServerRpcNew
-    {
-        readonly Server server;
-        readonly CallSafeHandle call;
-        readonly string method;
-        readonly string host;
-        readonly Timespec deadline;
-        readonly Metadata requestMetadata;
-
-        public ServerRpcNew(Server server, CallSafeHandle call, string method, string host, Timespec deadline, Metadata requestMetadata)
-        {
-            this.server = server;
-            this.call = call;
-            this.method = method;
-            this.host = host;
-            this.deadline = deadline;
-            this.requestMetadata = requestMetadata;
-        }
-
-        public Server Server
-        {
-            get
-            {
-                return this.server;
-            }
-        }
-
-        public CallSafeHandle Call
-        {
-            get
-            {
-                return this.call;
-            }
-        }
-
-        public string Method
-        {
-            get
-            {
-                return this.method;
-            }
-        }
-
-        public string Host
-        {
-            get
-            {
-                return this.host;
-            }
-        }
-
-        public Timespec Deadline
-        {
-            get
-            {
-                return this.deadline;
-            }
-        }
-
-        public Metadata RequestMetadata
-        {
-            get
-            {
-                return this.requestMetadata;
-            }
-        }
-    }
 }
diff --git a/src/csharp/Grpc.Core/Internal/Enums.cs b/src/csharp/Grpc.Core/Internal/CallError.cs
index 74f86d2a30..541575f5e6 100644
--- a/src/csharp/Grpc.Core/Internal/Enums.cs
+++ b/src/csharp/Grpc.Core/Internal/CallError.cs
@@ -40,7 +40,7 @@ namespace Grpc.Core.Internal
     /// <summary>
     /// grpc_call_error from grpc/grpc.h
     /// </summary>
-    internal enum GRPCCallError
+    internal enum CallError
     {
         /* everything went ok */
         OK = 0,
@@ -70,42 +70,9 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Checks the call API invocation's result is OK.
         /// </summary>
-        public static void CheckOk(this GRPCCallError callError)
+        public static void CheckOk(this CallError callError)
         {
-            GrpcPreconditions.CheckState(callError == GRPCCallError.OK, "Call error: " + callError);
+            GrpcPreconditions.CheckState(callError == CallError.OK, "Call error: " + callError);
         }
     }
-
-    /// <summary>
-    /// grpc_completion_type from grpc/grpc.h
-    /// </summary>
-    internal enum GRPCCompletionType
-    {
-        /* Shutting down */
-        Shutdown, 
-
-        /* No event before timeout */
-        Timeout,  
-
-        /* operation completion */
-        OpComplete
-    }
-
-    /// <summary>
-    /// gpr_clock_type from grpc/support/time.h
-    /// </summary>
-    internal enum GPRClockType
-    {
-        /* Monotonic clock */
-        Monotonic,
-
-        /* Realtime clock */
-        Realtime,
-
-        /* Precise clock good for performance profiling. */
-        Precise,
-
-        /* Timespan - the distance between two time points */
-        Timespan
-    }
 }
diff --git a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
index 244b97d4a4..82361f5797 100644
--- a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
@@ -47,16 +47,14 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
 
         const uint GRPC_WRITE_BUFFER_HINT = 1;
-        CompletionRegistry completionRegistry;
         CompletionQueueSafeHandle completionQueue;
 
         private CallSafeHandle()
         {
         }
 
-        public void Initialize(CompletionRegistry completionRegistry, CompletionQueueSafeHandle completionQueue)
+        public void Initialize(CompletionQueueSafeHandle completionQueue)
         {
-            this.completionRegistry = completionRegistry;
             this.completionQueue = completionQueue;
         }
 
@@ -70,7 +68,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
                     .CheckOk();
             }
@@ -90,7 +88,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_start_client_streaming(this, ctx, metadataArray).CheckOk();
             }
         }
@@ -100,7 +98,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
                 Native.grpcsharp_call_start_server_streaming(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags).CheckOk();
             }
         }
@@ -110,7 +108,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
                 Native.grpcsharp_call_start_duplex_streaming(this, ctx, metadataArray).CheckOk();
             }
         }
@@ -120,7 +118,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_message(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, sendEmptyInitialMetadata).CheckOk();
             }
         }
@@ -130,7 +128,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_close_from_client(this, ctx).CheckOk();
             }
         }
@@ -142,7 +140,7 @@ namespace Grpc.Core.Internal
             {
                 var ctx = BatchContextSafeHandle.Create();
                 var optionalPayloadLength = optionalPayload != null ? new UIntPtr((ulong)optionalPayload.Length) : UIntPtr.Zero;
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_status_from_server(this, ctx, status.StatusCode, status.Detail, metadataArray, sendEmptyInitialMetadata,
                     optionalPayload, optionalPayloadLength, writeFlags).CheckOk();
             }
@@ -153,7 +151,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
                 Native.grpcsharp_call_recv_message(this, ctx).CheckOk();
             }
         }
@@ -163,7 +161,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_recv_initial_metadata(this, ctx).CheckOk();
             }
         }
@@ -173,7 +171,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
                 Native.grpcsharp_call_start_serverside(this, ctx).CheckOk();
             }
         }
@@ -183,7 +181,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_initial_metadata(this, ctx, metadataArray).CheckOk();
             }
         }
diff --git a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
index 1dbd1f4e34..62864dff0c 100644
--- a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
@@ -63,7 +63,7 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_secure_channel_create(credentials, target, channelArgs);
         }
 
-        public CallSafeHandle CreateCall(CompletionRegistry registry, CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CallCredentialsSafeHandle credentials)
+        public CallSafeHandle CreateCall(CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CallCredentialsSafeHandle credentials)
         {
             using (Profilers.ForCurrentThread().NewScope("ChannelSafeHandle.CreateCall"))
             {
@@ -72,7 +72,7 @@ namespace Grpc.Core.Internal
                 {
                     result.SetCredentials(credentials);
                 }
-                result.Initialize(registry, cq);
+                result.Initialize(cq);
                 return result;
             }
         }
@@ -82,11 +82,10 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_channel_check_connectivity_state(this, tryToConnect ? 1 : 0);
         }
 
-        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq,
-            CompletionRegistry completionRegistry, BatchCompletionDelegate callback)
+        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq, BatchCompletionDelegate callback)
         {
             var ctx = BatchContextSafeHandle.Create();
-            completionRegistry.RegisterBatchCompletion(ctx, callback);
+            cq.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
             Native.grpcsharp_channel_watch_connectivity_state(this, lastObservedState, deadline, cq, ctx);
         }
 
diff --git a/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs b/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
index 013f00ff6f..924de028f5 100644
--- a/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
+++ b/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
@@ -50,16 +50,12 @@ namespace Grpc.Core.Internal
 
         public Task WriteAsync(TRequest message)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendMessage(message, GetWriteFlags(), taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendMessageAsync(message, GetWriteFlags());
         }
 
         public Task CompleteAsync()
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendCloseFromClient(taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendCloseFromClientAsync();
         }
 
         public WriteOptions WriteOptions
diff --git a/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs b/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs
new file mode 100644
index 0000000000..5727e3f11f
--- /dev/null
+++ b/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs
@@ -0,0 +1,70 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+// 
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using Grpc.Core;
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// Status + metadata received on client side when call finishes.
+    /// (when receive_status_on_client operation finishes).
+    /// </summary>
+    internal struct ClientSideStatus
+    {
+        readonly Status status;
+        readonly Metadata trailers;
+
+        public ClientSideStatus(Status status, Metadata trailers)
+        {
+            this.status = status;
+            this.trailers = trailers;
+        }
+
+        public Status Status
+        {
+            get
+            {
+                return this.status;
+            }
+        }
+
+        public Metadata Trailers
+        {
+            get
+            {
+                return this.trailers;
+            }
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core/Internal/ClockType.cs b/src/csharp/Grpc.Core/Internal/ClockType.cs
new file mode 100644
index 0000000000..57533c9d2f
--- /dev/null
+++ b/src/csharp/Grpc.Core/Internal/ClockType.cs
@@ -0,0 +1,53 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// gpr_clock_type from grpc/support/time.h
+    /// </summary>
+    internal enum ClockType
+    {
+        /* Monotonic clock */
+        Monotonic,
+
+        /* Realtime clock */
+        Realtime,
+
+        /* Precise clock good for performance profiling. */
+        Precise,
+
+        /* Timespan - the distance between two time points */
+        Timespan
+    }
+}
diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
index 288680792a..a78e9b70f3 100644
--- a/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
+++ b/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
@@ -44,7 +44,7 @@ namespace Grpc.Core.Internal
     {
         static readonly NativeMethods Native = NativeMethods.Get();
 
-        public GRPCCompletionType type;
+        public CompletionType type;
         public int success;
         public IntPtr tag;
 
@@ -55,5 +55,20 @@ namespace Grpc.Core.Internal
                 return Native.grpcsharp_sizeof_grpc_event();
             }
         }
+
+        /// <summary>
+        /// grpc_completion_type from grpc/grpc.h
+        /// </summary>
+        internal enum CompletionType
+        {
+            /* Shutting down */
+            Shutdown, 
+
+            /* No event before timeout */
+            Timeout,  
+
+            /* operation completion */
+            OpComplete
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
index 91364cdc70..46f5624223 100644
--- a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
@@ -45,6 +45,7 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
 
         AtomicCounter shutdownRefcount = new AtomicCounter(1);
+        CompletionRegistry completionRegistry;
 
         private CompletionQueueSafeHandle()
         {
@@ -53,7 +54,13 @@ namespace Grpc.Core.Internal
         public static CompletionQueueSafeHandle Create()
         {
             return Native.grpcsharp_completion_queue_create();
+        }
 
+        public static CompletionQueueSafeHandle Create(CompletionRegistry completionRegistry)
+        {
+            var cq = Native.grpcsharp_completion_queue_create();
+            cq.completionRegistry = completionRegistry;
+            return cq;
         }
 
         public CompletionQueueEvent Next()
@@ -83,6 +90,15 @@ namespace Grpc.Core.Internal
             DecrementShutdownRefcount();
         }
 
+        /// <summary>
+        /// Completion registry associated with this completion queue.
+        /// Doesn't need to be set if only using Pluck() operations.
+        /// </summary>
+        public CompletionRegistry CompletionRegistry
+        {
+            get { return completionRegistry; }
+        }
+
         protected override bool ReleaseHandle()
         {
             Native.grpcsharp_completion_queue_destroy(handle);
diff --git a/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs b/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs
index aa4dafd7f2..2a96e9920c 100644
--- a/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs
+++ b/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs
@@ -46,7 +46,7 @@ namespace Grpc.Core.Internal
     /// </summary>
     internal static class DefaultSslRootsOverride
     {
-        const string RootsPemResourceName = "Grpc.Core.Resources.roots.pem";
+        const string RootsPemResourceName = "Grpc.Core.roots.pem";
         static object staticLock = new object();
 
         /// <summary>
diff --git a/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs b/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
index 4b7124ee74..a446c1f99f 100644
--- a/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
+++ b/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
@@ -33,15 +33,16 @@
 
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Grpc.Core.Logging;
+using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
-    /// Pool of threads polling on the same completion queue.
+    /// Pool of threads polling on a set of completions queues.
     /// </summary>
     internal class GrpcThreadPool
     {
@@ -51,25 +52,33 @@ namespace Grpc.Core.Internal
         readonly object myLock = new object();
         readonly List<Thread> threads = new List<Thread>();
         readonly int poolSize;
+        readonly int completionQueueCount;
 
-        CompletionQueueSafeHandle cq;
+        bool stopRequested;
 
-        public GrpcThreadPool(GrpcEnvironment environment, int poolSize)
+        IReadOnlyCollection<CompletionQueueSafeHandle> completionQueues;
+
+        /// <summary>
+        /// Creates a thread pool threads polling on a set of completions queues.
+        /// </summary>
+        /// <param name="environment">Environment.</param>
+        /// <param name="poolSize">Pool size.</param>
+        /// <param name="completionQueueCount">Completion queue count.</param>
+        public GrpcThreadPool(GrpcEnvironment environment, int poolSize, int completionQueueCount)
         {
             this.environment = environment;
             this.poolSize = poolSize;
+            this.completionQueueCount = completionQueueCount;
+            GrpcPreconditions.CheckArgument(poolSize >= completionQueueCount,
+                "Thread pool size cannot be smaller than the number of completion queues used.");
         }
 
         public void Start()
         {
             lock (myLock)
             {
-                if (cq != null)
-                {
-                    throw new InvalidOperationException("Already started.");
-                }
-
-                cq = CompletionQueueSafeHandle.Create();
+                GrpcPreconditions.CheckState(completionQueues == null, "Already started.");
+                completionQueues = CreateCompletionQueueList(environment, completionQueueCount);
 
                 for (int i = 0; i < poolSize; i++)
                 {
@@ -78,53 +87,85 @@ namespace Grpc.Core.Internal
             }
         }
 
-        public void Stop()
+        public Task StopAsync()
         {
             lock (myLock)
             {
-                cq.Shutdown();
+                GrpcPreconditions.CheckState(!stopRequested, "Stop already requested.");
+                stopRequested = true;
+
+                foreach (var cq in completionQueues)
+                {
+                    cq.Shutdown();
+                }
+            }
+
+            return Task.Run(() =>
+            {
                 foreach (var thread in threads)
                 {
                     thread.Join();
                 }
 
-                cq.Dispose();
+                foreach (var cq in completionQueues)
+                {
+                    cq.Dispose();
+                }
+            });
+        }
+
+        /// <summary>
+        /// Returns true if there is at least one thread pool thread that hasn't
+        /// already stopped.
+        /// Threads can either stop because all completion queues shut down or
+        /// because all foreground threads have already shutdown and process is
+        /// going to exit.
+        /// </summary>
+        internal bool IsAlive
+        {
+            get
+            {
+                return threads.Any(t => t.ThreadState != ThreadState.Stopped);
             }
         }
 
-        internal CompletionQueueSafeHandle CompletionQueue
+        internal IReadOnlyCollection<CompletionQueueSafeHandle> CompletionQueues
         {
             get
             {
-                return cq;
+                return completionQueues;
             }
         }
 
-        private Thread CreateAndStartThread(int i)
+        private Thread CreateAndStartThread(int threadIndex)
         {
-            var thread = new Thread(new ThreadStart(RunHandlerLoop));
-            thread.IsBackground = false;
+            var cqIndex = threadIndex % completionQueues.Count;
+            var cq = completionQueues.ElementAt(cqIndex);
+
+            var thread = new Thread(new ThreadStart(() => RunHandlerLoop(cq)));
+            thread.IsBackground = true;
+            thread.Name = string.Format("grpc {0} (cq {1})", threadIndex, cqIndex);
             thread.Start();
-            thread.Name = "grpc " + i;
+
             return thread;
         }
 
         /// <summary>
         /// Body of the polling thread.
         /// </summary>
-        private void RunHandlerLoop()
+        private void RunHandlerLoop(CompletionQueueSafeHandle cq)
         {
             CompletionQueueEvent ev;
             do
             {
                 ev = cq.Next();
-                if (ev.type == GRPCCompletionType.OpComplete)
+                if (ev.type == CompletionQueueEvent.CompletionType.OpComplete)
                 {
                     bool success = (ev.success != 0);
                     IntPtr tag = ev.tag;
                     try
                     {
-                        var callback = environment.CompletionRegistry.Extract(tag);
+                        var callback = cq.CompletionRegistry.Extract(tag);
                         callback(success);
                     }
                     catch (Exception e)
@@ -133,7 +174,18 @@ namespace Grpc.Core.Internal
                     }
                 }
             }
-            while (ev.type != GRPCCompletionType.Shutdown);
+            while (ev.type != CompletionQueueEvent.CompletionType.Shutdown);
+        }
+
+        private static IReadOnlyCollection<CompletionQueueSafeHandle> CreateCompletionQueueList(GrpcEnvironment environment, int completionQueueCount)
+        {
+            var list = new List<CompletionQueueSafeHandle>();
+            for (int i = 0; i < completionQueueCount; i++)
+            {
+                var completionRegistry = new CompletionRegistry(environment);
+                list.Add(CompletionQueueSafeHandle.Create(completionRegistry));
+            }
+            return list.AsReadOnly();
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
index 25735d5262..dc9f62fdab 100644
--- a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
@@ -50,6 +50,11 @@ namespace Grpc.Core.Internal
         {
             using (Profilers.ForCurrentThread().NewScope("MetadataArraySafeHandle.Create"))
             {
+                if (metadata.Count == 0)
+                {
+                    return new MetadataArraySafeHandle();
+                }
+
                 // TODO(jtattermusch): we might wanna check that the metadata is readonly 
                 var metadataArray = Native.grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count));
                 for (int i = 0; i < metadata.Count; i++)
diff --git a/src/csharp/Grpc.Core/Internal/NativeExtension.cs b/src/csharp/Grpc.Core/Internal/NativeExtension.cs
index b45ba19c24..a6d7925816 100644
--- a/src/csharp/Grpc.Core/Internal/NativeExtension.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeExtension.cs
@@ -117,8 +117,8 @@ namespace Grpc.Core.Internal
         private static string GetAssemblyPath()
         {
             var assembly = typeof(NativeExtension).GetTypeInfo().Assembly;
-#if DOTNET5_4
-            // Assembly.EscapedCodeBase does not exit under CoreCLR, but assemblies imported from a nuget package
+#if NETSTANDARD1_5
+            // Assembly.EscapedCodeBase does not exist under CoreCLR, but assemblies imported from a nuget package
             // don't seem to be shadowed by DNX-based projects at all.
             return assembly.Location;
 #else
@@ -136,7 +136,7 @@ namespace Grpc.Core.Internal
 #endif
         }
 
-#if !DOTNET5_4
+#if !NETSTANDARD1_5
         private static bool IsFileUri(string uri)
         {
             return uri.ToLowerInvariant().StartsWith(Uri.UriSchemeFile);
diff --git a/src/csharp/Grpc.Core/Internal/NativeMethods.cs b/src/csharp/Grpc.Core/Internal/NativeMethods.cs
index c277c73ef0..65607ed120 100644
--- a/src/csharp/Grpc.Core/Internal/NativeMethods.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeMethods.cs
@@ -137,6 +137,7 @@ namespace Grpc.Core.Internal
         public readonly Delegates.grpcsharp_server_credentials_release_delegate grpcsharp_server_credentials_release;
 
         public readonly Delegates.grpcsharp_server_create_delegate grpcsharp_server_create;
+        public readonly Delegates.grpcsharp_server_register_completion_queue_delegate grpcsharp_server_register_completion_queue;
         public readonly Delegates.grpcsharp_server_add_insecure_http2_port_delegate grpcsharp_server_add_insecure_http2_port;
         public readonly Delegates.grpcsharp_server_add_secure_http2_port_delegate grpcsharp_server_add_secure_http2_port;
         public readonly Delegates.grpcsharp_server_start_delegate grpcsharp_server_start;
@@ -244,6 +245,7 @@ namespace Grpc.Core.Internal
                 this.grpcsharp_server_credentials_release = GetMethodDelegate<Delegates.grpcsharp_server_credentials_release_delegate>(library);
 
                 this.grpcsharp_server_create = GetMethodDelegate<Delegates.grpcsharp_server_create_delegate>(library);
+                this.grpcsharp_server_register_completion_queue = GetMethodDelegate<Delegates.grpcsharp_server_register_completion_queue_delegate>(library);
                 this.grpcsharp_server_add_insecure_http2_port = GetMethodDelegate<Delegates.grpcsharp_server_add_insecure_http2_port_delegate>(library);
                 this.grpcsharp_server_add_secure_http2_port = GetMethodDelegate<Delegates.grpcsharp_server_add_secure_http2_port_delegate>(library);
                 this.grpcsharp_server_start = GetMethodDelegate<Delegates.grpcsharp_server_start_delegate>(library);
@@ -348,6 +350,7 @@ namespace Grpc.Core.Internal
                 this.grpcsharp_server_credentials_release = PInvokeMethods.grpcsharp_server_credentials_release;
 
                 this.grpcsharp_server_create = PInvokeMethods.grpcsharp_server_create;
+                this.grpcsharp_server_register_completion_queue = PInvokeMethods.grpcsharp_server_register_completion_queue;
                 this.grpcsharp_server_add_insecure_http2_port = PInvokeMethods.grpcsharp_server_add_insecure_http2_port;
                 this.grpcsharp_server_add_secure_http2_port = PInvokeMethods.grpcsharp_server_add_secure_http2_port;
                 this.grpcsharp_server_start = PInvokeMethods.grpcsharp_server_start;
@@ -418,33 +421,33 @@ namespace Grpc.Core.Internal
             public delegate CallCredentialsSafeHandle grpcsharp_composite_call_credentials_create_delegate(CallCredentialsSafeHandle creds1, CallCredentialsSafeHandle creds2);
             public delegate void grpcsharp_call_credentials_release_delegate(IntPtr credentials);
 
-            public delegate GRPCCallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
-            public delegate GRPCCallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
-            public delegate GRPCCallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
+            public delegate CallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
+            public delegate CallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
                 MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
-            public delegate GRPCCallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
                 byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_recv_message_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_recv_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_start_serverside_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_serverside_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_send_initial_metadata_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_initial_metadata_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_set_credentials_delegate(CallSafeHandle call, CallCredentialsSafeHandle credentials);
+            public delegate CallError grpcsharp_call_set_credentials_delegate(CallSafeHandle call, CallCredentialsSafeHandle credentials);
             public delegate CStringSafeHandle grpcsharp_call_get_peer_delegate(CallSafeHandle call);
             public delegate void grpcsharp_call_destroy_delegate(IntPtr call);
 
@@ -493,23 +496,24 @@ namespace Grpc.Core.Internal
             public delegate ServerCredentialsSafeHandle grpcsharp_ssl_server_credentials_create_delegate(string pemRootCerts, string[] keyCertPairCertChainArray, string[] keyCertPairPrivateKeyArray, UIntPtr numKeyCertPairs, bool forceClientAuth);
             public delegate void grpcsharp_server_credentials_release_delegate(IntPtr credentials);
 
-            public delegate ServerSafeHandle grpcsharp_server_create_delegate(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args);
+            public delegate ServerSafeHandle grpcsharp_server_create_delegate(ChannelArgsSafeHandle args);
+            public delegate void grpcsharp_server_register_completion_queue_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq);
             public delegate int grpcsharp_server_add_insecure_http2_port_delegate(ServerSafeHandle server, string addr);
             public delegate int grpcsharp_server_add_secure_http2_port_delegate(ServerSafeHandle server, string addr, ServerCredentialsSafeHandle creds);
             public delegate void grpcsharp_server_start_delegate(ServerSafeHandle server);
-            public delegate GRPCCallError grpcsharp_server_request_call_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
+            public delegate CallError grpcsharp_server_request_call_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
             public delegate void grpcsharp_server_cancel_all_calls_delegate(ServerSafeHandle server);
             public delegate void grpcsharp_server_shutdown_and_notify_callback_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
             public delegate void grpcsharp_server_destroy_delegate(IntPtr server);
 
-            public delegate Timespec gprsharp_now_delegate(GPRClockType clockType);
-            public delegate Timespec gprsharp_inf_future_delegate(GPRClockType clockType);
-            public delegate Timespec gprsharp_inf_past_delegate(GPRClockType clockType);
+            public delegate Timespec gprsharp_now_delegate(ClockType clockType);
+            public delegate Timespec gprsharp_inf_future_delegate(ClockType clockType);
+            public delegate Timespec gprsharp_inf_past_delegate(ClockType clockType);
 
-            public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, GPRClockType targetClock);
+            public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, ClockType targetClock);
             public delegate int gprsharp_sizeof_timespec_delegate();
 
-            public delegate GRPCCallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public delegate CallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
             public delegate IntPtr grpcsharp_test_nop_delegate(IntPtr ptr);
         }
 
@@ -587,59 +591,59 @@ namespace Grpc.Core.Internal
             // CallSafeHandle
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_cancel(CallSafeHandle call);
+            public static extern CallError grpcsharp_call_cancel(CallSafeHandle call);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
+            public static extern CallError grpcsharp_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_unary(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_unary(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_client_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_client_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_server_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_server_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
                 MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_duplex_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_duplex_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_message(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_message(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_close_from_client(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_close_from_client(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_status_from_server(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_status_from_server(CallSafeHandle call,
                 BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
                 byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_recv_message(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_recv_message(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_recv_initial_metadata(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_recv_initial_metadata(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_serverside(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_serverside(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_initial_metadata(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_initial_metadata(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_set_credentials(CallSafeHandle call, CallCredentialsSafeHandle credentials);
+            public static extern CallError grpcsharp_call_set_credentials(CallSafeHandle call, CallCredentialsSafeHandle credentials);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern CStringSafeHandle grpcsharp_call_get_peer(CallSafeHandle call);
@@ -773,7 +777,10 @@ namespace Grpc.Core.Internal
             // ServerSafeHandle
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern ServerSafeHandle grpcsharp_server_create(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args);
+            public static extern ServerSafeHandle grpcsharp_server_create(ChannelArgsSafeHandle args);
+
+            [DllImport("grpc_csharp_ext.dll")]
+            public static extern void grpcsharp_server_register_completion_queue(ServerSafeHandle server, CompletionQueueSafeHandle cq);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern int grpcsharp_server_add_insecure_http2_port(ServerSafeHandle server, string addr);
@@ -785,7 +792,7 @@ namespace Grpc.Core.Internal
             public static extern void grpcsharp_server_start(ServerSafeHandle server);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_server_request_call(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
+            public static extern CallError grpcsharp_server_request_call(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern void grpcsharp_server_cancel_all_calls(ServerSafeHandle server);
@@ -799,16 +806,16 @@ namespace Grpc.Core.Internal
             // Timespec
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_now(GPRClockType clockType);
+            public static extern Timespec gprsharp_now(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_inf_future(GPRClockType clockType);
+            public static extern Timespec gprsharp_inf_future(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_inf_past(GPRClockType clockType);
+            public static extern Timespec gprsharp_inf_past(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_convert_clock_type(Timespec t, GPRClockType targetClock);
+            public static extern Timespec gprsharp_convert_clock_type(Timespec t, ClockType targetClock);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern int gprsharp_sizeof_timespec();
@@ -816,7 +823,7 @@ namespace Grpc.Core.Internal
             // Testing
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_test_callback([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public static extern CallError grpcsharp_test_callback([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern IntPtr grpcsharp_test_nop(IntPtr ptr);
diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs
index 5d8c44b589..15391ddc64 100644
--- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs
+++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs
@@ -53,7 +53,7 @@ namespace Grpc.Core.Internal
 
         static PlatformApis()
         {
-#if DNXCORE50
+#if NETSTANDARD1_5
             isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
             isMacOSX = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
             isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
diff --git a/src/csharp/Grpc.Core/Internal/SafeHandleZeroIsInvalid.cs b/src/csharp/Grpc.Core/Internal/SafeHandleZeroIsInvalid.cs
index 702aea2883..230faacff6 100644
--- a/src/csharp/Grpc.Core/Internal/SafeHandleZeroIsInvalid.cs
+++ b/src/csharp/Grpc.Core/Internal/SafeHandleZeroIsInvalid.cs
@@ -39,7 +39,7 @@ namespace Grpc.Core.Internal
     /// <summary>
     /// Safe handle to wrap native objects.
     /// </summary>
-    internal abstract class SafeHandleZeroIsInvalid : SafeHandle
+    internal abstract class SafeHandleZeroIsInvalid : System.Runtime.InteropServices.SafeHandle
     {
         public SafeHandleZeroIsInvalid() : base(IntPtr.Zero, true)
         {
diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
index 85b7a4b01e..6a2f520163 100644
--- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
@@ -44,7 +44,7 @@ namespace Grpc.Core.Internal
 {
     internal interface IServerCallHandler
     {
-        Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment);
+        Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq);
     }
 
     internal class UnaryServerCallHandler<TRequest, TResponse> : IServerCallHandler
@@ -62,14 +62,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -121,14 +121,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -179,14 +179,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -237,14 +237,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -281,13 +281,13 @@ namespace Grpc.Core.Internal
     {
         public static readonly NoSuchMethodCallHandler Instance = new NoSuchMethodCallHandler();
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             // We don't care about the payload type here.
             var asyncCall = new AsyncCallServer<byte[], byte[]>(
-                (payload) => payload, (payload) => payload, environment, newRpc.Server);
+                (payload) => payload, (payload) => payload, newRpc.Server);
             
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             await asyncCall.SendStatusFromServerAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty, null).ConfigureAwait(false);
             await finishedTask.ConfigureAwait(false);
@@ -317,7 +317,7 @@ namespace Grpc.Core.Internal
             where TRequest : class
             where TResponse : class
         {
-            DateTime realtimeDeadline = newRpc.Deadline.ToClockType(GPRClockType.Realtime).ToDateTime();
+            DateTime realtimeDeadline = newRpc.Deadline.ToClockType(ClockType.Realtime).ToDateTime();
 
             return new ServerCallContext(newRpc.Call, newRpc.Method, newRpc.Host, peer, realtimeDeadline,
                 newRpc.RequestMetadata, cancellationToken, serverResponseStream.WriteResponseHeadersAsync, serverResponseStream);
diff --git a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
index ecfee0bfdd..25b79b4398 100644
--- a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
@@ -52,16 +52,12 @@ namespace Grpc.Core.Internal
 
         public Task WriteAsync(TResponse message)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendMessage(message, GetWriteFlags(), taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendMessageAsync(message, GetWriteFlags());
         }
 
         public Task WriteResponseHeadersAsync(Metadata responseHeaders)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendInitialMetadata(responseHeaders, taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendInitialMetadataAsync(responseHeaders);
         }
 
         public WriteOptions WriteOptions
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCompletion.cs b/src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
index 7e86fddb4d..e4f1880bdb 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCompletion.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
@@ -32,63 +32,78 @@
 #endregion
 
 using System;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
-using System.Threading.Tasks;
-using Grpc.Core.Internal;
-using Grpc.Core.Utils;
+using Grpc.Core;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
-    /// If error != null, there's been an error or operation has been cancelled.
+    /// Details of a newly received RPC.
     /// </summary>
-    internal delegate void AsyncCompletionDelegate<T>(T result, Exception error);
-
-    /// <summary>
-    /// Helper for transforming AsyncCompletionDelegate into full-fledged Task.
-    /// </summary>
-    internal class AsyncCompletionTaskSource<T>
+    internal struct ServerRpcNew
     {
-        readonly TaskCompletionSource<T> tcs = new TaskCompletionSource<T>();
-        readonly AsyncCompletionDelegate<T> completionDelegate;
+        readonly Server server;
+        readonly CallSafeHandle call;
+        readonly string method;
+        readonly string host;
+        readonly Timespec deadline;
+        readonly Metadata requestMetadata;
+
+        public ServerRpcNew(Server server, CallSafeHandle call, string method, string host, Timespec deadline, Metadata requestMetadata)
+        {
+            this.server = server;
+            this.call = call;
+            this.method = method;
+            this.host = host;
+            this.deadline = deadline;
+            this.requestMetadata = requestMetadata;
+        }
+
+        public Server Server
+        {
+            get
+            {
+                return this.server;
+            }
+        }
 
-        public AsyncCompletionTaskSource()
+        public CallSafeHandle Call
         {
-            completionDelegate = new AsyncCompletionDelegate<T>(HandleCompletion);
+            get
+            {
+                return this.call;
+            }
         }
 
-        public Task<T> Task
+        public string Method
         {
             get
             {
-                return tcs.Task;
+                return this.method;
             }
         }
 
-        public AsyncCompletionDelegate<T> CompletionDelegate
+        public string Host
         {
             get
             {
-                return completionDelegate;
+                return this.host;
             }
         }
 
-        private void HandleCompletion(T value, Exception error)
+        public Timespec Deadline
         {
-            if (error == null)
+            get
             {
-                tcs.SetResult(value);
-                return;
+                return this.deadline;
             }
-            if (error is OperationCanceledException)
+        }
+
+        public Metadata RequestMetadata
+        {
+            get
             {
-                tcs.SetCanceled();
-                return;
+                return this.requestMetadata;
             }
-            tcs.SetException(error);
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
index 6b5f70e220..014a8db78f 100644
--- a/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
@@ -31,12 +31,6 @@
 
 #endregion
 
-using System;
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-using Grpc.Core.Utils;
-
 namespace Grpc.Core.Internal
 {
     /// <summary>
@@ -50,12 +44,20 @@ namespace Grpc.Core.Internal
         {
         }
 
-        public static ServerSafeHandle NewServer(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args)
+        public static ServerSafeHandle NewServer(ChannelArgsSafeHandle args)
         {
             // Increment reference count for the native gRPC environment to make sure we don't do grpc_shutdown() before destroying the server handle.
             // Doing so would make object finalizer crash if we end up abandoning the handle.
             GrpcEnvironment.GrpcNativeInit();
-            return Native.grpcsharp_server_create(cq, args);
+            return Native.grpcsharp_server_create(args);
+        }
+
+        public void RegisterCompletionQueue(CompletionQueueSafeHandle cq)
+        {
+            using (cq.NewScope())
+            {
+                Native.grpcsharp_server_register_completion_queue(this, cq);
+            }
         }
 
         public int AddInsecurePort(string addr)
@@ -73,18 +75,24 @@ namespace Grpc.Core.Internal
             Native.grpcsharp_server_start(this);
         }
     
-        public void ShutdownAndNotify(BatchCompletionDelegate callback, GrpcEnvironment environment)
+        public void ShutdownAndNotify(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
-            var ctx = BatchContextSafeHandle.Create();
-            environment.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
-            Native.grpcsharp_server_shutdown_and_notify_callback(this, environment.CompletionQueue, ctx);
+            using (completionQueue.NewScope())
+            {
+                var ctx = BatchContextSafeHandle.Create();
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+                Native.grpcsharp_server_shutdown_and_notify_callback(this, completionQueue, ctx);
+            }
         }
 
-        public void RequestCall(BatchCompletionDelegate callback, GrpcEnvironment environment)
+        public void RequestCall(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
-            var ctx = BatchContextSafeHandle.Create();
-            environment.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
-            Native.grpcsharp_server_request_call(this, environment.CompletionQueue, ctx).CheckOk();
+            using (completionQueue.NewScope())
+            {
+                var ctx = BatchContextSafeHandle.Create();
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+                Native.grpcsharp_server_request_call(this, completionQueue, ctx).CheckOk();
+            }
         }
 
         protected override bool ReleaseHandle()
diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs
index 56172a5dda..c9fd710e1e 100644
--- a/src/csharp/Grpc.Core/Internal/Timespec.cs
+++ b/src/csharp/Grpc.Core/Internal/Timespec.cs
@@ -49,11 +49,11 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
         static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
 
-        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, GPRClockType.Realtime)
+        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, ClockType.Realtime)
         {
         }
 
-        public Timespec(long tv_sec, int tv_nsec, GPRClockType clock_type)
+        public Timespec(long tv_sec, int tv_nsec, ClockType clock_type)
         {
             this.tv_sec = tv_sec;
             this.tv_nsec = tv_nsec;
@@ -62,7 +62,7 @@ namespace Grpc.Core.Internal
 
         private long tv_sec;
         private int tv_nsec;
-        private GPRClockType clock_type;
+        private ClockType clock_type;
 
         /// <summary>
         /// Timespec a long time in the future.
@@ -71,7 +71,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_inf_future(GPRClockType.Realtime);
+                return new Timespec(long.MaxValue, 0, ClockType.Realtime);
             }
         }
 
@@ -82,7 +82,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_inf_past(GPRClockType.Realtime);
+                return new Timespec(long.MinValue, 0, ClockType.Realtime);
             }
         }
 
@@ -93,7 +93,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_now(GPRClockType.Realtime);
+                return Native.gprsharp_now(ClockType.Realtime);
             }
         }
 
@@ -122,7 +122,7 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Converts the timespec to desired clock type.
         /// </summary>
-        public Timespec ToClockType(GPRClockType targetClock)
+        public Timespec ToClockType(ClockType targetClock)
         {
             return Native.gprsharp_convert_clock_type(this, targetClock);
         }
@@ -142,7 +142,7 @@ namespace Grpc.Core.Internal
         public DateTime ToDateTime()
         {
             GrpcPreconditions.CheckState(tv_nsec >= 0 && tv_nsec < NanosPerSecond);
-            GrpcPreconditions.CheckState(clock_type == GPRClockType.Realtime);
+            GrpcPreconditions.CheckState(clock_type == ClockType.Realtime);
 
             // fast path for InfFuture
             if (this.Equals(InfFuture))
@@ -227,10 +227,11 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_now(GPRClockType.Precise);
+                return Native.gprsharp_now(ClockType.Precise);
             }
         }
 
+        // for tests only
         internal static int NativeSize
         {
             get
@@ -238,5 +239,23 @@ namespace Grpc.Core.Internal
                 return Native.gprsharp_sizeof_timespec();
             }
         }
+
+        // for tests only
+        internal static Timespec NativeInfFuture
+        {
+            get
+            {
+                return Native.gprsharp_inf_future(ClockType.Realtime);
+            }
+        }
+
+        // for tests only
+        public static Timespec NativeInfPast
+        {
+            get
+            {
+                return Native.gprsharp_inf_past(ClockType.Realtime);
+            }
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Logging/LogLevel.cs b/src/csharp/Grpc.Core/Logging/LogLevel.cs
new file mode 100644
index 0000000000..d64e1f5fd0
--- /dev/null
+++ b/src/csharp/Grpc.Core/Logging/LogLevel.cs
@@ -0,0 +1,59 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+
+namespace Grpc.Core.Logging
+{
+    /// <summary>Standard logging levels.</summary>
+    public enum LogLevel
+    {
+        /// <summary>
+        /// Debug severity.
+        /// </summary>
+        Debug = 0,
+        /// <summary>
+        /// Info severity.
+        /// </summary>
+        Info,
+        /// <summary>
+        /// Warning severity.
+        /// </summary>
+        Warning,
+        /// <summary>
+        /// Error severity.
+        /// </summary>
+        Error
+    }
+}
diff --git a/src/csharp/Grpc.Core/Logging/LogLevelFilterLogger.cs b/src/csharp/Grpc.Core/Logging/LogLevelFilterLogger.cs
new file mode 100644
index 0000000000..4eeb79c783
--- /dev/null
+++ b/src/csharp/Grpc.Core/Logging/LogLevelFilterLogger.cs
@@ -0,0 +1,160 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using Grpc.Core.Utils;
+
+namespace Grpc.Core.Logging
+{
+    /// <summary>Logger that filters out messages below certain log level.</summary>
+    public class LogLevelFilterLogger : ILogger
+    {
+        readonly ILogger innerLogger;
+        readonly LogLevel logLevel;
+
+        /// <summary>
+        /// Creates and instance of <c>LogLevelFilter.</c>
+        /// </summary>
+        public LogLevelFilterLogger(ILogger logger, LogLevel logLevel)
+        {
+            this.innerLogger = GrpcPreconditions.CheckNotNull(logger);
+            this.logLevel = logLevel;
+        }
+
+        /// <summary>
+        /// Returns a logger associated with the specified type.
+        /// </summary>
+        public virtual ILogger ForType<T>()
+        {
+            var newInnerLogger = innerLogger.ForType<T>();
+            if (object.ReferenceEquals(this.innerLogger, newInnerLogger))
+            {
+                return this;
+            }
+            return new LogLevelFilterLogger(newInnerLogger, logLevel);
+        }
+
+        /// <summary>Logs a message with severity Debug.</summary>
+        public void Debug(string message)
+        {
+            if (logLevel <= LogLevel.Debug)
+            {
+                innerLogger.Debug(message);
+            }
+        }
+
+        /// <summary>Logs a formatted message with severity Debug.</summary>
+        public void Debug(string format, params object[] formatArgs)
+        {
+            if (logLevel <= LogLevel.Debug)
+            {
+                innerLogger.Debug(format, formatArgs);
+            }
+        }
+
+        /// <summary>Logs a message with severity Info.</summary>
+        public void Info(string message)
+        {
+            if (logLevel <= LogLevel.Info)
+            {
+                innerLogger.Info(message);
+            }
+        }
+
+        /// <summary>Logs a formatted message with severity Info.</summary>
+        public void Info(string format, params object[] formatArgs)
+        {
+            if (logLevel <= LogLevel.Info)
+            {
+                innerLogger.Info(format, formatArgs);
+            }
+        }
+
+        /// <summary>Logs a message with severity Warning.</summary>
+        public void Warning(string message)
+        {
+            if (logLevel <= LogLevel.Warning)
+            {
+                innerLogger.Warning(message);
+            }
+        }
+
+        /// <summary>Logs a formatted message with severity Warning.</summary>
+        public void Warning(string format, params object[] formatArgs)
+        {
+            if (logLevel <= LogLevel.Warning)
+            {
+                innerLogger.Warning(format, formatArgs);
+            }
+        }
+
+        /// <summary>Logs a message and an associated exception with severity Warning.</summary>
+        public void Warning(Exception exception, string message)
+        {
+            if (logLevel <= LogLevel.Warning)
+            {
+                innerLogger.Warning(exception, message);
+            }
+        }
+
+        /// <summary>Logs a message with severity Error.</summary>
+        public void Error(string message)
+        {
+            if (logLevel <= LogLevel.Error)
+            {
+                innerLogger.Error(message);
+            }
+        }
+
+        /// <summary>Logs a formatted message with severity Error.</summary>
+        public void Error(string format, params object[] formatArgs)
+        {
+            if (logLevel <= LogLevel.Error)
+            {
+                innerLogger.Error(format, formatArgs);
+            }
+        }
+
+        /// <summary>Logs a message and an associated exception with severity Error.</summary>
+        public void Error(Exception exception, string message)
+        {
+            if (logLevel <= LogLevel.Error)
+            {
+                innerLogger.Error(exception, message);
+            }
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core/Metadata.cs b/src/csharp/Grpc.Core/Metadata.cs
index e982fa0c48..915bec146c 100644
--- a/src/csharp/Grpc.Core/Metadata.cs
+++ b/src/csharp/Grpc.Core/Metadata.cs
@@ -63,6 +63,13 @@ namespace Grpc.Core
         /// </summary>
         public static readonly Metadata Empty = new Metadata().Freeze();
 
+        /// <summary>
+        /// To be used in initial metadata to request specific compression algorithm
+        /// for given call. Direct selection of compression algorithms is an internal
+        /// feature and is not part of public API.
+        /// </summary>
+        internal const string CompressionRequestAlgorithmMetadataKey = "grpc-internal-encoding-request";
+
         readonly List<Entry> entries;
         bool readOnly;
 
@@ -95,6 +102,7 @@ namespace Grpc.Core
 
         public void Insert(int index, Metadata.Entry item)
         {
+            GrpcPreconditions.CheckNotNull(item);
             CheckWriteable();
             entries.Insert(index, item);
         }
@@ -114,6 +122,7 @@ namespace Grpc.Core
 
             set
             {
+                GrpcPreconditions.CheckNotNull(value);
                 CheckWriteable();
                 entries[index] = value;
             }
@@ -121,6 +130,7 @@ namespace Grpc.Core
 
         public void Add(Metadata.Entry item)
         {
+            GrpcPreconditions.CheckNotNull(item);
             CheckWriteable();
             entries.Add(item);
         }
@@ -187,7 +197,7 @@ namespace Grpc.Core
         /// <summary>
         /// Metadata entry
         /// </summary>
-        public struct Entry
+        public class Entry
         {
             private static readonly Encoding Encoding = Encoding.ASCII;
             private static readonly Regex ValidKeyRegex = new Regex("^[a-z0-9_-]+$");
diff --git a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
index bde74945fb..370fa98687 100644
--- a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
+++ b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
@@ -16,6 +16,12 @@ using System.Runtime.CompilerServices;
     "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" +
     "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" +
     "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")]
+[assembly: InternalsVisibleTo("Grpc.IntegrationTesting,PublicKey=" +
+    "00240000048000009400000006020000002400005253413100040000010001002f5797a92c6fcde81bd4098f43" +
+    "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" +
+    "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" +
+    "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")]
 #else
 [assembly: InternalsVisibleTo("Grpc.Core.Tests")]
+[assembly: InternalsVisibleTo("Grpc.IntegrationTesting")]
 #endif
diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs
index 5b61b7f060..3b554e5e87 100644
--- a/src/csharp/Grpc.Core/Server.cs
+++ b/src/csharp/Grpc.Core/Server.cs
@@ -34,8 +34,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
+using System.Linq;
 using System.Threading.Tasks;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
@@ -48,6 +47,7 @@ namespace Grpc.Core
     /// </summary>
     public class Server
     {
+        const int InitialAllowRpcTokenCountPerCq = 10;
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<Server>();
 
         readonly AtomicCounter activeCallCounter = new AtomicCounter();
@@ -65,13 +65,21 @@ namespace Grpc.Core
         readonly TaskCompletionSource<object> shutdownTcs = new TaskCompletionSource<object>();
 
         bool startRequested;
-        bool shutdownRequested;
+        volatile bool shutdownRequested;
+
+
+        /// <summary>
+        /// Creates a new server.
+        /// </summary>
+        public Server() : this(null)
+        {
+        }
 
         /// <summary>
-        /// Create a new server.
+        /// Creates a new server.
         /// </summary>
         /// <param name="options">Channel options.</param>
-        public Server(IEnumerable<ChannelOption> options = null)
+        public Server(IEnumerable<ChannelOption> options)
         {
             this.serviceDefinitions = new ServiceDefinitionCollection(this);
             this.ports = new ServerPortCollection(this);
@@ -79,8 +87,14 @@ namespace Grpc.Core
             this.options = options != null ? new List<ChannelOption>(options) : new List<ChannelOption>();
             using (var channelArgs = ChannelOptions.CreateChannelArgs(this.options))
             {
-                this.handle = ServerSafeHandle.NewServer(environment.CompletionQueue, channelArgs);
+                this.handle = ServerSafeHandle.NewServer(channelArgs);
+            }
+
+            foreach (var cq in environment.CompletionQueues)
+            {
+                this.handle.RegisterCompletionQueue(cq);
             }
+            GrpcEnvironment.RegisterServer(this);
         }
 
         /// <summary>
@@ -126,10 +140,20 @@ namespace Grpc.Core
             lock (myLock)
             {
                 GrpcPreconditions.CheckState(!startRequested);
+                GrpcPreconditions.CheckState(!shutdownRequested);
                 startRequested = true;
                 
                 handle.Start();
-                AllowOneRpc();
+
+                // Starting with more than one AllowOneRpc tokens can significantly increase
+                // unary RPC throughput.
+                for (int i = 0; i < InitialAllowRpcTokenCountPerCq; i++)
+                {
+                    foreach (var cq in environment.CompletionQueues)
+                    {
+                        AllowOneRpc(cq);
+                    }
+                }
             }
         }
 
@@ -138,41 +162,24 @@ namespace Grpc.Core
         /// cleans up used resources. The returned task finishes when shutdown procedure
         /// is complete.
         /// </summary>
-        public async Task ShutdownAsync()
+        /// <remarks>
+        /// It is strongly recommended to shutdown all previously created servers before exiting from the process.
+        /// </remarks>
+        public Task ShutdownAsync()
         {
-            lock (myLock)
-            {
-                GrpcPreconditions.CheckState(startRequested);
-                GrpcPreconditions.CheckState(!shutdownRequested);
-                shutdownRequested = true;
-            }
-
-            handle.ShutdownAndNotify(HandleServerShutdown, environment);
-            await shutdownTcs.Task.ConfigureAwait(false);
-            DisposeHandle();
-
-            await Task.Run(() => GrpcEnvironment.Release()).ConfigureAwait(false);
+            return ShutdownInternalAsync(false);
         }
 
         /// <summary>
         /// Requests server shutdown while cancelling all the in-progress calls.
         /// The returned task finishes when shutdown procedure is complete.
         /// </summary>
-        public async Task KillAsync()
+        /// <remarks>
+        /// It is strongly recommended to shutdown all previously created servers before exiting from the process.
+        /// </remarks>
+        public Task KillAsync()
         {
-            lock (myLock)
-            {
-                GrpcPreconditions.CheckState(startRequested);
-                GrpcPreconditions.CheckState(!shutdownRequested);
-                shutdownRequested = true;
-            }
-
-            handle.ShutdownAndNotify(HandleServerShutdown, environment);
-            handle.CancelAllCalls();
-            await shutdownTcs.Task.ConfigureAwait(false);
-            DisposeHandle();
-
-            await Task.Run(() => GrpcEnvironment.Release()).ConfigureAwait(false);
+            return ShutdownInternalAsync(true);
         }
 
         internal void AddCallReference(object call)
@@ -191,6 +198,51 @@ namespace Grpc.Core
         }
 
         /// <summary>
+        /// Shuts down the server.
+        /// </summary>
+        private async Task ShutdownInternalAsync(bool kill)
+        {
+            lock (myLock)
+            {
+                GrpcPreconditions.CheckState(!shutdownRequested);
+                shutdownRequested = true;
+            }
+            GrpcEnvironment.UnregisterServer(this);
+
+            var cq = environment.CompletionQueues.First();  // any cq will do
+            handle.ShutdownAndNotify(HandleServerShutdown, cq);
+            if (kill)
+            {
+                handle.CancelAllCalls();
+            }
+            await ShutdownCompleteOrEnvironmentDeadAsync().ConfigureAwait(false);
+
+            DisposeHandle();
+
+            await GrpcEnvironment.ReleaseAsync().ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// In case the environment's threadpool becomes dead, the shutdown completion will
+        /// never be delivered, but we need to release the environment's handle anyway.
+        /// </summary>
+        private async Task ShutdownCompleteOrEnvironmentDeadAsync()
+        {
+            while (true)
+            {
+                var task = await Task.WhenAny(shutdownTcs.Task, Task.Delay(20)).ConfigureAwait(false);
+                if (shutdownTcs.Task == task)
+                {
+                    return;
+                }
+                if (!environment.IsAlive)
+                {
+                    return;
+                }
+            }
+        }
+
+        /// <summary>
         /// Adds a service definition.
         /// </summary>
         private void AddServiceDefinitionInternal(ServerServiceDefinition serviceDefinition)
@@ -237,14 +289,11 @@ namespace Grpc.Core
         /// <summary>
         /// Allows one new RPC call to be received by server.
         /// </summary>
-        private void AllowOneRpc()
+        private void AllowOneRpc(CompletionQueueSafeHandle cq)
         {
-            lock (myLock)
+            if (!shutdownRequested)
             {
-                if (!shutdownRequested)
-                {
-                    handle.RequestCall(HandleNewServerRpc, environment);
-                }
+                handle.RequestCall((success, ctx) => HandleNewServerRpc(success, ctx, cq), cq);
             }
         }
 
@@ -261,7 +310,7 @@ namespace Grpc.Core
         /// <summary>
         /// Selects corresponding handler for given call and handles the call.
         /// </summary>
-        private async Task HandleCallAsync(ServerRpcNew newRpc)
+        private async Task HandleCallAsync(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             try
             {
@@ -270,7 +319,7 @@ namespace Grpc.Core
                 {
                     callHandler = NoSuchMethodCallHandler.Instance;
                 }
-                await callHandler.HandleCall(newRpc, environment).ConfigureAwait(false);
+                await callHandler.HandleCall(newRpc, cq).ConfigureAwait(false);
             }
             catch (Exception e)
             {
@@ -281,8 +330,10 @@ namespace Grpc.Core
         /// <summary>
         /// Handles the native callback.
         /// </summary>
-        private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx)
+        private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx, CompletionQueueSafeHandle cq)
         {
+			Task.Run(() => AllowOneRpc(cq));
+
             if (success)
             {
                 ServerRpcNew newRpc = ctx.GetServerRpcNew(this);
@@ -290,11 +341,9 @@ namespace Grpc.Core
                 // after server shutdown, the callback returns with null call
                 if (!newRpc.Call.IsInvalid)
                 {
-                    Task.Run(async () => await HandleCallAsync(newRpc)).ConfigureAwait(false);
+                    HandleCallAsync(newRpc, cq);  // we don't need to await.
                 }
             }
-
-            AllowOneRpc();
         }
 
         /// <summary>
diff --git a/src/csharp/Grpc.Core/ServerServiceDefinition.cs b/src/csharp/Grpc.Core/ServerServiceDefinition.cs
index deb1431ca3..ac08c04bf6 100644
--- a/src/csharp/Grpc.Core/ServerServiceDefinition.cs
+++ b/src/csharp/Grpc.Core/ServerServiceDefinition.cs
@@ -63,11 +63,10 @@ namespace Grpc.Core
         /// <summary>
         /// Creates a new builder object for <c>ServerServiceDefinition</c>.
         /// </summary>
-        /// <param name="serviceName">The service name.</param>
         /// <returns>The builder object.</returns>
-        public static Builder CreateBuilder(string serviceName)
+        public static Builder CreateBuilder()
         {
-            return new Builder(serviceName);
+            return new Builder();
         }
 
         /// <summary>
@@ -75,16 +74,13 @@ namespace Grpc.Core
         /// </summary>
         public class Builder
         {
-            readonly string serviceName;
             readonly Dictionary<string, IServerCallHandler> callHandlers = new Dictionary<string, IServerCallHandler>();
 
             /// <summary>
             /// Creates a new instance of builder.
             /// </summary>
-            /// <param name="serviceName">The service name.</param>
-            public Builder(string serviceName)
+            public Builder()
             {
-                this.serviceName = serviceName;
             }
 
             /// <summary>
diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs
index 9b2f6ed1c3..cb20967680 100644
--- a/src/csharp/Grpc.Core/VersionInfo.cs
+++ b/src/csharp/Grpc.Core/VersionInfo.cs
@@ -48,11 +48,11 @@ namespace Grpc.Core
         /// <summary>
         /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies
         /// </summary>
-        public const string CurrentAssemblyFileVersion = "0.14.2.0";
+        public const string CurrentAssemblyFileVersion = "0.16.0.0";
 
         /// <summary>
         /// Current version of gRPC C#
         /// </summary>
-        public const string CurrentVersion = "0.14.2-pre1";
+        public const string CurrentVersion = "0.16.0-dev";
     }
 }
diff --git a/src/csharp/Grpc.Core/WriteOptions.cs b/src/csharp/Grpc.Core/WriteOptions.cs
index 7523ada84a..4c9706d966 100644
--- a/src/csharp/Grpc.Core/WriteOptions.cs
+++ b/src/csharp/Grpc.Core/WriteOptions.cs
@@ -1,6 +1,6 @@
 #region Copyright notice and license
 
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -64,7 +64,7 @@ namespace Grpc.Core
         /// </summary>
         public static readonly WriteOptions Default = new WriteOptions();
             
-        private WriteFlags flags;
+        private readonly WriteFlags flags;
 
         /// <summary>
         /// Initializes a new instance of <c>WriteOptions</c> class.
diff --git a/src/csharp/Grpc.Core/project.json b/src/csharp/Grpc.Core/project.json
new file mode 100644
index 0000000000..201e548801
--- /dev/null
+++ b/src/csharp/Grpc.Core/project.json
@@ -0,0 +1,48 @@
+{
+  "version": "0.16.0-dev",
+  "title": "gRPC C# Core",
+  "authors": [ "Google Inc." ],
+  "copyright": "Copyright 2015, Google Inc.",
+  "packOptions": {
+    "summary": "Core C# implementation of gRPC - an RPC library and framework",
+    "description": "Core C# implementation of gRPC - an RPC library and framework. See project site for more info.",
+    "owners": [ "grpc-packages" ],
+    "licenseUrl": "https://github.com/grpc/grpc/blob/master/LICENSE",
+    "projectUrl": "https://github.com/grpc/grpc",
+    "requireLicenseAcceptance": false,
+    "tags": [ "gRPC RPC Protocol HTTP/2" ],
+    "files": {
+      "mappings": {
+        "build/net45/": "Grpc.Core.targets",
+        "build/native/bin/windows_x86/": "../nativelibs/windows_x86/grpc_csharp_ext.dll",
+        "build/native/bin/windows_x64/": "../nativelibs/windows_x64/grpc_csharp_ext.dll",
+        "build/native/bin/linux_x86/": "../nativelibs/linux_x86/libgrpc_csharp_ext.so",
+        "build/native/bin/linux_x64/": "../nativelibs/linux_x64/libgrpc_csharp_ext.so",
+        "build/native/bin/macosx_x86/": "../nativelibs/macosx_x86/libgrpc_csharp_ext.dylib",
+        "build/native/bin/macosx_x64/": "../nativelibs/macosx_x64/libgrpc_csharp_ext.dylib"
+      }
+    }
+  },
+  "buildOptions": {
+    "embed": [ "../../../etc/roots.pem" ],
+    "define": [ "SIGNED" ],
+    "keyFile": "../keys/Grpc.snk",
+    "publicSign": true,
+    "xmlDoc": true
+  },
+  "dependencies": {
+    "Ix-Async": "1.2.5"
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027",
+        "System.Threading.Thread": "4.0.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.Dotnet.sln b/src/csharp/Grpc.Dotnet.sln
new file mode 100644
index 0000000000..98b3cd54ab
--- /dev/null
+++ b/src/csharp/Grpc.Dotnet.sln
@@ -0,0 +1,100 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25123.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Core", "Grpc.Core\Grpc.Core.xproj", "{DC9908B6-F291-4FC8-A46D-2EA2551790EC}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Auth", "Grpc.Auth\Grpc.Auth.xproj", "{C82631ED-06D1-4458-87BC-8257D12307A8}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Core.Tests", "Grpc.Core.Tests\Grpc.Core.Tests.xproj", "{759E23B2-FC04-4695-902D-B073CDED3599}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Examples", "Grpc.Examples\Grpc.Examples.xproj", "{C77B792D-FC78-4CE2-9522-B40B0803C636}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Examples.MathClient", "Grpc.Examples.MathClient\Grpc.Examples.MathClient.xproj", "{FD48DECA-1622-4173-B1D9-2101CF5E7C5F}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Examples.MathServer", "Grpc.Examples.MathServer\Grpc.Examples.MathServer.xproj", "{58579368-5372-4E67-ACD6-9B59CB9FA698}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Examples.Tests", "Grpc.Examples.Tests\Grpc.Examples.Tests.xproj", "{C61714A6-F633-44FB-97F4-C91F425C1D15}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.HealthCheck", "Grpc.HealthCheck\Grpc.HealthCheck.xproj", "{3BE4AD0B-2BF0-4D68-B625-F6018EF0DCFA}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.HealthCheck.Tests", "Grpc.HealthCheck.Tests\Grpc.HealthCheck.Tests.xproj", "{43DAFAC6-5343-4621-960E-A8A977EA3F0B}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting", "Grpc.IntegrationTesting\Grpc.IntegrationTesting.xproj", "{20354386-3E71-4046-A269-3BC2A06F3EC8}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting.Client", "Grpc.IntegrationTesting.Client\Grpc.IntegrationTesting.Client.xproj", "{48EA5BBE-70E2-4198-869D-D7E59C45F30D}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting.QpsWorker", "Grpc.IntegrationTesting.QpsWorker\Grpc.IntegrationTesting.QpsWorker.xproj", "{661B70D7-F56A-46E0-9B81-6227B591B5E7}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting.Server", "Grpc.IntegrationTesting.Server\Grpc.IntegrationTesting.Server.xproj", "{881F7AD1-A84E-47A2-9402-115C63C4031E}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting.StressClient", "Grpc.IntegrationTesting.StressClient\Grpc.IntegrationTesting.StressClient.xproj", "{0EBC910B-8867-4D3E-8686-91F34183D839}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9908B6-F291-4FC8-A46D-2EA2551790EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC9908B6-F291-4FC8-A46D-2EA2551790EC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC9908B6-F291-4FC8-A46D-2EA2551790EC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC9908B6-F291-4FC8-A46D-2EA2551790EC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C82631ED-06D1-4458-87BC-8257D12307A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C82631ED-06D1-4458-87BC-8257D12307A8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C82631ED-06D1-4458-87BC-8257D12307A8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C82631ED-06D1-4458-87BC-8257D12307A8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{759E23B2-FC04-4695-902D-B073CDED3599}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{759E23B2-FC04-4695-902D-B073CDED3599}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{759E23B2-FC04-4695-902D-B073CDED3599}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{759E23B2-FC04-4695-902D-B073CDED3599}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C77B792D-FC78-4CE2-9522-B40B0803C636}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C77B792D-FC78-4CE2-9522-B40B0803C636}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C77B792D-FC78-4CE2-9522-B40B0803C636}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C77B792D-FC78-4CE2-9522-B40B0803C636}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FD48DECA-1622-4173-B1D9-2101CF5E7C5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FD48DECA-1622-4173-B1D9-2101CF5E7C5F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FD48DECA-1622-4173-B1D9-2101CF5E7C5F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FD48DECA-1622-4173-B1D9-2101CF5E7C5F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{58579368-5372-4E67-ACD6-9B59CB9FA698}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{58579368-5372-4E67-ACD6-9B59CB9FA698}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{58579368-5372-4E67-ACD6-9B59CB9FA698}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{58579368-5372-4E67-ACD6-9B59CB9FA698}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C61714A6-F633-44FB-97F4-C91F425C1D15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C61714A6-F633-44FB-97F4-C91F425C1D15}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C61714A6-F633-44FB-97F4-C91F425C1D15}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C61714A6-F633-44FB-97F4-C91F425C1D15}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3BE4AD0B-2BF0-4D68-B625-F6018EF0DCFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3BE4AD0B-2BF0-4D68-B625-F6018EF0DCFA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3BE4AD0B-2BF0-4D68-B625-F6018EF0DCFA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3BE4AD0B-2BF0-4D68-B625-F6018EF0DCFA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43DAFAC6-5343-4621-960E-A8A977EA3F0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43DAFAC6-5343-4621-960E-A8A977EA3F0B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43DAFAC6-5343-4621-960E-A8A977EA3F0B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43DAFAC6-5343-4621-960E-A8A977EA3F0B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{20354386-3E71-4046-A269-3BC2A06F3EC8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{20354386-3E71-4046-A269-3BC2A06F3EC8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{20354386-3E71-4046-A269-3BC2A06F3EC8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{20354386-3E71-4046-A269-3BC2A06F3EC8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{48EA5BBE-70E2-4198-869D-D7E59C45F30D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{48EA5BBE-70E2-4198-869D-D7E59C45F30D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{48EA5BBE-70E2-4198-869D-D7E59C45F30D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{48EA5BBE-70E2-4198-869D-D7E59C45F30D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{661B70D7-F56A-46E0-9B81-6227B591B5E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{661B70D7-F56A-46E0-9B81-6227B591B5E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{661B70D7-F56A-46E0-9B81-6227B591B5E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{661B70D7-F56A-46E0-9B81-6227B591B5E7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{881F7AD1-A84E-47A2-9402-115C63C4031E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{881F7AD1-A84E-47A2-9402-115C63C4031E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{881F7AD1-A84E-47A2-9402-115C63C4031E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{881F7AD1-A84E-47A2-9402-115C63C4031E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0EBC910B-8867-4D3E-8686-91F34183D839}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0EBC910B-8867-4D3E-8686-91F34183D839}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0EBC910B-8867-4D3E-8686-91F34183D839}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0EBC910B-8867-4D3E-8686-91F34183D839}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj
index 35c0646a3f..65bf236def 100644
--- a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj
+++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj
@@ -57,4 +57,7 @@
       <Name>Grpc.Examples</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <None Include="Grpc.Examples.MathClient.project.json" />
+  </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.project.json b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.xproj b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.xproj
new file mode 100644
index 0000000000..4655bd4377
--- /dev/null
+++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>fd48deca-1622-4173-b1d9-2101cf5e7c5f</ProjectGuid>
+    <RootNamespace>Grpc.Examples.MathClient</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples.MathClient/project.json b/src/csharp/Grpc.Examples.MathClient/project.json
new file mode 100644
index 0000000000..b865cd5011
--- /dev/null
+++ b/src/csharp/Grpc.Examples.MathClient/project.json
@@ -0,0 +1,67 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.Examples": {
+      "target": "project"
+    }
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj
index 74d79b44d9..26b42b6936 100644
--- a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj
+++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj
@@ -57,4 +57,7 @@
       <Name>Grpc.Examples</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <None Include="Grpc.Examples.MathServer.project.json" />
+  </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.project.json b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.xproj b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.xproj
new file mode 100644
index 0000000000..38a449e8f2
--- /dev/null
+++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>58579368-5372-4e67-acd6-9b59cb9fa698</ProjectGuid>
+    <RootNamespace>Grpc.Examples.MathServer</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples.MathServer/project.json b/src/csharp/Grpc.Examples.MathServer/project.json
new file mode 100644
index 0000000000..b865cd5011
--- /dev/null
+++ b/src/csharp/Grpc.Examples.MathServer/project.json
@@ -0,0 +1,67 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.Examples": {
+      "target": "project"
+    }
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
index 3fd28c6528..4c7d89309a 100644
--- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
+++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
@@ -69,6 +69,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.Examples.Tests.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.project.json b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.xproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.xproj
new file mode 100644
index 0000000000..9cecd18b2e
--- /dev/null
+++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>c61714a6-f633-44fb-97f4-c91f425c1d15</ProjectGuid>
+    <RootNamespace>Grpc.Examples.Tests</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs b/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs
index 875202b950..50dacc2eaa 100644
--- a/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs
+++ b/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs
@@ -62,7 +62,7 @@ namespace Math.Tests
             };
             server.Start();
             channel = new Channel(Host, server.Ports.Single().BoundPort, ChannelCredentials.Insecure);
-            client = Math.NewClient(channel);
+            client = new Math.MathClient(channel);
         }
 
         [TestFixtureTearDown]
@@ -92,7 +92,7 @@ namespace Math.Tests
         public void DivByZero()
         {
             var ex = Assert.Throws<RpcException>(() => client.Div(new DivArgs { Dividend = 0, Divisor = 0 }));
-            Assert.AreEqual(StatusCode.Unknown, ex.Status.StatusCode);
+            Assert.AreEqual(StatusCode.InvalidArgument, ex.Status.StatusCode);
         }
 
         [Test]
@@ -110,7 +110,7 @@ namespace Math.Tests
             {
                 var responses = await call.ResponseStream.ToListAsync();
                 CollectionAssert.AreEqual(new List<long> { 1, 1, 2, 3, 5, 8 },
-                    responses.ConvertAll((n) => n.Num_));
+                    responses.Select((n) => n.Num_));
             }
         }
 
@@ -162,7 +162,7 @@ namespace Math.Tests
         {
             using (var call = client.Sum())
             {
-                var numbers = new List<long> { 10, 20, 30 }.ConvertAll(n => new Num { Num_ = n });
+                var numbers = new List<long> { 10, 20, 30 }.Select(n => new Num { Num_ = n });
 
                 await call.RequestStream.WriteAllAsync(numbers);
                 var result = await call.ResponseAsync;
@@ -185,8 +185,8 @@ namespace Math.Tests
                 await call.RequestStream.WriteAllAsync(divArgsList);
                 var result = await call.ResponseStream.ToListAsync();
 
-                CollectionAssert.AreEqual(new long[] { 3, 4, 3 }, result.ConvertAll((divReply) => divReply.Quotient));
-                CollectionAssert.AreEqual(new long[] { 1, 16, 1 }, result.ConvertAll((divReply) => divReply.Remainder));
+                CollectionAssert.AreEqual(new long[] { 3, 4, 3 }, result.Select((divReply) => divReply.Quotient));
+                CollectionAssert.AreEqual(new long[] { 1, 16, 1 }, result.Select((divReply) => divReply.Remainder));
             }
         }
     }
diff --git a/src/csharp/Grpc.Examples.Tests/NUnitMain.cs b/src/csharp/Grpc.Examples.Tests/NUnitMain.cs
index ea87802766..1a522cab93 100644
--- a/src/csharp/Grpc.Examples.Tests/NUnitMain.cs
+++ b/src/csharp/Grpc.Examples.Tests/NUnitMain.cs
@@ -49,7 +49,7 @@ namespace Grpc.Examples.Tests
         {
             // Make logger immune to NUnit capturing stdout and stderr to workaround https://github.com/nunit/nunit/issues/1406.
             GrpcEnvironment.SetLogger(new TextWriterLogger(Console.Error));
-#if DOTNET5_4
+#if NETSTANDARD1_5
             return new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args, new ExtendedTextWrapper(Console.Out), Console.In);
 #else
             return new AutoRun().Execute(args);
diff --git a/src/csharp/Grpc.Examples.Tests/project.json b/src/csharp/Grpc.Examples.Tests/project.json
new file mode 100644
index 0000000000..cc518eb6ff
--- /dev/null
+++ b/src/csharp/Grpc.Examples.Tests/project.json
@@ -0,0 +1,69 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.Examples": {
+      "target": "project"
+    },
+    "NUnit": "3.2.0",
+    "NUnitLite": "3.2.0-*"
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj
index 30170ab03c..3dfa84e896 100644
--- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj
+++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj
@@ -69,6 +69,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.Examples.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.project.json b/src/csharp/Grpc.Examples/Grpc.Examples.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.Examples/Grpc.Examples.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.xproj b/src/csharp/Grpc.Examples/Grpc.Examples.xproj
new file mode 100644
index 0000000000..d1d7e6d981
--- /dev/null
+++ b/src/csharp/Grpc.Examples/Grpc.Examples.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>c77b792d-fc78-4ce2-9522-b40b0803c636</ProjectGuid>
+    <RootNamespace>Grpc.Examples</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.Examples/MathExamples.cs b/src/csharp/Grpc.Examples/MathExamples.cs
index 6075420974..d260830b94 100644
--- a/src/csharp/Grpc.Examples/MathExamples.cs
+++ b/src/csharp/Grpc.Examples/MathExamples.cs
@@ -32,6 +32,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Grpc.Core;
 using Grpc.Core.Utils;
 
 namespace Math
@@ -109,5 +110,42 @@ namespace Math
             DivReply result = await client.DivAsync(new DivArgs { Dividend = sum.Num_, Divisor = numbers.Count });
             Console.WriteLine("Avg Result: " + result);
         }
+
+        /// <summary>
+        /// Shows how to handle a call ending with non-OK status.
+        /// </summary>
+        public static async Task HandleErrorExample(Math.MathClient client)
+        {
+            try
+            {
+                 DivReply result = await client.DivAsync(new DivArgs { Dividend = 5, Divisor = 0 });
+            }
+            catch (RpcException ex)
+            {
+                Console.WriteLine(string.Format("RPC ended with status {0}", ex.Status));
+            }
+        }
+
+        /// <summary>
+        /// Shows how to send request headers and how to access response headers
+        /// and response trailers.
+        /// </summary>
+        public static async Task MetadataExample(Math.MathClient client)
+        {
+            var requestHeaders = new Metadata
+            {
+                { "custom-header", "custom-value" }
+            };
+
+            var call = client.DivAsync(new DivArgs { Dividend = 5, Divisor = 0 }, requestHeaders);
+
+            // Get response headers
+            Metadata responseHeaders = await call.ResponseHeadersAsync;
+
+            var result = await call;
+
+            // Get response trailers after the call has finished.
+            Metadata responseTrailers = call.GetTrailers();
+        }
     }
 }
diff --git a/src/csharp/Grpc.Examples/MathGrpc.cs b/src/csharp/Grpc.Examples/MathGrpc.cs
index d700a18778..25abc51419 100644
--- a/src/csharp/Grpc.Examples/MathGrpc.cs
+++ b/src/csharp/Grpc.Examples/MathGrpc.cs
@@ -81,103 +81,12 @@ namespace Math {
       get { return global::Math.MathReflection.Descriptor.Services[0]; }
     }
 
-    /// <summary>Client for Math</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IMathClient
-    {
-      /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
-      /// </summary>
-      global::Math.DivReply Div(global::Math.DivArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
-      /// </summary>
-      global::Math.DivReply Div(global::Math.DivArgs request, CallOptions options);
-      /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
-      /// </summary>
-      AsyncUnaryCall<global::Math.DivReply> DivAsync(global::Math.DivArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
-      /// </summary>
-      AsyncUnaryCall<global::Math.DivReply> DivAsync(global::Math.DivArgs request, CallOptions options);
-      /// <summary>
-      ///  DivMany accepts an arbitrary number of division args from the client stream
-      ///  and sends back the results in the reply stream.  The stream continues until
-      ///  the client closes its end; the server does the same after sending all the
-      ///  replies.  The stream ends immediately if either end aborts.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Math.DivArgs, global::Math.DivReply> DivMany(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  DivMany accepts an arbitrary number of division args from the client stream
-      ///  and sends back the results in the reply stream.  The stream continues until
-      ///  the client closes its end; the server does the same after sending all the
-      ///  replies.  The stream ends immediately if either end aborts.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Math.DivArgs, global::Math.DivReply> DivMany(CallOptions options);
-      /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
-      ///  generates up to limit numbers; otherwise it continues until the call is
-      ///  canceled.  Unlike Fib above, Fib has no final FibReply.
-      /// </summary>
-      AsyncServerStreamingCall<global::Math.Num> Fib(global::Math.FibArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
-      ///  generates up to limit numbers; otherwise it continues until the call is
-      ///  canceled.  Unlike Fib above, Fib has no final FibReply.
-      /// </summary>
-      AsyncServerStreamingCall<global::Math.Num> Fib(global::Math.FibArgs request, CallOptions options);
-      /// <summary>
-      ///  Sum sums a stream of numbers, returning the final result once the stream
-      ///  is closed.
-      /// </summary>
-      AsyncClientStreamingCall<global::Math.Num, global::Math.Num> Sum(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Sum sums a stream of numbers, returning the final result once the stream
-      ///  is closed.
-      /// </summary>
-      AsyncClientStreamingCall<global::Math.Num, global::Math.Num> Sum(CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of Math</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IMath
-    {
-      /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Math.DivReply> Div(global::Math.DivArgs request, ServerCallContext context);
-      /// <summary>
-      ///  DivMany accepts an arbitrary number of division args from the client stream
-      ///  and sends back the results in the reply stream.  The stream continues until
-      ///  the client closes its end; the server does the same after sending all the
-      ///  replies.  The stream ends immediately if either end aborts.
-      /// </summary>
-      global::System.Threading.Tasks.Task DivMany(IAsyncStreamReader<global::Math.DivArgs> requestStream, IServerStreamWriter<global::Math.DivReply> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
-      ///  generates up to limit numbers; otherwise it continues until the call is
-      ///  canceled.  Unlike Fib above, Fib has no final FibReply.
-      /// </summary>
-      global::System.Threading.Tasks.Task Fib(global::Math.FibArgs request, IServerStreamWriter<global::Math.Num> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  Sum sums a stream of numbers, returning the final result once the stream
-      ///  is closed.
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Math.Num> Sum(IAsyncStreamReader<global::Math.Num> requestStream, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of Math</summary>
     public abstract class MathBase
     {
       /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
+      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      ///  and remainder.
       /// </summary>
       public virtual global::System.Threading.Tasks.Task<global::Math.DivReply> Div(global::Math.DivArgs request, ServerCallContext context)
       {
@@ -196,7 +105,7 @@ namespace Math {
       }
 
       /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+      ///  Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
       ///  generates up to limit numbers; otherwise it continues until the call is
       ///  canceled.  Unlike Fib above, Fib has no final FibReply.
       /// </summary>
@@ -217,52 +126,55 @@ namespace Math {
     }
 
     /// <summary>Client for Math</summary>
-    #pragma warning disable 0618
-    public class MathClient : ClientBase<MathClient>, IMathClient
-    #pragma warning restore 0618
+    public class MathClient : ClientBase<MathClient>
     {
+      /// <summary>Creates a new client for Math</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public MathClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for Math that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public MathClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected MathClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected MathClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
 
       /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
+      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      ///  and remainder.
       /// </summary>
       public virtual global::Math.DivReply Div(global::Math.DivArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return Div(request, new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
+      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      ///  and remainder.
       /// </summary>
       public virtual global::Math.DivReply Div(global::Math.DivArgs request, CallOptions options)
       {
         return CallInvoker.BlockingUnaryCall(__Method_Div, null, options, request);
       }
       /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
+      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      ///  and remainder.
       /// </summary>
       public virtual AsyncUnaryCall<global::Math.DivReply> DivAsync(global::Math.DivArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return DivAsync(request, new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Div divides args.dividend by args.divisor and returns the quotient and
-      ///  remainder.
+      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      ///  and remainder.
       /// </summary>
       public virtual AsyncUnaryCall<global::Math.DivReply> DivAsync(global::Math.DivArgs request, CallOptions options)
       {
@@ -289,7 +201,7 @@ namespace Math {
         return CallInvoker.AsyncDuplexStreamingCall(__Method_DivMany, null, options);
       }
       /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+      ///  Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
       ///  generates up to limit numbers; otherwise it continues until the call is
       ///  canceled.  Unlike Fib above, Fib has no final FibReply.
       /// </summary>
@@ -298,7 +210,7 @@ namespace Math {
         return Fib(request, new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+      ///  Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
       ///  generates up to limit numbers; otherwise it continues until the call is
       ///  canceled.  Unlike Fib above, Fib has no final FibReply.
       /// </summary>
@@ -328,30 +240,10 @@ namespace Math {
       }
     }
 
-    /// <summary>Creates a new client for Math</summary>
-    public static MathClient NewClient(Channel channel)
-    {
-      return new MathClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IMath serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_Div, serviceImpl.Div)
-          .AddMethod(__Method_DivMany, serviceImpl.DivMany)
-          .AddMethod(__Method_Fib, serviceImpl.Fib)
-          .AddMethod(__Method_Sum, serviceImpl.Sum).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(MathBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_Div, serviceImpl.Div)
           .AddMethod(__Method_DivMany, serviceImpl.DivMany)
           .AddMethod(__Method_Fib, serviceImpl.Fib)
diff --git a/src/csharp/Grpc.Examples/MathServiceImpl.cs b/src/csharp/Grpc.Examples/MathServiceImpl.cs
index 79c56e57a8..a28020f62f 100644
--- a/src/csharp/Grpc.Examples/MathServiceImpl.cs
+++ b/src/csharp/Grpc.Examples/MathServiceImpl.cs
@@ -52,23 +52,15 @@ namespace Math
 
         public override async Task Fib(FibArgs request, IServerStreamWriter<Num> responseStream, ServerCallContext context)
         {
-            if (request.Limit <= 0)
-            {
-                // keep streaming the sequence until cancelled.
-                IEnumerator<Num> fibEnumerator = FibInternal(long.MaxValue).GetEnumerator();
-                while (!context.CancellationToken.IsCancellationRequested && fibEnumerator.MoveNext())
-                {
-                    await responseStream.WriteAsync(fibEnumerator.Current);
-                    await Task.Delay(100);
-                }
-            }
+            var limit = request.Limit > 0 ? request.Limit : long.MaxValue;
+            var fibEnumerator = FibInternal(limit).GetEnumerator();
 
-            if (request.Limit > 0)
+            // Keep streaming the sequence until the call is cancelled.
+            // Use CancellationToken from ServerCallContext to detect the cancellation.
+            while (!context.CancellationToken.IsCancellationRequested && fibEnumerator.MoveNext())
             {
-                foreach (var num in FibInternal(request.Limit))
-                {
-                    await responseStream.WriteAsync(num);
-                }
+                await responseStream.WriteAsync(fibEnumerator.Current);
+                await Task.Delay(100);
             }
         }
 
@@ -89,6 +81,13 @@ namespace Math
 
         static DivReply DivInternal(DivArgs args)
         {
+            if (args.Divisor == 0)
+            {
+                // One can finish the RPC with non-ok status by throwing RpcException instance.
+                // Alternatively, resulting status can be set using ServerCallContext.Status
+                throw new RpcException(new Status(StatusCode.InvalidArgument, "Division by zero"));
+            }
+
             long quotient = args.Dividend / args.Divisor;
             long remainder = args.Dividend % args.Divisor;
             return new DivReply { Quotient = quotient, Remainder = remainder };
diff --git a/src/csharp/Grpc.Examples/project.json b/src/csharp/Grpc.Examples/project.json
new file mode 100644
index 0000000000..7d3f4dcbb1
--- /dev/null
+++ b/src/csharp/Grpc.Examples/project.json
@@ -0,0 +1,27 @@
+{
+  "buildOptions": {
+  },
+
+  "dependencies": {
+    "Grpc.Core": {
+      "target": "project"
+    },
+    "Google.Protobuf": "3.0.0-beta3"
+  },
+  "frameworks": {
+    "net45": {
+      "frameworkAssemblies": {
+        "System.Runtime": "",
+        "System.IO": ""
+      }
+    },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
index a5ee4fdb46..aefacfbcc0 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
+++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
@@ -74,6 +74,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.HealthCheck.Tests.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.project.json b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.xproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.xproj
new file mode 100644
index 0000000000..724c5b2a16
--- /dev/null
+++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>43dafac6-5343-4621-960e-a8a977ea3f0b</ProjectGuid>
+    <RootNamespace>Grpc.HealthCheck.Tests</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs
index 070674bae9..25a58fb386 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs
+++ b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs
@@ -65,7 +65,7 @@ namespace Grpc.HealthCheck.Tests
             server.Start();
             channel = new Channel(Host, server.Ports.Single().BoundPort, ChannelCredentials.Insecure);
 
-            client = Grpc.Health.V1.Health.NewClient(channel);
+            client = new Grpc.Health.V1.Health.HealthClient(channel);
         }
 
         [TestFixtureTearDown]
diff --git a/src/csharp/Grpc.HealthCheck.Tests/NUnitMain.cs b/src/csharp/Grpc.HealthCheck.Tests/NUnitMain.cs
index 0820523f35..44634671ce 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/NUnitMain.cs
+++ b/src/csharp/Grpc.HealthCheck.Tests/NUnitMain.cs
@@ -49,7 +49,7 @@ namespace Grpc.HealthCheck.Tests
         {
             // Make logger immune to NUnit capturing stdout and stderr to workaround https://github.com/nunit/nunit/issues/1406.
             GrpcEnvironment.SetLogger(new TextWriterLogger(Console.Error));
-#if DOTNET5_4
+#if NETSTANDARD1_5
             return new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args, new ExtendedTextWrapper(Console.Out), Console.In);
 #else
             return new AutoRun().Execute(args);
diff --git a/src/csharp/Grpc.HealthCheck.Tests/project.json b/src/csharp/Grpc.HealthCheck.Tests/project.json
new file mode 100644
index 0000000000..fbf8d92f04
--- /dev/null
+++ b/src/csharp/Grpc.HealthCheck.Tests/project.json
@@ -0,0 +1,69 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.HealthCheck": {
+      "target": "project"
+    },
+    "NUnit": "3.2.0",
+    "NUnitLite": "3.2.0-*"
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
index 2697b74f59..7db8b2d38e 100644
--- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
+++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
@@ -65,6 +65,7 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="Grpc.HealthCheck.nuspec" />
+    <None Include="Grpc.HealthCheck.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.project.json b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.xproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.xproj
new file mode 100644
index 0000000000..5806a7af97
--- /dev/null
+++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>3be4ad0b-2bf0-4d68-b625-f6018ef0dcfa</ProjectGuid>
+    <RootNamespace>Grpc.HealthCheck</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs
index 51c6a39b1d..43eea0f22f 100644
--- a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs
+++ b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs
@@ -58,23 +58,6 @@ namespace Grpc.Health.V1 {
       get { return global::Grpc.Health.V1.HealthReflection.Descriptor.Services[0]; }
     }
 
-    /// <summary>Client for Health</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IHealthClient
-    {
-      global::Grpc.Health.V1.HealthCheckResponse Check(global::Grpc.Health.V1.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      global::Grpc.Health.V1.HealthCheckResponse Check(global::Grpc.Health.V1.HealthCheckRequest request, CallOptions options);
-      AsyncUnaryCall<global::Grpc.Health.V1.HealthCheckResponse> CheckAsync(global::Grpc.Health.V1.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      AsyncUnaryCall<global::Grpc.Health.V1.HealthCheckResponse> CheckAsync(global::Grpc.Health.V1.HealthCheckRequest request, CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of Health</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IHealth
-    {
-      global::System.Threading.Tasks.Task<global::Grpc.Health.V1.HealthCheckResponse> Check(global::Grpc.Health.V1.HealthCheckRequest request, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of Health</summary>
     public abstract class HealthBase
     {
@@ -86,21 +69,24 @@ namespace Grpc.Health.V1 {
     }
 
     /// <summary>Client for Health</summary>
-    #pragma warning disable 0618
-    public class HealthClient : ClientBase<HealthClient>, IHealthClient
-    #pragma warning restore 0618
+    public class HealthClient : ClientBase<HealthClient>
     {
+      /// <summary>Creates a new client for Health</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public HealthClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for Health that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public HealthClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected HealthClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected HealthClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -127,27 +113,10 @@ namespace Grpc.Health.V1 {
       }
     }
 
-    /// <summary>Creates a new client for Health</summary>
-    public static HealthClient NewClient(Channel channel)
-    {
-      return new HealthClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IHealth serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_Check, serviceImpl.Check).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(HealthBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_Check, serviceImpl.Check).Build();
     }
 
diff --git a/src/csharp/Grpc.HealthCheck/project.json b/src/csharp/Grpc.HealthCheck/project.json
new file mode 100644
index 0000000000..d9daef720f
--- /dev/null
+++ b/src/csharp/Grpc.HealthCheck/project.json
@@ -0,0 +1,44 @@
+{
+  "version": "0.16.0-dev",
+  "title": "gRPC C# Healthchecking",
+  "authors": [ "Google Inc." ],
+  "copyright": "Copyright 2015, Google Inc.",
+  "packOptions": {
+    "summary": "Implementation of gRPC health service",
+    "description": "Example implementation of grpc.health.v1 service that can be used for health-checking.",
+    "owners": [ "grpc-packages" ],
+    "licenseUrl": "https://github.com/grpc/grpc/blob/master/LICENSE",
+    "projectUrl": "https://github.com/grpc/grpc",
+    "requireLicenseAcceptance": false,
+    "tags": [ "gRPC health check" ]
+  },
+  "buildOptions": {
+    "define": [ "SIGNED" ],
+    "keyFile": "../keys/Grpc.snk",
+    "publicSign": true,
+    "xmlDoc": true,
+    "compile": {
+      "includeFiles": [ "../Grpc.Core/Version.cs" ]
+    }
+  },
+  "dependencies": {
+    "Grpc.Core": "0.16.0-dev",
+    "Google.Protobuf": "3.0.0-beta3"
+  },
+  "frameworks": {
+    "net45": {
+      "frameworkAssemblies": {
+        "System.Runtime": "",
+        "System.IO": ""
+      }
+    },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj
index 339a754c02..91fb3ce5bc 100644
--- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj
+++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj
@@ -83,6 +83,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.IntegrationTesting.Client.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.project.json b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.xproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.xproj
new file mode 100644
index 0000000000..7f456cfaef
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>48ea5bbe-70e2-4198-869d-d7e59c45f30d</ProjectGuid>
+    <RootNamespace>Grpc.IntegrationTesting.Client</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.Client/project.json b/src/csharp/Grpc.IntegrationTesting.Client/project.json
new file mode 100644
index 0000000000..4a2846feea
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.Client/project.json
@@ -0,0 +1,70 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.IntegrationTesting": {
+      "target": "project"
+    }
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45",
+        "net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj
index 0dc2751b04..dda26a6892 100644
--- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj
+++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj
@@ -59,5 +59,6 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />
+    <None Include="Grpc.IntegrationTesting.QpsWorker.project.json" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.project.json b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.xproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.xproj
new file mode 100644
index 0000000000..15bec443d6
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>661b70d7-f56a-46e0-9b81-6227b591b5e7</ProjectGuid>
+    <RootNamespace>Grpc.IntegrationTesting.QpsWorker</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json
new file mode 100644
index 0000000000..4a2846feea
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json
@@ -0,0 +1,70 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.IntegrationTesting": {
+      "target": "project"
+    }
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45",
+        "net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj
index 27a5650308..f73d99dbd1 100644
--- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj
+++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj
@@ -83,6 +83,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.IntegrationTesting.Server.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.project.json b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.xproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.xproj
new file mode 100644
index 0000000000..689eb0b842
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>881f7ad1-a84e-47a2-9402-115c63c4031e</ProjectGuid>
+    <RootNamespace>Grpc.IntegrationTesting.Server</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.Server/project.json b/src/csharp/Grpc.IntegrationTesting.Server/project.json
new file mode 100644
index 0000000000..4a2846feea
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.Server/project.json
@@ -0,0 +1,70 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.IntegrationTesting": {
+      "target": "project"
+    }
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45",
+        "net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj
index d6eba74289..8bd3d78913 100644
--- a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj
+++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -57,4 +57,7 @@
       <Name>Grpc.IntegrationTesting</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <None Include="Grpc.IntegrationTesting.StressClient.project.json" />
+  </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.project.json b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.xproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.xproj
new file mode 100644
index 0000000000..2f4fdcbb47
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.xproj
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>0ebc910b-8867-4d3e-8686-91f34183d839</ProjectGuid>
+    <RootNamespace>Grpc.IntegrationTesting.StressClient</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json
new file mode 100644
index 0000000000..4a2846feea
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json
@@ -0,0 +1,70 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.IntegrationTesting": {
+      "target": "project"
+    }
+  },
+  "frameworks": {
+    "net45": { },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45",
+        "net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
index 79a88f3f3c..b9c0fe6d0d 100644
--- a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
@@ -32,6 +32,7 @@
 #endregion
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -41,7 +42,9 @@ using System.Threading;
 using System.Threading.Tasks;
 using Google.Protobuf;
 using Grpc.Core;
+using Grpc.Core.Internal;
 using Grpc.Core.Logging;
+using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 using Grpc.Testing;
@@ -55,6 +58,15 @@ namespace Grpc.IntegrationTesting
     {
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<ClientRunners>();
 
+        // Profilers to use for clients.
+        static readonly BlockingCollection<BasicProfiler> profilers = new BlockingCollection<BasicProfiler>();
+
+        internal static void AddProfiler(BasicProfiler profiler)
+        {
+            GrpcPreconditions.CheckNotNull(profiler);
+            profilers.Add(profiler);
+        }
+
         /// <summary>
         /// Creates a started client runner.
         /// </summary>
@@ -83,7 +95,8 @@ namespace Grpc.IntegrationTesting
                 config.OutstandingRpcsPerChannel,
                 config.LoadParams,
                 config.PayloadConfig,
-                config.HistogramParams);
+                config.HistogramParams,
+                () => GetNextProfiler());
         }
 
         private static List<Channel> CreateChannels(int clientChannels, IEnumerable<string> serverTargets, SecurityParams securityParams)
@@ -110,9 +123,16 @@ namespace Grpc.IntegrationTesting
             }
             return result;
         }
+
+        private static BasicProfiler GetNextProfiler()
+        {
+            BasicProfiler result = null;
+            profilers.TryTake(out result);
+            return result;
+        }
     }
 
-    public class ClientRunnerImpl : IClientRunner
+    internal class ClientRunnerImpl : IClientRunner
     {
         const double SecondsToNanos = 1e9;
 
@@ -125,8 +145,9 @@ namespace Grpc.IntegrationTesting
         readonly List<Task> runnerTasks;
         readonly CancellationTokenSource stoppedCts = new CancellationTokenSource();
         readonly WallClockStopwatch wallClockStopwatch = new WallClockStopwatch();
+        readonly AtomicCounter statsResetCount = new AtomicCounter();
         
-        public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams)
+        public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams, Func<BasicProfiler> profilerFactory)
         {
             GrpcPreconditions.CheckArgument(outstandingRpcsPerChannel > 0, "outstandingRpcsPerChannel");
             GrpcPreconditions.CheckNotNull(histogramParams, "histogramParams");
@@ -142,8 +163,8 @@ namespace Grpc.IntegrationTesting
                 for (int i = 0; i < outstandingRpcsPerChannel; i++)
                 {
                     var timer = CreateTimer(loadParams, 1.0 / this.channels.Count / outstandingRpcsPerChannel);
-                    var threadBody = GetThreadBody(channel, timer);
-                    this.runnerTasks.Add(Task.Factory.StartNew(threadBody, TaskCreationOptions.LongRunning));
+                    var optionalProfiler = profilerFactory();
+                    this.runnerTasks.Add(RunClientAsync(channel, timer, optionalProfiler));
                 }
             }
         }
@@ -153,6 +174,11 @@ namespace Grpc.IntegrationTesting
             var histogramData = histogram.GetSnapshot(reset);
             var secondsElapsed = wallClockStopwatch.GetElapsedSnapshot(reset).TotalSeconds;
 
+            if (reset)
+            {
+                statsResetCount.Increment();
+            }
+
             // TODO: populate user time and system time
             return new ClientStats
             {
@@ -176,14 +202,28 @@ namespace Grpc.IntegrationTesting
             }
         }
 
-        private void RunUnary(Channel channel, IInterarrivalTimer timer)
+        private void RunUnary(Channel channel, IInterarrivalTimer timer, BasicProfiler optionalProfiler)
         {
-            var client = BenchmarkService.NewClient(channel);
+            if (optionalProfiler != null)
+            {
+                Profilers.SetForCurrentThread(optionalProfiler);
+            }
+
+            bool profilerReset = false;
+
+            var client = new BenchmarkService.BenchmarkServiceClient(channel);
             var request = CreateSimpleRequest();
             var stopwatch = new Stopwatch();
 
             while (!stoppedCts.Token.IsCancellationRequested)
             {
+                // after the first stats reset, also reset the profiler.
+                if (optionalProfiler != null && !profilerReset && statsResetCount.Count > 0)
+                {
+                    optionalProfiler.Reset();
+                    profilerReset = true;
+                }
+
                 stopwatch.Restart();
                 client.UnaryCall(request);
                 stopwatch.Stop();
@@ -197,7 +237,7 @@ namespace Grpc.IntegrationTesting
 
         private async Task RunUnaryAsync(Channel channel, IInterarrivalTimer timer)
         {
-            var client = BenchmarkService.NewClient(channel);
+            var client = new BenchmarkService.BenchmarkServiceClient(channel);
             var request = CreateSimpleRequest();
             var stopwatch = new Stopwatch();
 
@@ -216,7 +256,7 @@ namespace Grpc.IntegrationTesting
 
         private async Task RunStreamingPingPongAsync(Channel channel, IInterarrivalTimer timer)
         {
-            var client = BenchmarkService.NewClient(channel);
+            var client = new BenchmarkService.BenchmarkServiceClient(channel);
             var request = CreateSimpleRequest();
             var stopwatch = new Stopwatch();
 
@@ -269,38 +309,30 @@ namespace Grpc.IntegrationTesting
             }
         }
 
-        private Action GetThreadBody(Channel channel, IInterarrivalTimer timer)
+        private Task RunClientAsync(Channel channel, IInterarrivalTimer timer, BasicProfiler optionalProfiler)
         {
             if (payloadConfig.PayloadCase == PayloadConfig.PayloadOneofCase.BytebufParams)
             {
                 GrpcPreconditions.CheckArgument(clientType == ClientType.AsyncClient, "Generic client only supports async API");
                 GrpcPreconditions.CheckArgument(rpcType == RpcType.Streaming, "Generic client only supports streaming calls");
-                return () =>
-                {
-                    RunGenericStreamingAsync(channel, timer).Wait();
-                };
+                return RunGenericStreamingAsync(channel, timer);
             }
 
             GrpcPreconditions.CheckNotNull(payloadConfig.SimpleParams);
             if (clientType == ClientType.SyncClient)
             {
                 GrpcPreconditions.CheckArgument(rpcType == RpcType.Unary, "Sync client can only be used for Unary calls in C#");
-                return () => RunUnary(channel, timer);
+                // create a dedicated thread for the synchronous client
+                return Task.Factory.StartNew(() => RunUnary(channel, timer, optionalProfiler), TaskCreationOptions.LongRunning);
             }
             else if (clientType == ClientType.AsyncClient)
             {
                 switch (rpcType)
                 {
                     case RpcType.Unary:
-                        return () =>
-                        {
-                            RunUnaryAsync(channel, timer).Wait();
-                        };
+                        return RunUnaryAsync(channel, timer);
                     case RpcType.Streaming:
-                        return () =>
-                        {
-                            RunStreamingPingPongAsync(channel, timer).Wait();
-                        };
+                        return RunStreamingPingPongAsync(channel, timer);
                 }
             }
             throw new ArgumentException("Unsupported configuration.");
diff --git a/src/csharp/Grpc.IntegrationTesting/GeneratedClientTest.cs b/src/csharp/Grpc.IntegrationTesting/GeneratedClientTest.cs
index 37786b6c30..eb7b55a286 100644
--- a/src/csharp/Grpc.IntegrationTesting/GeneratedClientTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/GeneratedClientTest.cs
@@ -40,7 +40,6 @@ using System.Threading.Tasks;
 using Grpc.Core;
 using Grpc.Core.Utils;
 using Grpc.Testing;
-using Moq;
 using NUnit.Framework;
 
 namespace Grpc.IntegrationTesting
@@ -49,14 +48,16 @@ namespace Grpc.IntegrationTesting
     {
         TestService.TestServiceClient unimplementedClient = new UnimplementedTestServiceClient();
 
+        // TODO: replace Moq by some mocking library with CoreCLR support.
+#if !NETSTANDARD1_5
         [Test]
         public void ExpandedParamOverloadCanBeMocked()
         {
             var expected = new SimpleResponse();
 
-            var mockClient = new Mock<TestService.TestServiceClient>();
+            var mockClient = new Moq.Mock<TestService.TestServiceClient>();
             // mocking is relatively clumsy because one needs to specify value for all the optional params.
-            mockClient.Setup(m => m.UnaryCall(It.IsAny<SimpleRequest>(), null, null, CancellationToken.None)).Returns(expected);
+            mockClient.Setup(m => m.UnaryCall(Moq.It.IsAny<SimpleRequest>(), null, null, CancellationToken.None)).Returns(expected);
 
             Assert.AreSame(expected, mockClient.Object.UnaryCall(new SimpleRequest()));
         }
@@ -66,11 +67,12 @@ namespace Grpc.IntegrationTesting
         {
             var expected = new SimpleResponse();
 
-            var mockClient = new Mock<TestService.TestServiceClient>();
-            mockClient.Setup(m => m.UnaryCall(It.IsAny<SimpleRequest>(), It.IsAny<CallOptions>())).Returns(expected);
+            var mockClient = new Moq.Mock<TestService.TestServiceClient>();
+            mockClient.Setup(m => m.UnaryCall(Moq.It.IsAny<SimpleRequest>(), Moq.It.IsAny<CallOptions>())).Returns(expected);
 
             Assert.AreSame(expected, mockClient.Object.UnaryCall(new SimpleRequest(), new CallOptions()));
         }
+#endif
 
         [Test]
         public void DefaultMethodStubThrows_UnaryCall()
diff --git a/src/csharp/Grpc.IntegrationTesting/GeneratedServiceBaseTest.cs b/src/csharp/Grpc.IntegrationTesting/GeneratedServiceBaseTest.cs
index 5fd0e14e78..4216dc1d6b 100644
--- a/src/csharp/Grpc.IntegrationTesting/GeneratedServiceBaseTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/GeneratedServiceBaseTest.cs
@@ -40,7 +40,6 @@ using System.Threading.Tasks;
 using Grpc.Core;
 using Grpc.Core.Utils;
 using Grpc.Testing;
-using Moq;
 using NUnit.Framework;
 
 namespace Grpc.IntegrationTesting
@@ -62,7 +61,7 @@ namespace Grpc.IntegrationTesting
             };
             server.Start();
             channel = new Channel(Host, server.Ports.Single().BoundPort, ChannelCredentials.Insecure);
-            client = TestService.NewClient(channel);
+            client = new TestService.TestServiceClient(channel);
         }
 
         [TearDown]
diff --git a/src/csharp/Grpc.IntegrationTesting/GenericService.cs b/src/csharp/Grpc.IntegrationTesting/GenericService.cs
index c6128264ac..53fa1ee5f6 100644
--- a/src/csharp/Grpc.IntegrationTesting/GenericService.cs
+++ b/src/csharp/Grpc.IntegrationTesting/GenericService.cs
@@ -64,7 +64,7 @@ namespace Grpc.IntegrationTesting
 
         public static ServerServiceDefinition BindHandler(DuplexStreamingServerMethod<byte[], byte[]> handler)
         {
-            return ServerServiceDefinition.CreateBuilder(StreamingCallMethod.ServiceName)
+            return ServerServiceDefinition.CreateBuilder()
                 .AddMethod(StreamingCallMethod, handler).Build();
         }
     }
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
index 0089049408..3a0764230d 100644
--- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
@@ -129,6 +129,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Grpc.IntegrationTesting.project.json" />
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.project.json b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.project.json
new file mode 100644
index 0000000000..c2f5bcb163
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.project.json
@@ -0,0 +1,8 @@
+{
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win": { }
+  }
+}
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.xproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.xproj
new file mode 100644
index 0000000000..357300ecb9
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.xproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>20354386-3e71-4046-a269-3bc2a06f3ec8</ProjectGuid>
+    <RootNamespace>Grpc.IntegrationTesting</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
index 1541cfd7bb..e27fe5b3d8 100644
--- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
+++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
@@ -145,16 +145,26 @@ namespace Grpc.IntegrationTesting
 
             if (options.TestCase == "jwt_token_creds")
             {
+#if !NETSTANDARD1_5
                 var googleCredential = await GoogleCredential.GetApplicationDefaultAsync();
                 Assert.IsTrue(googleCredential.IsCreateScopedRequired);
                 credentials = ChannelCredentials.Create(credentials, googleCredential.ToCallCredentials());
+#else
+                // TODO(jtattermusch): implement this
+                throw new NotImplementedException("Not supported on CoreCLR yet");
+#endif
             }
 
             if (options.TestCase == "compute_engine_creds")
             {
+#if !NETSTANDARD1_5
                 var googleCredential = await GoogleCredential.GetApplicationDefaultAsync();
                 Assert.IsFalse(googleCredential.IsCreateScopedRequired);
                 credentials = ChannelCredentials.Create(credentials, googleCredential.ToCallCredentials());
+#else
+                // TODO(jtattermusch): implement this
+                throw new NotImplementedException("Not supported on CoreCLR yet");
+#endif
             }
             return credentials;
         }
@@ -212,6 +222,12 @@ namespace Grpc.IntegrationTesting
                 case "unimplemented_method":
                     RunUnimplementedMethod(new UnimplementedService.UnimplementedServiceClient(channel));
                     break;
+                case "client_compressed_unary":
+                    RunClientCompressedUnary(client);
+                    break;
+                case "client_compressed_streaming":
+                    await RunClientCompressedStreamingAsync(client);
+                    break;
                 default:
                     throw new ArgumentException("Unknown test case " + options.TestCase);
             }
@@ -230,13 +246,11 @@ namespace Grpc.IntegrationTesting
             Console.WriteLine("running large_unary");
             var request = new SimpleRequest
             {
-                ResponseType = PayloadType.Compressable,
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828)
             };
             var response = client.UnaryCall(request);
 
-            Assert.AreEqual(PayloadType.Compressable, response.Payload.Type);
             Assert.AreEqual(314159, response.Payload.Body.Length);
             Console.WriteLine("Passed!");
         }
@@ -245,7 +259,7 @@ namespace Grpc.IntegrationTesting
         {
             Console.WriteLine("running client_streaming");
 
-            var bodySizes = new List<int> { 27182, 8, 1828, 45904 }.ConvertAll((size) => new StreamingInputCallRequest { Payload = CreateZerosPayload(size) });
+            var bodySizes = new List<int> { 27182, 8, 1828, 45904 }.Select((size) => new StreamingInputCallRequest { Payload = CreateZerosPayload(size) });
 
             using (var call = client.StreamingInputCall())
             {
@@ -265,18 +279,13 @@ namespace Grpc.IntegrationTesting
 
             var request = new StreamingOutputCallRequest
             {
-                ResponseType = PayloadType.Compressable,
-                ResponseParameters = { bodySizes.ConvertAll((size) => new ResponseParameters { Size = size }) }
+                ResponseParameters = { bodySizes.Select((size) => new ResponseParameters { Size = size }) }
             };
 
             using (var call = client.StreamingOutputCall(request))
             {
                 var responseList = await call.ResponseStream.ToListAsync();
-                foreach (var res in responseList)
-                {
-                    Assert.AreEqual(PayloadType.Compressable, res.Payload.Type);
-                }
-                CollectionAssert.AreEqual(bodySizes, responseList.ConvertAll((item) => item.Payload.Body.Length));
+                CollectionAssert.AreEqual(bodySizes, responseList.Select((item) => item.Payload.Body.Length));
             }
             Console.WriteLine("Passed!");
         }
@@ -289,46 +298,38 @@ namespace Grpc.IntegrationTesting
             {
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 31415 } },
                     Payload = CreateZerosPayload(27182)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(31415, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 9 } },
                     Payload = CreateZerosPayload(8)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(9, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 2653 } },
                     Payload = CreateZerosPayload(1828)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(2653, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 58979 } },
                     Payload = CreateZerosPayload(45904)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(58979, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.CompleteAsync();
@@ -357,7 +358,6 @@ namespace Grpc.IntegrationTesting
 
             var request = new SimpleRequest
             {
-                ResponseType = PayloadType.Compressable,
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828),
                 FillUsername = true,
@@ -367,7 +367,6 @@ namespace Grpc.IntegrationTesting
             // not setting credentials here because they were set on channel already
             var response = client.UnaryCall(request);
 
-            Assert.AreEqual(PayloadType.Compressable, response.Payload.Type);
             Assert.AreEqual(314159, response.Payload.Body.Length);
             Assert.False(string.IsNullOrEmpty(response.OauthScope));
             Assert.True(oauthScope.Contains(response.OauthScope));
@@ -381,7 +380,6 @@ namespace Grpc.IntegrationTesting
            
             var request = new SimpleRequest
             {
-                ResponseType = PayloadType.Compressable,
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828),
                 FillUsername = true,
@@ -390,7 +388,6 @@ namespace Grpc.IntegrationTesting
             // not setting credentials here because they were set on channel already
             var response = client.UnaryCall(request);
 
-            Assert.AreEqual(PayloadType.Compressable, response.Payload.Type);
             Assert.AreEqual(314159, response.Payload.Body.Length);
             Assert.AreEqual(GetEmailFromServiceAccountFile(), response.Username);
             Console.WriteLine("Passed!");
@@ -398,6 +395,7 @@ namespace Grpc.IntegrationTesting
 
         public static async Task RunOAuth2AuthTokenAsync(TestService.TestServiceClient client, string oauthScope)
         {
+#if !NETSTANDARD1_5
             Console.WriteLine("running oauth2_auth_token");
             ITokenAccess credential = (await GoogleCredential.GetApplicationDefaultAsync()).CreateScoped(new[] { oauthScope });
             string oauth2Token = await credential.GetAccessTokenForRequestAsync();
@@ -415,10 +413,15 @@ namespace Grpc.IntegrationTesting
             Assert.True(oauthScope.Contains(response.OauthScope));
             Assert.AreEqual(GetEmailFromServiceAccountFile(), response.Username);
             Console.WriteLine("Passed!");
+#else
+            // TODO(jtattermusch): implement this
+            throw new NotImplementedException("Not supported on CoreCLR yet");
+#endif
         }
 
         public static async Task RunPerRpcCredsAsync(TestService.TestServiceClient client, string oauthScope)
         {
+#if !NETSTANDARD1_5
             Console.WriteLine("running per_rpc_creds");
             ITokenAccess googleCredential = await GoogleCredential.GetApplicationDefaultAsync();
 
@@ -432,6 +435,10 @@ namespace Grpc.IntegrationTesting
 
             Assert.AreEqual(GetEmailFromServiceAccountFile(), response.Username);
             Console.WriteLine("Passed!");
+#else
+            // TODO(jtattermusch): implement this
+            throw new NotImplementedException("Not supported on CoreCLR yet");
+#endif
         }
 
         public static async Task RunCancelAfterBeginAsync(TestService.TestServiceClient client)
@@ -460,19 +467,25 @@ namespace Grpc.IntegrationTesting
             {
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 31415 } },
                     Payload = CreateZerosPayload(27182)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(31415, call.ResponseStream.Current.Payload.Body.Length);
 
                 cts.Cancel();
 
-                var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.MoveNext());
-                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await call.ResponseStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (RpcException ex)
+                {
+                    Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+                }
             }
             Console.WriteLine("Passed!");
         }
@@ -497,9 +510,16 @@ namespace Grpc.IntegrationTesting
                     // Deadline was reached before write has started. Eat the exception and continue.
                 }
 
-                var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.MoveNext());
-                // We can't guarantee the status code always DeadlineExceeded. See issue #2685.
-                Assert.Contains(ex.Status.StatusCode, new[] { StatusCode.DeadlineExceeded, StatusCode.Internal });
+                try
+                {
+                    await call.ResponseStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (RpcException ex)
+                {
+                    // We can't guarantee the status code always DeadlineExceeded. See issue #2685.
+                    Assert.Contains(ex.Status.StatusCode, new[] { StatusCode.DeadlineExceeded, StatusCode.Internal });
+                }
             }
             Console.WriteLine("Passed!");
         }
@@ -511,7 +531,6 @@ namespace Grpc.IntegrationTesting
                 // step 1: test unary call
                 var request = new SimpleRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseSize = 314159,
                     Payload = CreateZerosPayload(271828)
                 };
@@ -530,7 +549,6 @@ namespace Grpc.IntegrationTesting
                 // step 2: test full duplex call
                 var request = new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 31415 } },
                     Payload = CreateZerosPayload(27182)
                 };
@@ -577,9 +595,17 @@ namespace Grpc.IntegrationTesting
                 await call.RequestStream.WriteAsync(request);
                 await call.RequestStream.CompleteAsync();
 
-                var e = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.ToListAsync());
-                Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
-                Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await call.ResponseStream.ToListAsync();
+                    Assert.Fail();
+                }
+                catch (RpcException e)
+                {
+                    Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
+                    Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+                }
             }
 
             Console.WriteLine("Passed!");
@@ -595,21 +621,127 @@ namespace Grpc.IntegrationTesting
             Console.WriteLine("Passed!");
         }
 
+        public static void RunClientCompressedUnary(TestService.TestServiceClient client)
+        {
+            Console.WriteLine("running client_compressed_unary");
+            var probeRequest = new SimpleRequest
+            {
+                ExpectCompressed = new BoolValue
+                {
+                    Value = true  // lie about compression
+                },
+                ResponseSize = 314159,
+                Payload = CreateZerosPayload(271828)
+            };
+            var e = Assert.Throws<RpcException>(() => client.UnaryCall(probeRequest, CreateClientCompressionMetadata(false)));
+            Assert.AreEqual(StatusCode.InvalidArgument, e.Status.StatusCode);
+
+            var compressedRequest = new SimpleRequest
+            {
+                ExpectCompressed = new BoolValue
+                {
+                    Value = true
+                },
+                ResponseSize = 314159,
+                Payload = CreateZerosPayload(271828)
+            };
+            var response1 = client.UnaryCall(compressedRequest, CreateClientCompressionMetadata(true));
+            Assert.AreEqual(314159, response1.Payload.Body.Length);
+
+            var uncompressedRequest = new SimpleRequest
+            {
+                ExpectCompressed = new BoolValue
+                {
+                    Value = false
+                },
+                ResponseSize = 314159,
+                Payload = CreateZerosPayload(271828)
+            };
+            var response2 = client.UnaryCall(uncompressedRequest, CreateClientCompressionMetadata(false));
+            Assert.AreEqual(314159, response2.Payload.Body.Length);
+
+            Console.WriteLine("Passed!");
+        }
+
+        public static async Task RunClientCompressedStreamingAsync(TestService.TestServiceClient client)
+        {
+            Console.WriteLine("running client_compressed_streaming");
+            try
+            {
+                var probeCall = client.StreamingInputCall(CreateClientCompressionMetadata(false));
+                await probeCall.RequestStream.WriteAsync(new StreamingInputCallRequest
+                {
+                    ExpectCompressed = new BoolValue
+                    {
+                        Value = true
+                    },
+                    Payload = CreateZerosPayload(27182)
+                });
+
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await probeCall;
+                Assert.Fail();
+            }
+            catch (RpcException e)
+            {
+                Assert.AreEqual(StatusCode.InvalidArgument, e.Status.StatusCode);
+            }
+
+            var call = client.StreamingInputCall(CreateClientCompressionMetadata(true));
+            await call.RequestStream.WriteAsync(new StreamingInputCallRequest
+            {
+                ExpectCompressed = new BoolValue
+                {
+                    Value = true
+                },
+                Payload = CreateZerosPayload(27182)
+            });
+
+            call.RequestStream.WriteOptions = new WriteOptions(WriteFlags.NoCompress);
+            await call.RequestStream.WriteAsync(new StreamingInputCallRequest
+            {
+                ExpectCompressed = new BoolValue
+                {
+                    Value = false
+                },
+                Payload = CreateZerosPayload(45904)
+            });
+            await call.RequestStream.CompleteAsync();
+
+            var response = await call.ResponseAsync;
+            Assert.AreEqual(73086, response.AggregatedPayloadSize);
+
+            Console.WriteLine("Passed!");
+        }
+
         private static Payload CreateZerosPayload(int size)
         {
             return new Payload { Body = ByteString.CopyFrom(new byte[size]) };
         }
 
+        private static Metadata CreateClientCompressionMetadata(bool compressed)
+        {
+            var algorithmName = compressed ? "gzip" : "identity";
+            return new Metadata
+            {
+                { new Metadata.Entry(Metadata.CompressionRequestAlgorithmMetadataKey, algorithmName) }
+            };
+        }
+
         // extracts the client_email field from service account file used for auth test cases
         private static string GetEmailFromServiceAccountFile()
         {
+#if !NETSTANDARD1_5
             string keyFile = Environment.GetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS");
             Assert.IsNotNull(keyFile);
-
             var jobject = JObject.Parse(File.ReadAllText(keyFile));
             string email = jobject.GetValue("client_email").Value<string>();
             Assert.IsTrue(email.Length > 0);  // spec requires nonempty client email.
             return email;
+#else
+            // TODO(jtattermusch): implement this
+            throw new NotImplementedException("Not supported on CoreCLR yet");
+#endif
         }
 
         private static Metadata CreateTestMetadata()
diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
index 4ee1ff5ec8..f907f630da 100644
--- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
@@ -69,7 +69,7 @@ namespace Grpc.IntegrationTesting
             };
             int port = server.Ports.Single().BoundPort;
             channel = new Channel(Host, port, TestCredentials.CreateSslCredentials(), options);
-            client = TestService.NewClient(channel);
+            client = new TestService.TestServiceClient(channel);
         }
 
         [TestFixtureTearDown]
@@ -148,7 +148,7 @@ namespace Grpc.IntegrationTesting
         [Test]
         public void UnimplementedMethod()
         {
-            InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel));
+            InteropClient.RunUnimplementedMethod(new UnimplementedService.UnimplementedServiceClient(channel));
         }
     }
 }
diff --git a/src/csharp/Grpc.IntegrationTesting/Messages.cs b/src/csharp/Grpc.IntegrationTesting/Messages.cs
index d42501aa5b..1240db128b 100644
--- a/src/csharp/Grpc.IntegrationTesting/Messages.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Messages.cs
@@ -24,46 +24,48 @@ namespace Grpc.Testing {
       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
             "CiVzcmMvcHJvdG8vZ3JwYy90ZXN0aW5nL21lc3NhZ2VzLnByb3RvEgxncnBj",
-            "LnRlc3RpbmciQAoHUGF5bG9hZBInCgR0eXBlGAEgASgOMhkuZ3JwYy50ZXN0",
-            "aW5nLlBheWxvYWRUeXBlEgwKBGJvZHkYAiABKAwiKwoKRWNob1N0YXR1cxIM",
-            "CgRjb2RlGAEgASgFEg8KB21lc3NhZ2UYAiABKAkioQIKDVNpbXBsZVJlcXVl",
-            "c3QSMAoNcmVzcG9uc2VfdHlwZRgBIAEoDjIZLmdycGMudGVzdGluZy5QYXls",
-            "b2FkVHlwZRIVCg1yZXNwb25zZV9zaXplGAIgASgFEiYKB3BheWxvYWQYAyAB",
-            "KAsyFS5ncnBjLnRlc3RpbmcuUGF5bG9hZBIVCg1maWxsX3VzZXJuYW1lGAQg",
-            "ASgIEhgKEGZpbGxfb2F1dGhfc2NvcGUYBSABKAgSOwoUcmVzcG9uc2VfY29t",
-            "cHJlc3Npb24YBiABKA4yHS5ncnBjLnRlc3RpbmcuQ29tcHJlc3Npb25UeXBl",
-            "EjEKD3Jlc3BvbnNlX3N0YXR1cxgHIAEoCzIYLmdycGMudGVzdGluZy5FY2hv",
-            "U3RhdHVzIl8KDlNpbXBsZVJlc3BvbnNlEiYKB3BheWxvYWQYASABKAsyFS5n",
-            "cnBjLnRlc3RpbmcuUGF5bG9hZBIQCgh1c2VybmFtZRgCIAEoCRITCgtvYXV0",
-            "aF9zY29wZRgDIAEoCSJDChlTdHJlYW1pbmdJbnB1dENhbGxSZXF1ZXN0EiYK",
-            "B3BheWxvYWQYASABKAsyFS5ncnBjLnRlc3RpbmcuUGF5bG9hZCI9ChpTdHJl",
-            "YW1pbmdJbnB1dENhbGxSZXNwb25zZRIfChdhZ2dyZWdhdGVkX3BheWxvYWRf",
-            "c2l6ZRgBIAEoBSI3ChJSZXNwb25zZVBhcmFtZXRlcnMSDAoEc2l6ZRgBIAEo",
-            "BRITCgtpbnRlcnZhbF91cxgCIAEoBSKlAgoaU3RyZWFtaW5nT3V0cHV0Q2Fs",
-            "bFJlcXVlc3QSMAoNcmVzcG9uc2VfdHlwZRgBIAEoDjIZLmdycGMudGVzdGlu",
-            "Zy5QYXlsb2FkVHlwZRI9ChNyZXNwb25zZV9wYXJhbWV0ZXJzGAIgAygLMiAu",
-            "Z3JwYy50ZXN0aW5nLlJlc3BvbnNlUGFyYW1ldGVycxImCgdwYXlsb2FkGAMg",
-            "ASgLMhUuZ3JwYy50ZXN0aW5nLlBheWxvYWQSOwoUcmVzcG9uc2VfY29tcHJl",
-            "c3Npb24YBiABKA4yHS5ncnBjLnRlc3RpbmcuQ29tcHJlc3Npb25UeXBlEjEK",
-            "D3Jlc3BvbnNlX3N0YXR1cxgHIAEoCzIYLmdycGMudGVzdGluZy5FY2hvU3Rh",
-            "dHVzIkUKG1N0cmVhbWluZ091dHB1dENhbGxSZXNwb25zZRImCgdwYXlsb2Fk",
-            "GAEgASgLMhUuZ3JwYy50ZXN0aW5nLlBheWxvYWQiMwoPUmVjb25uZWN0UGFy",
-            "YW1zEiAKGG1heF9yZWNvbm5lY3RfYmFja29mZl9tcxgBIAEoBSIzCg1SZWNv",
-            "bm5lY3RJbmZvEg4KBnBhc3NlZBgBIAEoCBISCgpiYWNrb2ZmX21zGAIgAygF",
-            "Kj8KC1BheWxvYWRUeXBlEhAKDENPTVBSRVNTQUJMRRAAEhIKDlVOQ09NUFJF",
-            "U1NBQkxFEAESCgoGUkFORE9NEAIqMgoPQ29tcHJlc3Npb25UeXBlEggKBE5P",
-            "TkUQABIICgRHWklQEAESCwoHREVGTEFURRACYgZwcm90bzM="));
+            "LnRlc3RpbmciGgoJQm9vbFZhbHVlEg0KBXZhbHVlGAEgASgIIkAKB1BheWxv",
+            "YWQSJwoEdHlwZRgBIAEoDjIZLmdycGMudGVzdGluZy5QYXlsb2FkVHlwZRIM",
+            "CgRib2R5GAIgASgMIisKCkVjaG9TdGF0dXMSDAoEY29kZRgBIAEoBRIPCgdt",
+            "ZXNzYWdlGAIgASgJIs4CCg1TaW1wbGVSZXF1ZXN0EjAKDXJlc3BvbnNlX3R5",
+            "cGUYASABKA4yGS5ncnBjLnRlc3RpbmcuUGF5bG9hZFR5cGUSFQoNcmVzcG9u",
+            "c2Vfc2l6ZRgCIAEoBRImCgdwYXlsb2FkGAMgASgLMhUuZ3JwYy50ZXN0aW5n",
+            "LlBheWxvYWQSFQoNZmlsbF91c2VybmFtZRgEIAEoCBIYChBmaWxsX29hdXRo",
+            "X3Njb3BlGAUgASgIEjQKE3Jlc3BvbnNlX2NvbXByZXNzZWQYBiABKAsyFy5n",
+            "cnBjLnRlc3RpbmcuQm9vbFZhbHVlEjEKD3Jlc3BvbnNlX3N0YXR1cxgHIAEo",
+            "CzIYLmdycGMudGVzdGluZy5FY2hvU3RhdHVzEjIKEWV4cGVjdF9jb21wcmVz",
+            "c2VkGAggASgLMhcuZ3JwYy50ZXN0aW5nLkJvb2xWYWx1ZSJfCg5TaW1wbGVS",
+            "ZXNwb25zZRImCgdwYXlsb2FkGAEgASgLMhUuZ3JwYy50ZXN0aW5nLlBheWxv",
+            "YWQSEAoIdXNlcm5hbWUYAiABKAkSEwoLb2F1dGhfc2NvcGUYAyABKAkidwoZ",
+            "U3RyZWFtaW5nSW5wdXRDYWxsUmVxdWVzdBImCgdwYXlsb2FkGAEgASgLMhUu",
+            "Z3JwYy50ZXN0aW5nLlBheWxvYWQSMgoRZXhwZWN0X2NvbXByZXNzZWQYAiAB",
+            "KAsyFy5ncnBjLnRlc3RpbmcuQm9vbFZhbHVlIj0KGlN0cmVhbWluZ0lucHV0",
+            "Q2FsbFJlc3BvbnNlEh8KF2FnZ3JlZ2F0ZWRfcGF5bG9hZF9zaXplGAEgASgF",
+            "ImQKElJlc3BvbnNlUGFyYW1ldGVycxIMCgRzaXplGAEgASgFEhMKC2ludGVy",
+            "dmFsX3VzGAIgASgFEisKCmNvbXByZXNzZWQYAyABKAsyFy5ncnBjLnRlc3Rp",
+            "bmcuQm9vbFZhbHVlIugBChpTdHJlYW1pbmdPdXRwdXRDYWxsUmVxdWVzdBIw",
+            "Cg1yZXNwb25zZV90eXBlGAEgASgOMhkuZ3JwYy50ZXN0aW5nLlBheWxvYWRU",
+            "eXBlEj0KE3Jlc3BvbnNlX3BhcmFtZXRlcnMYAiADKAsyIC5ncnBjLnRlc3Rp",
+            "bmcuUmVzcG9uc2VQYXJhbWV0ZXJzEiYKB3BheWxvYWQYAyABKAsyFS5ncnBj",
+            "LnRlc3RpbmcuUGF5bG9hZBIxCg9yZXNwb25zZV9zdGF0dXMYByABKAsyGC5n",
+            "cnBjLnRlc3RpbmcuRWNob1N0YXR1cyJFChtTdHJlYW1pbmdPdXRwdXRDYWxs",
+            "UmVzcG9uc2USJgoHcGF5bG9hZBgBIAEoCzIVLmdycGMudGVzdGluZy5QYXls",
+            "b2FkIjMKD1JlY29ubmVjdFBhcmFtcxIgChhtYXhfcmVjb25uZWN0X2JhY2tv",
+            "ZmZfbXMYASABKAUiMwoNUmVjb25uZWN0SW5mbxIOCgZwYXNzZWQYASABKAgS",
+            "EgoKYmFja29mZl9tcxgCIAMoBSofCgtQYXlsb2FkVHlwZRIQCgxDT01QUkVT",
+            "U0FCTEUQAGIGcHJvdG8z"));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Grpc.Testing.PayloadType), typeof(global::Grpc.Testing.CompressionType), }, new pbr::GeneratedClrTypeInfo[] {
+          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Grpc.Testing.PayloadType), }, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.BoolValue), global::Grpc.Testing.BoolValue.Parser, new[]{ "Value" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Payload), global::Grpc.Testing.Payload.Parser, new[]{ "Type", "Body" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoStatus), global::Grpc.Testing.EchoStatus.Parser, new[]{ "Code", "Message" }, null, null, null),
-            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SimpleRequest), global::Grpc.Testing.SimpleRequest.Parser, new[]{ "ResponseType", "ResponseSize", "Payload", "FillUsername", "FillOauthScope", "ResponseCompression", "ResponseStatus" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SimpleRequest), global::Grpc.Testing.SimpleRequest.Parser, new[]{ "ResponseType", "ResponseSize", "Payload", "FillUsername", "FillOauthScope", "ResponseCompressed", "ResponseStatus", "ExpectCompressed" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SimpleResponse), global::Grpc.Testing.SimpleResponse.Parser, new[]{ "Payload", "Username", "OauthScope" }, null, null, null),
-            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingInputCallRequest), global::Grpc.Testing.StreamingInputCallRequest.Parser, new[]{ "Payload" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingInputCallRequest), global::Grpc.Testing.StreamingInputCallRequest.Parser, new[]{ "Payload", "ExpectCompressed" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingInputCallResponse), global::Grpc.Testing.StreamingInputCallResponse.Parser, new[]{ "AggregatedPayloadSize" }, null, null, null),
-            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParameters), global::Grpc.Testing.ResponseParameters.Parser, new[]{ "Size", "IntervalUs" }, null, null, null),
-            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingOutputCallRequest), global::Grpc.Testing.StreamingOutputCallRequest.Parser, new[]{ "ResponseType", "ResponseParameters", "Payload", "ResponseCompression", "ResponseStatus" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParameters), global::Grpc.Testing.ResponseParameters.Parser, new[]{ "Size", "IntervalUs", "Compressed" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingOutputCallRequest), global::Grpc.Testing.StreamingOutputCallRequest.Parser, new[]{ "ResponseType", "ResponseParameters", "Payload", "ResponseStatus" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingOutputCallResponse), global::Grpc.Testing.StreamingOutputCallResponse.Parser, new[]{ "Payload" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ReconnectParams), global::Grpc.Testing.ReconnectParams.Parser, new[]{ "MaxReconnectBackoffMs" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ReconnectInfo), global::Grpc.Testing.ReconnectInfo.Parser, new[]{ "Passed", "BackoffMs" }, null, null, null)
@@ -74,6 +76,7 @@ namespace Grpc.Testing {
   }
   #region Enums
   /// <summary>
+  ///  DEPRECATED, don't use. To be removed shortly.
   ///  The type of payload that should be returned.
   /// </summary>
   public enum PayloadType {
@@ -81,31 +84,122 @@ namespace Grpc.Testing {
     ///  Compressable text format.
     /// </summary>
     [pbr::OriginalName("COMPRESSABLE")] Compressable = 0,
-    /// <summary>
-    ///  Uncompressable binary format.
-    /// </summary>
-    [pbr::OriginalName("UNCOMPRESSABLE")] Uncompressable = 1,
-    /// <summary>
-    ///  Randomly chosen from all other formats defined in this enum.
-    /// </summary>
-    [pbr::OriginalName("RANDOM")] Random = 2,
   }
 
+  #endregion
+
+  #region Messages
   /// <summary>
-  ///  Compression algorithms
+  ///  TODO(dgq): Go back to using well-known types once
+  ///  https://github.com/grpc/grpc/issues/6980 has been fixed.
+  ///  import "google/protobuf/wrappers.proto";
   /// </summary>
-  public enum CompressionType {
+  [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+  public sealed partial class BoolValue : pb::IMessage<BoolValue> {
+    private static readonly pb::MessageParser<BoolValue> _parser = new pb::MessageParser<BoolValue>(() => new BoolValue());
+    public static pb::MessageParser<BoolValue> Parser { get { return _parser; } }
+
+    public static pbr::MessageDescriptor Descriptor {
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[0]; }
+    }
+
+    pbr::MessageDescriptor pb::IMessage.Descriptor {
+      get { return Descriptor; }
+    }
+
+    public BoolValue() {
+      OnConstruction();
+    }
+
+    partial void OnConstruction();
+
+    public BoolValue(BoolValue other) : this() {
+      value_ = other.value_;
+    }
+
+    public BoolValue Clone() {
+      return new BoolValue(this);
+    }
+
+    /// <summary>Field number for the "value" field.</summary>
+    public const int ValueFieldNumber = 1;
+    private bool value_;
     /// <summary>
-    ///  No compression
+    ///  The bool value.
     /// </summary>
-    [pbr::OriginalName("NONE")] None = 0,
-    [pbr::OriginalName("GZIP")] Gzip = 1,
-    [pbr::OriginalName("DEFLATE")] Deflate = 2,
-  }
+    public bool Value {
+      get { return value_; }
+      set {
+        value_ = value;
+      }
+    }
 
-  #endregion
+    public override bool Equals(object other) {
+      return Equals(other as BoolValue);
+    }
+
+    public bool Equals(BoolValue other) {
+      if (ReferenceEquals(other, null)) {
+        return false;
+      }
+      if (ReferenceEquals(other, this)) {
+        return true;
+      }
+      if (Value != other.Value) return false;
+      return true;
+    }
+
+    public override int GetHashCode() {
+      int hash = 1;
+      if (Value != false) hash ^= Value.GetHashCode();
+      return hash;
+    }
+
+    public override string ToString() {
+      return pb::JsonFormatter.ToDiagnosticString(this);
+    }
+
+    public void WriteTo(pb::CodedOutputStream output) {
+      if (Value != false) {
+        output.WriteRawTag(8);
+        output.WriteBool(Value);
+      }
+    }
+
+    public int CalculateSize() {
+      int size = 0;
+      if (Value != false) {
+        size += 1 + 1;
+      }
+      return size;
+    }
+
+    public void MergeFrom(BoolValue other) {
+      if (other == null) {
+        return;
+      }
+      if (other.Value != false) {
+        Value = other.Value;
+      }
+    }
+
+    public void MergeFrom(pb::CodedInputStream input) {
+      uint tag;
+      while ((tag = input.ReadTag()) != 0) {
+        switch(tag) {
+          default:
+            input.SkipLastField();
+            break;
+          case 8: {
+            Value = input.ReadBool();
+            break;
+          }
+        }
+      }
+    }
+
+  }
 
-  #region Messages
   /// <summary>
   ///  A block of data, to simply increase gRPC message size.
   /// </summary>
@@ -115,7 +209,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<Payload> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[0]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[1]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -141,6 +235,7 @@ namespace Grpc.Testing {
     public const int TypeFieldNumber = 1;
     private global::Grpc.Testing.PayloadType type_ = 0;
     /// <summary>
+    ///  DEPRECATED, don't use. To be removed shortly.
     ///  The type of data in body.
     /// </summary>
     public global::Grpc.Testing.PayloadType Type {
@@ -255,7 +350,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<EchoStatus> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[1]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[2]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -388,7 +483,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<SimpleRequest> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[2]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[3]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -407,8 +502,9 @@ namespace Grpc.Testing {
       Payload = other.payload_ != null ? other.Payload.Clone() : null;
       fillUsername_ = other.fillUsername_;
       fillOauthScope_ = other.fillOauthScope_;
-      responseCompression_ = other.responseCompression_;
+      ResponseCompressed = other.responseCompressed_ != null ? other.ResponseCompressed.Clone() : null;
       ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null;
+      ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null;
     }
 
     public SimpleRequest Clone() {
@@ -419,6 +515,7 @@ namespace Grpc.Testing {
     public const int ResponseTypeFieldNumber = 1;
     private global::Grpc.Testing.PayloadType responseType_ = 0;
     /// <summary>
+    ///  DEPRECATED, don't use. To be removed shortly.
     ///  Desired payload type in the response from the server.
     ///  If response_type is RANDOM, server randomly chooses one from other formats.
     /// </summary>
@@ -434,7 +531,6 @@ namespace Grpc.Testing {
     private int responseSize_;
     /// <summary>
     ///  Desired payload size in the response from the server.
-    ///  If response_type is COMPRESSABLE, this denotes the size before compression.
     /// </summary>
     public int ResponseSize {
       get { return responseSize_; }
@@ -482,16 +578,19 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Field number for the "response_compression" field.</summary>
-    public const int ResponseCompressionFieldNumber = 6;
-    private global::Grpc.Testing.CompressionType responseCompression_ = 0;
+    /// <summary>Field number for the "response_compressed" field.</summary>
+    public const int ResponseCompressedFieldNumber = 6;
+    private global::Grpc.Testing.BoolValue responseCompressed_;
     /// <summary>
-    ///  Compression algorithm to be used by the server for the response (stream)
+    ///  Whether to request the server to compress the response. This field is
+    ///  "nullable" in order to interoperate seamlessly with clients not able to
+    ///  implement the full compression tests by introspecting the call to verify
+    ///  the response's compression status.
     /// </summary>
-    public global::Grpc.Testing.CompressionType ResponseCompression {
-      get { return responseCompression_; }
+    public global::Grpc.Testing.BoolValue ResponseCompressed {
+      get { return responseCompressed_; }
       set {
-        responseCompression_ = value;
+        responseCompressed_ = value;
       }
     }
 
@@ -508,6 +607,19 @@ namespace Grpc.Testing {
       }
     }
 
+    /// <summary>Field number for the "expect_compressed" field.</summary>
+    public const int ExpectCompressedFieldNumber = 8;
+    private global::Grpc.Testing.BoolValue expectCompressed_;
+    /// <summary>
+    ///  Whether the server should expect this request to be compressed.
+    /// </summary>
+    public global::Grpc.Testing.BoolValue ExpectCompressed {
+      get { return expectCompressed_; }
+      set {
+        expectCompressed_ = value;
+      }
+    }
+
     public override bool Equals(object other) {
       return Equals(other as SimpleRequest);
     }
@@ -524,8 +636,9 @@ namespace Grpc.Testing {
       if (!object.Equals(Payload, other.Payload)) return false;
       if (FillUsername != other.FillUsername) return false;
       if (FillOauthScope != other.FillOauthScope) return false;
-      if (ResponseCompression != other.ResponseCompression) return false;
+      if (!object.Equals(ResponseCompressed, other.ResponseCompressed)) return false;
       if (!object.Equals(ResponseStatus, other.ResponseStatus)) return false;
+      if (!object.Equals(ExpectCompressed, other.ExpectCompressed)) return false;
       return true;
     }
 
@@ -536,8 +649,9 @@ namespace Grpc.Testing {
       if (payload_ != null) hash ^= Payload.GetHashCode();
       if (FillUsername != false) hash ^= FillUsername.GetHashCode();
       if (FillOauthScope != false) hash ^= FillOauthScope.GetHashCode();
-      if (ResponseCompression != 0) hash ^= ResponseCompression.GetHashCode();
+      if (responseCompressed_ != null) hash ^= ResponseCompressed.GetHashCode();
       if (responseStatus_ != null) hash ^= ResponseStatus.GetHashCode();
+      if (expectCompressed_ != null) hash ^= ExpectCompressed.GetHashCode();
       return hash;
     }
 
@@ -566,14 +680,18 @@ namespace Grpc.Testing {
         output.WriteRawTag(40);
         output.WriteBool(FillOauthScope);
       }
-      if (ResponseCompression != 0) {
-        output.WriteRawTag(48);
-        output.WriteEnum((int) ResponseCompression);
+      if (responseCompressed_ != null) {
+        output.WriteRawTag(50);
+        output.WriteMessage(ResponseCompressed);
       }
       if (responseStatus_ != null) {
         output.WriteRawTag(58);
         output.WriteMessage(ResponseStatus);
       }
+      if (expectCompressed_ != null) {
+        output.WriteRawTag(66);
+        output.WriteMessage(ExpectCompressed);
+      }
     }
 
     public int CalculateSize() {
@@ -593,12 +711,15 @@ namespace Grpc.Testing {
       if (FillOauthScope != false) {
         size += 1 + 1;
       }
-      if (ResponseCompression != 0) {
-        size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ResponseCompression);
+      if (responseCompressed_ != null) {
+        size += 1 + pb::CodedOutputStream.ComputeMessageSize(ResponseCompressed);
       }
       if (responseStatus_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(ResponseStatus);
       }
+      if (expectCompressed_ != null) {
+        size += 1 + pb::CodedOutputStream.ComputeMessageSize(ExpectCompressed);
+      }
       return size;
     }
 
@@ -624,8 +745,11 @@ namespace Grpc.Testing {
       if (other.FillOauthScope != false) {
         FillOauthScope = other.FillOauthScope;
       }
-      if (other.ResponseCompression != 0) {
-        ResponseCompression = other.ResponseCompression;
+      if (other.responseCompressed_ != null) {
+        if (responseCompressed_ == null) {
+          responseCompressed_ = new global::Grpc.Testing.BoolValue();
+        }
+        ResponseCompressed.MergeFrom(other.ResponseCompressed);
       }
       if (other.responseStatus_ != null) {
         if (responseStatus_ == null) {
@@ -633,6 +757,12 @@ namespace Grpc.Testing {
         }
         ResponseStatus.MergeFrom(other.ResponseStatus);
       }
+      if (other.expectCompressed_ != null) {
+        if (expectCompressed_ == null) {
+          expectCompressed_ = new global::Grpc.Testing.BoolValue();
+        }
+        ExpectCompressed.MergeFrom(other.ExpectCompressed);
+      }
     }
 
     public void MergeFrom(pb::CodedInputStream input) {
@@ -665,8 +795,11 @@ namespace Grpc.Testing {
             FillOauthScope = input.ReadBool();
             break;
           }
-          case 48: {
-            responseCompression_ = (global::Grpc.Testing.CompressionType) input.ReadEnum();
+          case 50: {
+            if (responseCompressed_ == null) {
+              responseCompressed_ = new global::Grpc.Testing.BoolValue();
+            }
+            input.ReadMessage(responseCompressed_);
             break;
           }
           case 58: {
@@ -676,6 +809,13 @@ namespace Grpc.Testing {
             input.ReadMessage(responseStatus_);
             break;
           }
+          case 66: {
+            if (expectCompressed_ == null) {
+              expectCompressed_ = new global::Grpc.Testing.BoolValue();
+            }
+            input.ReadMessage(expectCompressed_);
+            break;
+          }
         }
       }
     }
@@ -691,7 +831,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<SimpleResponse> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[3]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[4]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -867,7 +1007,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<StreamingInputCallRequest> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[4]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[5]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -882,6 +1022,7 @@ namespace Grpc.Testing {
 
     public StreamingInputCallRequest(StreamingInputCallRequest other) : this() {
       Payload = other.payload_ != null ? other.Payload.Clone() : null;
+      ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null;
     }
 
     public StreamingInputCallRequest Clone() {
@@ -901,6 +1042,22 @@ namespace Grpc.Testing {
       }
     }
 
+    /// <summary>Field number for the "expect_compressed" field.</summary>
+    public const int ExpectCompressedFieldNumber = 2;
+    private global::Grpc.Testing.BoolValue expectCompressed_;
+    /// <summary>
+    ///  Whether the server should expect this request to be compressed. This field
+    ///  is "nullable" in order to interoperate seamlessly with servers not able to
+    ///  implement the full compression tests by introspecting the call to verify
+    ///  the request's compression status.
+    /// </summary>
+    public global::Grpc.Testing.BoolValue ExpectCompressed {
+      get { return expectCompressed_; }
+      set {
+        expectCompressed_ = value;
+      }
+    }
+
     public override bool Equals(object other) {
       return Equals(other as StreamingInputCallRequest);
     }
@@ -913,12 +1070,14 @@ namespace Grpc.Testing {
         return true;
       }
       if (!object.Equals(Payload, other.Payload)) return false;
+      if (!object.Equals(ExpectCompressed, other.ExpectCompressed)) return false;
       return true;
     }
 
     public override int GetHashCode() {
       int hash = 1;
       if (payload_ != null) hash ^= Payload.GetHashCode();
+      if (expectCompressed_ != null) hash ^= ExpectCompressed.GetHashCode();
       return hash;
     }
 
@@ -931,6 +1090,10 @@ namespace Grpc.Testing {
         output.WriteRawTag(10);
         output.WriteMessage(Payload);
       }
+      if (expectCompressed_ != null) {
+        output.WriteRawTag(18);
+        output.WriteMessage(ExpectCompressed);
+      }
     }
 
     public int CalculateSize() {
@@ -938,6 +1101,9 @@ namespace Grpc.Testing {
       if (payload_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(Payload);
       }
+      if (expectCompressed_ != null) {
+        size += 1 + pb::CodedOutputStream.ComputeMessageSize(ExpectCompressed);
+      }
       return size;
     }
 
@@ -951,6 +1117,12 @@ namespace Grpc.Testing {
         }
         Payload.MergeFrom(other.Payload);
       }
+      if (other.expectCompressed_ != null) {
+        if (expectCompressed_ == null) {
+          expectCompressed_ = new global::Grpc.Testing.BoolValue();
+        }
+        ExpectCompressed.MergeFrom(other.ExpectCompressed);
+      }
     }
 
     public void MergeFrom(pb::CodedInputStream input) {
@@ -967,6 +1139,13 @@ namespace Grpc.Testing {
             input.ReadMessage(payload_);
             break;
           }
+          case 18: {
+            if (expectCompressed_ == null) {
+              expectCompressed_ = new global::Grpc.Testing.BoolValue();
+            }
+            input.ReadMessage(expectCompressed_);
+            break;
+          }
         }
       }
     }
@@ -982,7 +1161,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<StreamingInputCallResponse> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[5]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[6]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -1091,7 +1270,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<ResponseParameters> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[6]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[7]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -1107,6 +1286,7 @@ namespace Grpc.Testing {
     public ResponseParameters(ResponseParameters other) : this() {
       size_ = other.size_;
       intervalUs_ = other.intervalUs_;
+      Compressed = other.compressed_ != null ? other.Compressed.Clone() : null;
     }
 
     public ResponseParameters Clone() {
@@ -1118,7 +1298,6 @@ namespace Grpc.Testing {
     private int size_;
     /// <summary>
     ///  Desired payload sizes in responses from the server.
-    ///  If response_type is COMPRESSABLE, this denotes the size before compression.
     /// </summary>
     public int Size {
       get { return size_; }
@@ -1141,6 +1320,22 @@ namespace Grpc.Testing {
       }
     }
 
+    /// <summary>Field number for the "compressed" field.</summary>
+    public const int CompressedFieldNumber = 3;
+    private global::Grpc.Testing.BoolValue compressed_;
+    /// <summary>
+    ///  Whether to request the server to compress the response. This field is
+    ///  "nullable" in order to interoperate seamlessly with clients not able to
+    ///  implement the full compression tests by introspecting the call to verify
+    ///  the response's compression status.
+    /// </summary>
+    public global::Grpc.Testing.BoolValue Compressed {
+      get { return compressed_; }
+      set {
+        compressed_ = value;
+      }
+    }
+
     public override bool Equals(object other) {
       return Equals(other as ResponseParameters);
     }
@@ -1154,6 +1349,7 @@ namespace Grpc.Testing {
       }
       if (Size != other.Size) return false;
       if (IntervalUs != other.IntervalUs) return false;
+      if (!object.Equals(Compressed, other.Compressed)) return false;
       return true;
     }
 
@@ -1161,6 +1357,7 @@ namespace Grpc.Testing {
       int hash = 1;
       if (Size != 0) hash ^= Size.GetHashCode();
       if (IntervalUs != 0) hash ^= IntervalUs.GetHashCode();
+      if (compressed_ != null) hash ^= Compressed.GetHashCode();
       return hash;
     }
 
@@ -1177,6 +1374,10 @@ namespace Grpc.Testing {
         output.WriteRawTag(16);
         output.WriteInt32(IntervalUs);
       }
+      if (compressed_ != null) {
+        output.WriteRawTag(26);
+        output.WriteMessage(Compressed);
+      }
     }
 
     public int CalculateSize() {
@@ -1187,6 +1388,9 @@ namespace Grpc.Testing {
       if (IntervalUs != 0) {
         size += 1 + pb::CodedOutputStream.ComputeInt32Size(IntervalUs);
       }
+      if (compressed_ != null) {
+        size += 1 + pb::CodedOutputStream.ComputeMessageSize(Compressed);
+      }
       return size;
     }
 
@@ -1200,6 +1404,12 @@ namespace Grpc.Testing {
       if (other.IntervalUs != 0) {
         IntervalUs = other.IntervalUs;
       }
+      if (other.compressed_ != null) {
+        if (compressed_ == null) {
+          compressed_ = new global::Grpc.Testing.BoolValue();
+        }
+        Compressed.MergeFrom(other.Compressed);
+      }
     }
 
     public void MergeFrom(pb::CodedInputStream input) {
@@ -1217,6 +1427,13 @@ namespace Grpc.Testing {
             IntervalUs = input.ReadInt32();
             break;
           }
+          case 26: {
+            if (compressed_ == null) {
+              compressed_ = new global::Grpc.Testing.BoolValue();
+            }
+            input.ReadMessage(compressed_);
+            break;
+          }
         }
       }
     }
@@ -1232,7 +1449,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<StreamingOutputCallRequest> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[7]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[8]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -1249,7 +1466,6 @@ namespace Grpc.Testing {
       responseType_ = other.responseType_;
       responseParameters_ = other.responseParameters_.Clone();
       Payload = other.payload_ != null ? other.Payload.Clone() : null;
-      responseCompression_ = other.responseCompression_;
       ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null;
     }
 
@@ -1261,6 +1477,7 @@ namespace Grpc.Testing {
     public const int ResponseTypeFieldNumber = 1;
     private global::Grpc.Testing.PayloadType responseType_ = 0;
     /// <summary>
+    ///  DEPRECATED, don't use. To be removed shortly.
     ///  Desired payload type in the response from the server.
     ///  If response_type is RANDOM, the payload from each response in the stream
     ///  might be of different types. This is to simulate a mixed type of payload
@@ -1298,19 +1515,6 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Field number for the "response_compression" field.</summary>
-    public const int ResponseCompressionFieldNumber = 6;
-    private global::Grpc.Testing.CompressionType responseCompression_ = 0;
-    /// <summary>
-    ///  Compression algorithm to be used by the server for the response (stream)
-    /// </summary>
-    public global::Grpc.Testing.CompressionType ResponseCompression {
-      get { return responseCompression_; }
-      set {
-        responseCompression_ = value;
-      }
-    }
-
     /// <summary>Field number for the "response_status" field.</summary>
     public const int ResponseStatusFieldNumber = 7;
     private global::Grpc.Testing.EchoStatus responseStatus_;
@@ -1338,7 +1542,6 @@ namespace Grpc.Testing {
       if (ResponseType != other.ResponseType) return false;
       if(!responseParameters_.Equals(other.responseParameters_)) return false;
       if (!object.Equals(Payload, other.Payload)) return false;
-      if (ResponseCompression != other.ResponseCompression) return false;
       if (!object.Equals(ResponseStatus, other.ResponseStatus)) return false;
       return true;
     }
@@ -1348,7 +1551,6 @@ namespace Grpc.Testing {
       if (ResponseType != 0) hash ^= ResponseType.GetHashCode();
       hash ^= responseParameters_.GetHashCode();
       if (payload_ != null) hash ^= Payload.GetHashCode();
-      if (ResponseCompression != 0) hash ^= ResponseCompression.GetHashCode();
       if (responseStatus_ != null) hash ^= ResponseStatus.GetHashCode();
       return hash;
     }
@@ -1367,10 +1569,6 @@ namespace Grpc.Testing {
         output.WriteRawTag(26);
         output.WriteMessage(Payload);
       }
-      if (ResponseCompression != 0) {
-        output.WriteRawTag(48);
-        output.WriteEnum((int) ResponseCompression);
-      }
       if (responseStatus_ != null) {
         output.WriteRawTag(58);
         output.WriteMessage(ResponseStatus);
@@ -1386,9 +1584,6 @@ namespace Grpc.Testing {
       if (payload_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(Payload);
       }
-      if (ResponseCompression != 0) {
-        size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ResponseCompression);
-      }
       if (responseStatus_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(ResponseStatus);
       }
@@ -1409,9 +1604,6 @@ namespace Grpc.Testing {
         }
         Payload.MergeFrom(other.Payload);
       }
-      if (other.ResponseCompression != 0) {
-        ResponseCompression = other.ResponseCompression;
-      }
       if (other.responseStatus_ != null) {
         if (responseStatus_ == null) {
           responseStatus_ = new global::Grpc.Testing.EchoStatus();
@@ -1442,10 +1634,6 @@ namespace Grpc.Testing {
             input.ReadMessage(payload_);
             break;
           }
-          case 48: {
-            responseCompression_ = (global::Grpc.Testing.CompressionType) input.ReadEnum();
-            break;
-          }
           case 58: {
             if (responseStatus_ == null) {
               responseStatus_ = new global::Grpc.Testing.EchoStatus();
@@ -1468,7 +1656,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<StreamingOutputCallResponse> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[8]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[9]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -1584,7 +1772,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<ReconnectParams> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[9]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[10]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
@@ -1692,7 +1880,7 @@ namespace Grpc.Testing {
     public static pb::MessageParser<ReconnectInfo> Parser { get { return _parser; } }
 
     public static pbr::MessageDescriptor Descriptor {
-      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[10]; }
+      get { return global::Grpc.Testing.MessagesReflection.Descriptor.MessageTypes[11]; }
     }
 
     pbr::MessageDescriptor pb::IMessage.Descriptor {
diff --git a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
index f95af5008f..eb3bb8a28b 100644
--- a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
@@ -40,7 +40,6 @@ using System.Threading.Tasks;
 using Grpc.Core;
 using Grpc.Core.Utils;
 using Grpc.Testing;
-using Moq;
 using NUnit.Framework;
 
 namespace Grpc.IntegrationTesting
@@ -52,19 +51,14 @@ namespace Grpc.IntegrationTesting
         Channel channel;
         TestService.TestServiceClient client;
         List<ChannelOption> options;
-        Mock<TestService.TestServiceBase> serviceMock;
         AsyncAuthInterceptor asyncAuthInterceptor;
 
         [SetUp]
         public void Init()
         {
-            serviceMock = new Mock<TestService.TestServiceBase>();
-            serviceMock.Setup(m => m.UnaryCall(It.IsAny<SimpleRequest>(), It.IsAny<ServerCallContext>()))
-                .Returns(new Func<SimpleRequest, ServerCallContext, Task<SimpleResponse>>(UnaryCallHandler));
-
             server = new Server
             {
-                Services = { TestService.BindService(serviceMock.Object) },
+                Services = { TestService.BindService(new FakeTestService()) },
                 Ports = { { Host, ServerPort.PickUnused, TestCredentials.CreateSslServerCredentials() } }
             };
             server.Start();
@@ -94,26 +88,29 @@ namespace Grpc.IntegrationTesting
             var channelCredentials = ChannelCredentials.Create(TestCredentials.CreateSslCredentials(),
                 CallCredentials.FromInterceptor(asyncAuthInterceptor));
             channel = new Channel(Host, server.Ports.Single().BoundPort, channelCredentials, options);
-            client = TestService.NewClient(channel);
+            client = new TestService.TestServiceClient(channel);
 
-            client.UnaryCall(new SimpleRequest {});
+            client.UnaryCall(new SimpleRequest { });
         }
 
         [Test]
         public void MetadataCredentials_PerCall()
         {
             channel = new Channel(Host, server.Ports.Single().BoundPort, TestCredentials.CreateSslCredentials(), options);
-            client = TestService.NewClient(channel);
+            client = new TestService.TestServiceClient(channel);
 
             var callCredentials = CallCredentials.FromInterceptor(asyncAuthInterceptor);
             client.UnaryCall(new SimpleRequest { }, new CallOptions(credentials: callCredentials));
         }
 
-        private Task<SimpleResponse> UnaryCallHandler(SimpleRequest request, ServerCallContext context)
+        private class FakeTestService : TestService.TestServiceBase
         {
-            var authToken = context.RequestHeaders.First((entry) => entry.Key == "authorization").Value;
-            Assert.AreEqual("SECRET_TOKEN", authToken);
-            return Task.FromResult(new SimpleResponse());
+            public override Task<SimpleResponse> UnaryCall(SimpleRequest request, ServerCallContext context)
+            {
+                var authToken = context.RequestHeaders.First((entry) => entry.Key == "authorization").Value;
+                Assert.AreEqual("SECRET_TOKEN", authToken);
+                return Task.FromResult(new SimpleResponse());
+            }
         }
     }
 }
diff --git a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs
index 9d31d1c514..040798e3c2 100644
--- a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs
+++ b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs
@@ -72,53 +72,6 @@ namespace Grpc.Testing {
       get { return global::Grpc.Testing.MetricsReflection.Descriptor.Services[0]; }
     }
 
-    /// <summary>Client for MetricsService</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IMetricsServiceClient
-    {
-      /// <summary>
-      ///  Returns the values of all the gauges that are currently being maintained by
-      ///  the service
-      /// </summary>
-      AsyncServerStreamingCall<global::Grpc.Testing.GaugeResponse> GetAllGauges(global::Grpc.Testing.EmptyMessage request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Returns the values of all the gauges that are currently being maintained by
-      ///  the service
-      /// </summary>
-      AsyncServerStreamingCall<global::Grpc.Testing.GaugeResponse> GetAllGauges(global::Grpc.Testing.EmptyMessage request, CallOptions options);
-      /// <summary>
-      ///  Returns the value of one gauge
-      /// </summary>
-      global::Grpc.Testing.GaugeResponse GetGauge(global::Grpc.Testing.GaugeRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Returns the value of one gauge
-      /// </summary>
-      global::Grpc.Testing.GaugeResponse GetGauge(global::Grpc.Testing.GaugeRequest request, CallOptions options);
-      /// <summary>
-      ///  Returns the value of one gauge
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.GaugeResponse> GetGaugeAsync(global::Grpc.Testing.GaugeRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Returns the value of one gauge
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.GaugeResponse> GetGaugeAsync(global::Grpc.Testing.GaugeRequest request, CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of MetricsService</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IMetricsService
-    {
-      /// <summary>
-      ///  Returns the values of all the gauges that are currently being maintained by
-      ///  the service
-      /// </summary>
-      global::System.Threading.Tasks.Task GetAllGauges(global::Grpc.Testing.EmptyMessage request, IServerStreamWriter<global::Grpc.Testing.GaugeResponse> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  Returns the value of one gauge
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.GaugeResponse> GetGauge(global::Grpc.Testing.GaugeRequest request, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of MetricsService</summary>
     public abstract class MetricsServiceBase
     {
@@ -142,21 +95,24 @@ namespace Grpc.Testing {
     }
 
     /// <summary>Client for MetricsService</summary>
-    #pragma warning disable 0618
-    public class MetricsServiceClient : ClientBase<MetricsServiceClient>, IMetricsServiceClient
-    #pragma warning restore 0618
+    public class MetricsServiceClient : ClientBase<MetricsServiceClient>
     {
+      /// <summary>Creates a new client for MetricsService</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public MetricsServiceClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for MetricsService that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public MetricsServiceClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected MetricsServiceClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected MetricsServiceClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -211,28 +167,10 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Creates a new client for MetricsService</summary>
-    public static MetricsServiceClient NewClient(Channel channel)
-    {
-      return new MetricsServiceClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IMetricsService serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_GetAllGauges, serviceImpl.GetAllGauges)
-          .AddMethod(__Method_GetGauge, serviceImpl.GetGauge).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(MetricsServiceBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_GetAllGauges, serviceImpl.GetAllGauges)
           .AddMethod(__Method_GetGauge, serviceImpl.GetGauge).Build();
     }
diff --git a/src/csharp/Grpc.IntegrationTesting/NUnitMain.cs b/src/csharp/Grpc.IntegrationTesting/NUnitMain.cs
index d8902de08f..100ff0b5de 100644
--- a/src/csharp/Grpc.IntegrationTesting/NUnitMain.cs
+++ b/src/csharp/Grpc.IntegrationTesting/NUnitMain.cs
@@ -49,7 +49,7 @@ namespace Grpc.IntegrationTesting
         {
             // Make logger immune to NUnit capturing stdout and stderr to workaround https://github.com/nunit/nunit/issues/1406.
             GrpcEnvironment.SetLogger(new TextWriterLogger(Console.Error));
-#if DOTNET5_4
+#if NETSTANDARD1_5
             return new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args, new ExtendedTextWrapper(Console.Out), Console.In);
 #else
             return new AutoRun().Execute(args);
diff --git a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs
index f7071ebf6b..e205dea93e 100644
--- a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs
@@ -67,58 +67,6 @@ namespace Grpc.Testing {
       get { return global::Grpc.Testing.ServicesReflection.Descriptor.Services[0]; }
     }
 
-    /// <summary>Client for BenchmarkService</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IBenchmarkServiceClient
-    {
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      global::Grpc.Testing.SimpleResponse UnaryCall(global::Grpc.Testing.SimpleRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      global::Grpc.Testing.SimpleResponse UnaryCall(global::Grpc.Testing.SimpleRequest request, CallOptions options);
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.SimpleResponse> UnaryCallAsync(global::Grpc.Testing.SimpleRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.SimpleResponse> UnaryCallAsync(global::Grpc.Testing.SimpleRequest request, CallOptions options);
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.SimpleRequest, global::Grpc.Testing.SimpleResponse> StreamingCall(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.SimpleRequest, global::Grpc.Testing.SimpleResponse> StreamingCall(CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of BenchmarkService</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IBenchmarkService
-    {
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.SimpleResponse> UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context);
-      /// <summary>
-      ///  One request followed by one response.
-      ///  The server returns the client payload as-is.
-      /// </summary>
-      global::System.Threading.Tasks.Task StreamingCall(IAsyncStreamReader<global::Grpc.Testing.SimpleRequest> requestStream, IServerStreamWriter<global::Grpc.Testing.SimpleResponse> responseStream, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of BenchmarkService</summary>
     public abstract class BenchmarkServiceBase
     {
@@ -143,21 +91,24 @@ namespace Grpc.Testing {
     }
 
     /// <summary>Client for BenchmarkService</summary>
-    #pragma warning disable 0618
-    public class BenchmarkServiceClient : ClientBase<BenchmarkServiceClient>, IBenchmarkServiceClient
-    #pragma warning restore 0618
+    public class BenchmarkServiceClient : ClientBase<BenchmarkServiceClient>
     {
+      /// <summary>Creates a new client for BenchmarkService</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public BenchmarkServiceClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for BenchmarkService that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public BenchmarkServiceClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected BenchmarkServiceClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected BenchmarkServiceClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -216,28 +167,10 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Creates a new client for BenchmarkService</summary>
-    public static BenchmarkServiceClient NewClient(Channel channel)
-    {
-      return new BenchmarkServiceClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IBenchmarkService serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_UnaryCall, serviceImpl.UnaryCall)
-          .AddMethod(__Method_StreamingCall, serviceImpl.StreamingCall).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(BenchmarkServiceBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_UnaryCall, serviceImpl.UnaryCall)
           .AddMethod(__Method_StreamingCall, serviceImpl.StreamingCall).Build();
     }
@@ -289,112 +222,6 @@ namespace Grpc.Testing {
       get { return global::Grpc.Testing.ServicesReflection.Descriptor.Services[1]; }
     }
 
-    /// <summary>Client for WorkerService</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IWorkerServiceClient
-    {
-      /// <summary>
-      ///  Start server with specified workload.
-      ///  First request sent specifies the ServerConfig followed by ServerStatus
-      ///  response. After that, a "Mark" can be sent anytime to request the latest
-      ///  stats. Closing the stream will initiate shutdown of the test server
-      ///  and once the shutdown has finished, the OK status is sent to terminate
-      ///  this RPC.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.ServerArgs, global::Grpc.Testing.ServerStatus> RunServer(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Start server with specified workload.
-      ///  First request sent specifies the ServerConfig followed by ServerStatus
-      ///  response. After that, a "Mark" can be sent anytime to request the latest
-      ///  stats. Closing the stream will initiate shutdown of the test server
-      ///  and once the shutdown has finished, the OK status is sent to terminate
-      ///  this RPC.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.ServerArgs, global::Grpc.Testing.ServerStatus> RunServer(CallOptions options);
-      /// <summary>
-      ///  Start client with specified workload.
-      ///  First request sent specifies the ClientConfig followed by ClientStatus
-      ///  response. After that, a "Mark" can be sent anytime to request the latest
-      ///  stats. Closing the stream will initiate shutdown of the test client
-      ///  and once the shutdown has finished, the OK status is sent to terminate
-      ///  this RPC.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.ClientArgs, global::Grpc.Testing.ClientStatus> RunClient(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Start client with specified workload.
-      ///  First request sent specifies the ClientConfig followed by ClientStatus
-      ///  response. After that, a "Mark" can be sent anytime to request the latest
-      ///  stats. Closing the stream will initiate shutdown of the test client
-      ///  and once the shutdown has finished, the OK status is sent to terminate
-      ///  this RPC.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.ClientArgs, global::Grpc.Testing.ClientStatus> RunClient(CallOptions options);
-      /// <summary>
-      ///  Just return the core count - unary call
-      /// </summary>
-      global::Grpc.Testing.CoreResponse CoreCount(global::Grpc.Testing.CoreRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Just return the core count - unary call
-      /// </summary>
-      global::Grpc.Testing.CoreResponse CoreCount(global::Grpc.Testing.CoreRequest request, CallOptions options);
-      /// <summary>
-      ///  Just return the core count - unary call
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.CoreResponse> CoreCountAsync(global::Grpc.Testing.CoreRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Just return the core count - unary call
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.CoreResponse> CoreCountAsync(global::Grpc.Testing.CoreRequest request, CallOptions options);
-      /// <summary>
-      ///  Quit this worker
-      /// </summary>
-      global::Grpc.Testing.Void QuitWorker(global::Grpc.Testing.Void request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Quit this worker
-      /// </summary>
-      global::Grpc.Testing.Void QuitWorker(global::Grpc.Testing.Void request, CallOptions options);
-      /// <summary>
-      ///  Quit this worker
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.Void> QuitWorkerAsync(global::Grpc.Testing.Void request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  Quit this worker
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.Void> QuitWorkerAsync(global::Grpc.Testing.Void request, CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of WorkerService</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IWorkerService
-    {
-      /// <summary>
-      ///  Start server with specified workload.
-      ///  First request sent specifies the ServerConfig followed by ServerStatus
-      ///  response. After that, a "Mark" can be sent anytime to request the latest
-      ///  stats. Closing the stream will initiate shutdown of the test server
-      ///  and once the shutdown has finished, the OK status is sent to terminate
-      ///  this RPC.
-      /// </summary>
-      global::System.Threading.Tasks.Task RunServer(IAsyncStreamReader<global::Grpc.Testing.ServerArgs> requestStream, IServerStreamWriter<global::Grpc.Testing.ServerStatus> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  Start client with specified workload.
-      ///  First request sent specifies the ClientConfig followed by ClientStatus
-      ///  response. After that, a "Mark" can be sent anytime to request the latest
-      ///  stats. Closing the stream will initiate shutdown of the test client
-      ///  and once the shutdown has finished, the OK status is sent to terminate
-      ///  this RPC.
-      /// </summary>
-      global::System.Threading.Tasks.Task RunClient(IAsyncStreamReader<global::Grpc.Testing.ClientArgs> requestStream, IServerStreamWriter<global::Grpc.Testing.ClientStatus> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  Just return the core count - unary call
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.CoreResponse> CoreCount(global::Grpc.Testing.CoreRequest request, ServerCallContext context);
-      /// <summary>
-      ///  Quit this worker
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.Void> QuitWorker(global::Grpc.Testing.Void request, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of WorkerService</summary>
     public abstract class WorkerServiceBase
     {
@@ -443,21 +270,24 @@ namespace Grpc.Testing {
     }
 
     /// <summary>Client for WorkerService</summary>
-    #pragma warning disable 0618
-    public class WorkerServiceClient : ClientBase<WorkerServiceClient>, IWorkerServiceClient
-    #pragma warning restore 0618
+    public class WorkerServiceClient : ClientBase<WorkerServiceClient>
     {
+      /// <summary>Creates a new client for WorkerService</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public WorkerServiceClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for WorkerService that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public WorkerServiceClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected WorkerServiceClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected WorkerServiceClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -572,30 +402,10 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Creates a new client for WorkerService</summary>
-    public static WorkerServiceClient NewClient(Channel channel)
-    {
-      return new WorkerServiceClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IWorkerService serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_RunServer, serviceImpl.RunServer)
-          .AddMethod(__Method_RunClient, serviceImpl.RunClient)
-          .AddMethod(__Method_CoreCount, serviceImpl.CoreCount)
-          .AddMethod(__Method_QuitWorker, serviceImpl.QuitWorker).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(WorkerServiceBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_RunServer, serviceImpl.RunServer)
           .AddMethod(__Method_RunClient, serviceImpl.RunClient)
           .AddMethod(__Method_CoreCount, serviceImpl.CoreCount)
diff --git a/src/csharp/Grpc.IntegrationTesting/SslCredentialsTest.cs b/src/csharp/Grpc.IntegrationTesting/SslCredentialsTest.cs
index 3df45b5f70..f85e272711 100644
--- a/src/csharp/Grpc.IntegrationTesting/SslCredentialsTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/SslCredentialsTest.cs
@@ -79,7 +79,7 @@ namespace Grpc.IntegrationTesting
             };
 
             channel = new Channel(Host, server.Ports.Single().BoundPort, clientCredentials, options);
-            client = TestService.NewClient(channel);
+            client = new TestService.TestServiceClient(channel);
         }
 
         [TestFixtureTearDown]
diff --git a/src/csharp/Grpc.IntegrationTesting/StressTestClient.cs b/src/csharp/Grpc.IntegrationTesting/StressTestClient.cs
index 8db691cb04..74ee040ae4 100644
--- a/src/csharp/Grpc.IntegrationTesting/StressTestClient.cs
+++ b/src/csharp/Grpc.IntegrationTesting/StressTestClient.cs
@@ -148,7 +148,7 @@ namespace Grpc.IntegrationTesting
                     channels.Add(channel);
                     for (int j = 0; j < options.NumStubsPerChannel; j++)
                     {
-                        var client = TestService.NewClient(channel);
+                        var client = new TestService.TestServiceClient(channel);
                         var task = Task.Factory.StartNew(() => RunBodyAsync(client).GetAwaiter().GetResult(),
                             TaskCreationOptions.LongRunning);
                         tasks.Add(task);  
@@ -311,7 +311,7 @@ namespace Grpc.IntegrationTesting
                 var snapshot = histogram.GetSnapshot(true);
                 var elapsedSnapshot = wallClockStopwatch.GetElapsedSnapshot(true);
 
-                return (long) (snapshot.Count / elapsedSnapshot.Seconds);
+                return (long) (snapshot.Count / elapsedSnapshot.TotalSeconds);
             }
         }
     }
diff --git a/src/csharp/Grpc.IntegrationTesting/TestCredentials.cs b/src/csharp/Grpc.IntegrationTesting/TestCredentials.cs
index 774563d752..60b9cf4e0b 100644
--- a/src/csharp/Grpc.IntegrationTesting/TestCredentials.cs
+++ b/src/csharp/Grpc.IntegrationTesting/TestCredentials.cs
@@ -90,7 +90,7 @@ namespace Grpc.IntegrationTesting
 
         private static string GetPath(string relativePath)
         {
-            var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+            var assemblyDir = Path.GetDirectoryName(typeof(TestCredentials).GetTypeInfo().Assembly.Location);
             return Path.Combine(assemblyDir, relativePath);
         }
     }
diff --git a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs
index cf43a77118..3e149da3e0 100644
--- a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs
+++ b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs
@@ -105,127 +105,6 @@ namespace Grpc.Testing {
       get { return global::Grpc.Testing.TestReflection.Descriptor.Services[0]; }
     }
 
-    /// <summary>Client for TestService</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface ITestServiceClient
-    {
-      /// <summary>
-      ///  One empty request followed by one empty response.
-      /// </summary>
-      global::Grpc.Testing.Empty EmptyCall(global::Grpc.Testing.Empty request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One empty request followed by one empty response.
-      /// </summary>
-      global::Grpc.Testing.Empty EmptyCall(global::Grpc.Testing.Empty request, CallOptions options);
-      /// <summary>
-      ///  One empty request followed by one empty response.
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.Empty> EmptyCallAsync(global::Grpc.Testing.Empty request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One empty request followed by one empty response.
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.Empty> EmptyCallAsync(global::Grpc.Testing.Empty request, CallOptions options);
-      /// <summary>
-      ///  One request followed by one response.
-      /// </summary>
-      global::Grpc.Testing.SimpleResponse UnaryCall(global::Grpc.Testing.SimpleRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One request followed by one response.
-      /// </summary>
-      global::Grpc.Testing.SimpleResponse UnaryCall(global::Grpc.Testing.SimpleRequest request, CallOptions options);
-      /// <summary>
-      ///  One request followed by one response.
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.SimpleResponse> UnaryCallAsync(global::Grpc.Testing.SimpleRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One request followed by one response.
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.SimpleResponse> UnaryCallAsync(global::Grpc.Testing.SimpleRequest request, CallOptions options);
-      /// <summary>
-      ///  One request followed by a sequence of responses (streamed download).
-      ///  The server returns the payload with client desired type and sizes.
-      /// </summary>
-      AsyncServerStreamingCall<global::Grpc.Testing.StreamingOutputCallResponse> StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  One request followed by a sequence of responses (streamed download).
-      ///  The server returns the payload with client desired type and sizes.
-      /// </summary>
-      AsyncServerStreamingCall<global::Grpc.Testing.StreamingOutputCallResponse> StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, CallOptions options);
-      /// <summary>
-      ///  A sequence of requests followed by one response (streamed upload).
-      ///  The server returns the aggregated size of client payload as the result.
-      /// </summary>
-      AsyncClientStreamingCall<global::Grpc.Testing.StreamingInputCallRequest, global::Grpc.Testing.StreamingInputCallResponse> StreamingInputCall(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  A sequence of requests followed by one response (streamed upload).
-      ///  The server returns the aggregated size of client payload as the result.
-      /// </summary>
-      AsyncClientStreamingCall<global::Grpc.Testing.StreamingInputCallRequest, global::Grpc.Testing.StreamingInputCallResponse> StreamingInputCall(CallOptions options);
-      /// <summary>
-      ///  A sequence of requests with each request served by the server immediately.
-      ///  As one request could lead to multiple responses, this interface
-      ///  demonstrates the idea of full duplexing.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.StreamingOutputCallRequest, global::Grpc.Testing.StreamingOutputCallResponse> FullDuplexCall(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  A sequence of requests with each request served by the server immediately.
-      ///  As one request could lead to multiple responses, this interface
-      ///  demonstrates the idea of full duplexing.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.StreamingOutputCallRequest, global::Grpc.Testing.StreamingOutputCallResponse> FullDuplexCall(CallOptions options);
-      /// <summary>
-      ///  A sequence of requests followed by a sequence of responses.
-      ///  The server buffers all the client requests and then serves them in order. A
-      ///  stream of responses are returned to the client when the server starts with
-      ///  first request.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.StreamingOutputCallRequest, global::Grpc.Testing.StreamingOutputCallResponse> HalfDuplexCall(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  A sequence of requests followed by a sequence of responses.
-      ///  The server buffers all the client requests and then serves them in order. A
-      ///  stream of responses are returned to the client when the server starts with
-      ///  first request.
-      /// </summary>
-      AsyncDuplexStreamingCall<global::Grpc.Testing.StreamingOutputCallRequest, global::Grpc.Testing.StreamingOutputCallResponse> HalfDuplexCall(CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of TestService</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface ITestService
-    {
-      /// <summary>
-      ///  One empty request followed by one empty response.
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.Empty> EmptyCall(global::Grpc.Testing.Empty request, ServerCallContext context);
-      /// <summary>
-      ///  One request followed by one response.
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.SimpleResponse> UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context);
-      /// <summary>
-      ///  One request followed by a sequence of responses (streamed download).
-      ///  The server returns the payload with client desired type and sizes.
-      /// </summary>
-      global::System.Threading.Tasks.Task StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, IServerStreamWriter<global::Grpc.Testing.StreamingOutputCallResponse> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  A sequence of requests followed by one response (streamed upload).
-      ///  The server returns the aggregated size of client payload as the result.
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.StreamingInputCallResponse> StreamingInputCall(IAsyncStreamReader<global::Grpc.Testing.StreamingInputCallRequest> requestStream, ServerCallContext context);
-      /// <summary>
-      ///  A sequence of requests with each request served by the server immediately.
-      ///  As one request could lead to multiple responses, this interface
-      ///  demonstrates the idea of full duplexing.
-      /// </summary>
-      global::System.Threading.Tasks.Task FullDuplexCall(IAsyncStreamReader<global::Grpc.Testing.StreamingOutputCallRequest> requestStream, IServerStreamWriter<global::Grpc.Testing.StreamingOutputCallResponse> responseStream, ServerCallContext context);
-      /// <summary>
-      ///  A sequence of requests followed by a sequence of responses.
-      ///  The server buffers all the client requests and then serves them in order. A
-      ///  stream of responses are returned to the client when the server starts with
-      ///  first request.
-      /// </summary>
-      global::System.Threading.Tasks.Task HalfDuplexCall(IAsyncStreamReader<global::Grpc.Testing.StreamingOutputCallRequest> requestStream, IServerStreamWriter<global::Grpc.Testing.StreamingOutputCallResponse> responseStream, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of TestService</summary>
     public abstract class TestServiceBase
     {
@@ -287,21 +166,24 @@ namespace Grpc.Testing {
     }
 
     /// <summary>Client for TestService</summary>
-    #pragma warning disable 0618
-    public class TestServiceClient : ClientBase<TestServiceClient>, ITestServiceClient
-    #pragma warning restore 0618
+    public class TestServiceClient : ClientBase<TestServiceClient>
     {
+      /// <summary>Creates a new client for TestService</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public TestServiceClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for TestService that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public TestServiceClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected TestServiceClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected TestServiceClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -438,32 +320,10 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Creates a new client for TestService</summary>
-    public static TestServiceClient NewClient(Channel channel)
-    {
-      return new TestServiceClient(channel);
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(ITestService serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_EmptyCall, serviceImpl.EmptyCall)
-          .AddMethod(__Method_UnaryCall, serviceImpl.UnaryCall)
-          .AddMethod(__Method_StreamingOutputCall, serviceImpl.StreamingOutputCall)
-          .AddMethod(__Method_StreamingInputCall, serviceImpl.StreamingInputCall)
-          .AddMethod(__Method_FullDuplexCall, serviceImpl.FullDuplexCall)
-          .AddMethod(__Method_HalfDuplexCall, serviceImpl.HalfDuplexCall).Build();
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(TestServiceBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_EmptyCall, serviceImpl.EmptyCall)
           .AddMethod(__Method_UnaryCall, serviceImpl.UnaryCall)
           .AddMethod(__Method_StreamingOutputCall, serviceImpl.StreamingOutputCall)
@@ -496,38 +356,6 @@ namespace Grpc.Testing {
       get { return global::Grpc.Testing.TestReflection.Descriptor.Services[1]; }
     }
 
-    /// <summary>Client for UnimplementedService</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IUnimplementedServiceClient
-    {
-      /// <summary>
-      ///  A call that no server should implement
-      /// </summary>
-      global::Grpc.Testing.Empty UnimplementedCall(global::Grpc.Testing.Empty request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  A call that no server should implement
-      /// </summary>
-      global::Grpc.Testing.Empty UnimplementedCall(global::Grpc.Testing.Empty request, CallOptions options);
-      /// <summary>
-      ///  A call that no server should implement
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.Empty> UnimplementedCallAsync(global::Grpc.Testing.Empty request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      /// <summary>
-      ///  A call that no server should implement
-      /// </summary>
-      AsyncUnaryCall<global::Grpc.Testing.Empty> UnimplementedCallAsync(global::Grpc.Testing.Empty request, CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of UnimplementedService</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IUnimplementedService
-    {
-      /// <summary>
-      ///  A call that no server should implement
-      /// </summary>
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.Empty> UnimplementedCall(global::Grpc.Testing.Empty request, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of UnimplementedService</summary>
     public abstract class UnimplementedServiceBase
     {
@@ -542,21 +370,24 @@ namespace Grpc.Testing {
     }
 
     /// <summary>Client for UnimplementedService</summary>
-    #pragma warning disable 0618
-    public class UnimplementedServiceClient : ClientBase<UnimplementedServiceClient>, IUnimplementedServiceClient
-    #pragma warning restore 0618
+    public class UnimplementedServiceClient : ClientBase<UnimplementedServiceClient>
     {
+      /// <summary>Creates a new client for UnimplementedService</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public UnimplementedServiceClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for UnimplementedService that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public UnimplementedServiceClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected UnimplementedServiceClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected UnimplementedServiceClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -595,27 +426,10 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Creates a new client for UnimplementedService</summary>
-    public static UnimplementedServiceClient NewClient(Channel channel)
-    {
-      return new UnimplementedServiceClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IUnimplementedService serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_UnimplementedCall, serviceImpl.UnimplementedCall).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(UnimplementedServiceBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_UnimplementedCall, serviceImpl.UnimplementedCall).Build();
     }
 
@@ -651,28 +465,6 @@ namespace Grpc.Testing {
       get { return global::Grpc.Testing.TestReflection.Descriptor.Services[2]; }
     }
 
-    /// <summary>Client for ReconnectService</summary>
-    [System.Obsolete("Client side interfaced will be removed in the next release. Use client class directly.")]
-    public interface IReconnectServiceClient
-    {
-      global::Grpc.Testing.Empty Start(global::Grpc.Testing.ReconnectParams request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      global::Grpc.Testing.Empty Start(global::Grpc.Testing.ReconnectParams request, CallOptions options);
-      AsyncUnaryCall<global::Grpc.Testing.Empty> StartAsync(global::Grpc.Testing.ReconnectParams request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      AsyncUnaryCall<global::Grpc.Testing.Empty> StartAsync(global::Grpc.Testing.ReconnectParams request, CallOptions options);
-      global::Grpc.Testing.ReconnectInfo Stop(global::Grpc.Testing.Empty request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      global::Grpc.Testing.ReconnectInfo Stop(global::Grpc.Testing.Empty request, CallOptions options);
-      AsyncUnaryCall<global::Grpc.Testing.ReconnectInfo> StopAsync(global::Grpc.Testing.Empty request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken));
-      AsyncUnaryCall<global::Grpc.Testing.ReconnectInfo> StopAsync(global::Grpc.Testing.Empty request, CallOptions options);
-    }
-
-    /// <summary>Interface of server-side implementations of ReconnectService</summary>
-    [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")]
-    public interface IReconnectService
-    {
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.Empty> Start(global::Grpc.Testing.ReconnectParams request, ServerCallContext context);
-      global::System.Threading.Tasks.Task<global::Grpc.Testing.ReconnectInfo> Stop(global::Grpc.Testing.Empty request, ServerCallContext context);
-    }
-
     /// <summary>Base class for server-side implementations of ReconnectService</summary>
     public abstract class ReconnectServiceBase
     {
@@ -689,21 +481,24 @@ namespace Grpc.Testing {
     }
 
     /// <summary>Client for ReconnectService</summary>
-    #pragma warning disable 0618
-    public class ReconnectServiceClient : ClientBase<ReconnectServiceClient>, IReconnectServiceClient
-    #pragma warning restore 0618
+    public class ReconnectServiceClient : ClientBase<ReconnectServiceClient>
     {
+      /// <summary>Creates a new client for ReconnectService</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
       public ReconnectServiceClient(Channel channel) : base(channel)
       {
       }
+      /// <summary>Creates a new client for ReconnectService that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
       public ReconnectServiceClient(CallInvoker callInvoker) : base(callInvoker)
       {
       }
-      ///<summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
       protected ReconnectServiceClient() : base()
       {
       }
-      ///<summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
       protected ReconnectServiceClient(ClientBaseConfiguration configuration) : base(configuration)
       {
       }
@@ -746,28 +541,10 @@ namespace Grpc.Testing {
       }
     }
 
-    /// <summary>Creates a new client for ReconnectService</summary>
-    public static ReconnectServiceClient NewClient(Channel channel)
-    {
-      return new ReconnectServiceClient(channel);
-    }
-
-    /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
-    public static ServerServiceDefinition BindService(IReconnectService serviceImpl)
-    #pragma warning restore 0618
-    {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
-          .AddMethod(__Method_Start, serviceImpl.Start)
-          .AddMethod(__Method_Stop, serviceImpl.Stop).Build();
-    }
-
     /// <summary>Creates service definition that can be registered with a server</summary>
-    #pragma warning disable 0618
     public static ServerServiceDefinition BindService(ReconnectServiceBase serviceImpl)
-    #pragma warning restore 0618
     {
-      return ServerServiceDefinition.CreateBuilder(__ServiceName)
+      return ServerServiceDefinition.CreateBuilder()
           .AddMethod(__Method_Start, serviceImpl.Start)
           .AddMethod(__Method_Stop, serviceImpl.Stop).Build();
     }
diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
index 80dad9fdd9..c9eca73452 100644
--- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
+++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
@@ -64,7 +64,7 @@ namespace Grpc.Testing
             {
                 Stats = runner.GetStats(false),
                 Port = runner.BoundPort,
-                Cores = 0,  // TODO: set number of cores
+                Cores = Environment.ProcessorCount,
             });
                 
             while (await requestStream.MoveNext())
diff --git a/src/csharp/Grpc.IntegrationTesting/project.json b/src/csharp/Grpc.IntegrationTesting/project.json
new file mode 100644
index 0000000000..6297600ddc
--- /dev/null
+++ b/src/csharp/Grpc.IntegrationTesting/project.json
@@ -0,0 +1,86 @@
+{
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+  "configurations": {
+    "Debug": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/dbg/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    },
+    "Release": {
+      "buildOptions": {
+        "define": [ "SIGNED" ],
+        "keyFile": "../keys/Grpc.snk",
+        "publicSign": true,
+        "xmlDoc": true,
+        "compile": {
+          "includeFiles": [ "../Grpc.Core/Version.cs" ]
+        },
+        "copyToOutput": {
+          "include": "data/*",
+          "mappings": {
+            "nativelibs/windows_x64/grpc_csharp_ext.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll",
+            "nativelibs/windows_x86/grpc_csharp_ext.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll",
+            "nativelibs/linux_x64/libgrpc_csharp_ext.so": "../../../libs/opt/libgrpc_csharp_ext.so",
+            "nativelibs/macosx_x64/libgrpc_csharp_ext.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib"
+          }
+        }
+      }
+    }
+  },
+  "runtimes": {
+    "win7-x64": { },
+    "debian.8-x64": { },
+    "osx.10.11-x64": { }
+  },
+
+  "dependencies": {
+    "Grpc.Auth": {
+      "target": "project"
+    },
+    "Grpc.Core": {
+      "target": "project"
+    },
+    "Google.Protobuf": "3.0.0-beta3",
+    "CommandLineParser": "1.9.71",
+    "NUnit": "3.2.0",
+    "NUnitLite": "3.2.0-*"
+  },
+  "frameworks": {
+    "net45": {
+      "dependencies": {
+        "Moq": "4.2.1510.2205"
+      },
+      "frameworkAssemblies": {
+        "System.Runtime": "",
+        "System.IO": ""
+      }
+    },
+    "netstandard1.5": {
+      "imports": [
+        "portable-net45",
+        "net45"
+      ],
+      "dependencies": {
+        "NETStandard.Library": "1.5.0-rc2-24027",
+        "System.Linq.Expressions": "4.0.11-rc2-24027"
+      }
+    }
+  }
+}
diff --git a/src/csharp/README.md b/src/csharp/README.md
index 201c5ab0b5..86394135c8 100644
--- a/src/csharp/README.md
+++ b/src/csharp/README.md
@@ -19,33 +19,13 @@ PREREQUISITES
 HOW TO USE
 --------------
 
-**Windows**
+**Windows, Linux, Mac OS X**
 
-- Open Visual Studio and start a new project/solution.
+- Open Visual Studio / MonoDevelop / Xamarin Studio and start a new project/solution.
 
 - Add NuGet package `Grpc` as a dependency (Project options -> Manage NuGet Packages).
-  That will also pull all the transitive dependencies (including the gRPC native library that
-  gRPC C# is using internally).
-
-**Linux (Debian)**
-
-- Open MonoDevelop and start a new project/solution.
-
-- Add NuGet package `Grpc` as a dependency (Project -> Add NuGet packages).
-  That will also pull all the transitive dependencies (including the gRPC native library that
-  gRPC C# is using internally).
-
-- NOTE: gRPC C# doesn't have a good story yet for shipping precompiled Linux version of Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin. You can install them using [gRPC Linuxbrew instructions][].
 
-**Mac OS X**
-
-- Open Xamarin Studio and start a new project/solution.
-
-- Add NuGet package `Grpc` as a dependency (Project -> Add NuGet packages).
-  That will also pull all the transitive dependencies (including the gRPC native library that
-  gRPC C# is using internally).
-
-- NOTE: gRPC C# doesn't have a good story yet for shipping precompiled Mac OS X version of Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin. You can install them using [gRPC Homebrew instructions][].
+- To be able to generate code from Protocol Buffer (`.proto`) file definitions, add NuGet package `Grpc.Tools` that contains Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin.
 
 BUILD FROM SOURCE
 -----------------
@@ -61,26 +41,15 @@ If you are a user of gRPC C#, go to Usage section above.
 - Open `src\csharp\Grpc.sln` (path is relative to gRPC repository root)
   using Visual Studio
 
-**Linux**
+**Linux and Mac OS X**
 
 - The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution:
   ```sh
   # from the gRPC repository root
-  $ make CONFIG=dbg grpc_csharp_ext
-  ```
-
-- Use MonoDevelop to open the solution Grpc.sln
-
-**Mac OS X**
-
-- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution.
-
-  ```sh
-  # from the gRPC repository root
   $ tools/run_tests/run_tests.py -c dbg -l csharp --build_only
   ```
 
-- Use Xamarin Studio to open the solution Grpc.sln
+- Use MonoDevelop / Xamarin Studio to open the solution Grpc.sln
 
 RUNNING TESTS
 -------------
@@ -100,10 +69,17 @@ different languages.
 tools/run_tests/run_tests.py -l csharp
 ```
 
+ON .NET CORE SUPPORT
+------------------
+
+We are committed to providing full support for [.NET Core](https://dotnet.github.io/) in near future,
+but currently, the support is for .NET Core is experimental/work-in-progress.
+
 DOCUMENTATION
 -------------
-- the gRPC C# reference documentation is available online at [grpc.io][]
-- [Helloworld example][]
+- [API Reference][]
+- [Helloworld Example][]
+- [RouteGuide Tutorial][]
 
 CONTENTS
 --------
@@ -111,15 +87,15 @@ CONTENTS
 - ext:
   The extension library that wraps C API to be more digestible by C#.
 - Grpc.Auth:
-  gRPC OAuth2 support.
+  gRPC OAuth2/JWT support.
 - Grpc.Core:
   The main gRPC C# library.
 - Grpc.Examples:
   API examples for math.proto
 - Grpc.Examples.MathClient:
-  An example client that sends some requests to math server.
+  An example client that sends requests to math server.
 - Grpc.Examples.MathServer:
-  An example client that sends some requests to math server.
+  An example server that implements a simple math service.
 - Grpc.IntegrationTesting:
   Cross-language gRPC implementation testing (interop testing).
 
@@ -130,10 +106,6 @@ Internally, gRPC C# uses a native library written in C (gRPC C core) and invokes
 
 Prior to version 0.13, installing `grpc_csharp_ext` was required to make gRPC work on Linux and MacOS. Starting with version 0.13, we have improved the packaging story significantly and precompiled versions of the native library for all supported platforms are now shipped with the NuGet package. Just installing the `Grpc` NuGet package should be the only step needed to use gRPC C#, regardless of your platform (Windows, Linux or Mac) and the bitness (32 or 64bit).
 
-[gRPC Linuxbrew instructions]:https://github.com/grpc/homebrew-grpc#quick-install-linux
-[gRPC Homebrew instructions]:https://github.com/grpc/homebrew-grpc#quick-install-linux
-[homebrew]:http://brew.sh
-[gRPC install script]:https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install
-[grpc.io]: http://www.grpc.io/docs/installation/csharp.html
-[Debian jessie-backports]:http://backports.debian.org/Instructions/
-[Helloworld example]:../../examples/csharp/helloworld
+[API Reference]: http://www.grpc.io/grpc/csharp/
+[Helloworld Example]: ../../examples/csharp/helloworld
+[RouteGuide Tutorial]: http://www.grpc.io/docs/tutorials/basic/csharp.html 
diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat
index 90f4c58d03..272b30f385 100644
--- a/src/csharp/build_packages.bat
+++ b/src/csharp/build_packages.bat
@@ -1,7 +1,36 @@
+@rem Copyright 2016, Google Inc.
+@rem All rights reserved.
+@rem
+@rem Redistribution and use in source and binary forms, with or without
+@rem modification, are permitted provided that the following conditions are
+@rem met:
+@rem
+@rem     * Redistributions of source code must retain the above copyright
+@rem notice, this list of conditions and the following disclaimer.
+@rem     * Redistributions in binary form must reproduce the above
+@rem copyright notice, this list of conditions and the following disclaimer
+@rem in the documentation and/or other materials provided with the
+@rem distribution.
+@rem     * Neither the name of Google Inc. nor the names of its
+@rem contributors may be used to endorse or promote products derived from
+@rem this software without specific prior written permission.
+@rem
+@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 @rem Builds gRPC NuGet packages
 
 @rem Current package versions
-set VERSION=0.14.2-pre1
+set VERSION=0.16.0-dev
 set PROTOBUF_VERSION=3.0.0-beta3
 
 @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well.
@@ -12,12 +41,12 @@ set NUGET=C:\nuget\nuget.exe
 
 @rem Collect the artifacts built by the previous build step if running on Jenkins
 @rem TODO(jtattermusch): is there a better way to do this?
-xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=windows\artifacts\* Grpc.Core\windows_x86\
-xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=windows\artifacts\* Grpc.Core\windows_x64\
-xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=linux\artifacts\* Grpc.Core\linux_x86\
-xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=linux\artifacts\* Grpc.Core\linux_x64\
-xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=macos\artifacts\* Grpc.Core\macosx_x86\
-xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=macos\artifacts\* Grpc.Core\macosx_x64\
+xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=windows\artifacts\* nativelibs\windows_x86\
+xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=windows\artifacts\* nativelibs\windows_x64\
+xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=linux\artifacts\* nativelibs\linux_x86\
+xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=linux\artifacts\* nativelibs\linux_x64\
+xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=macos\artifacts\* nativelibs\macosx_x86\
+xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=macos\artifacts\* nativelibs\macosx_x64\
 
 @rem Collect protoc artifacts built by the previous build step
 xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=windows\artifacts\* protoc_plugins\windows_x86\
diff --git a/src/csharp/buildall.bat b/src/csharp/buildall.bat
index f800756dfe..0beb30c198 100644
--- a/src/csharp/buildall.bat
+++ b/src/csharp/buildall.bat
@@ -1,3 +1,32 @@
+@rem Copyright 2016, Google Inc.
+@rem All rights reserved.
+@rem
+@rem Redistribution and use in source and binary forms, with or without
+@rem modification, are permitted provided that the following conditions are
+@rem met:
+@rem
+@rem     * Redistributions of source code must retain the above copyright
+@rem notice, this list of conditions and the following disclaimer.
+@rem     * Redistributions in binary form must reproduce the above
+@rem copyright notice, this list of conditions and the following disclaimer
+@rem in the documentation and/or other materials provided with the
+@rem distribution.
+@rem     * Neither the name of Google Inc. nor the names of its
+@rem contributors may be used to endorse or promote products derived from
+@rem this software without specific prior written permission.
+@rem
+@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 @rem Convenience script to build gRPC C# from command line
 
 setlocal
diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c
index 5b8ff9b819..9b8d050ea5 100644
--- a/src/csharp/ext/grpc_csharp_ext.c
+++ b/src/csharp/ext/grpc_csharp_ext.c
@@ -45,7 +45,7 @@
 
 #include <string.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 #define GPR_EXPORT __declspec(dllexport)
 #define GPR_CALLTYPE __stdcall
 #endif
@@ -249,10 +249,12 @@ grpcsharp_batch_context_recv_initial_metadata(
 
 GPR_EXPORT intptr_t GPR_CALLTYPE grpcsharp_batch_context_recv_message_length(
     const grpcsharp_batch_context *ctx) {
+  grpc_byte_buffer_reader reader;
   if (!ctx->recv_message) {
     return -1;
   }
-  return (intptr_t)grpc_byte_buffer_length(ctx->recv_message);
+  grpc_byte_buffer_reader_init(&reader, ctx->recv_message);
+  return (intptr_t)grpc_byte_buffer_length(reader.buffer_out);
 }
 
 /*
@@ -503,6 +505,7 @@ grpcsharp_call_start_unary(grpc_call *call, grpcsharp_batch_context *ctx,
                            grpc_metadata_array *initial_metadata, uint32_t write_flags) {
   /* TODO: don't use magic number */
   grpc_op ops[6];
+  memset(ops, 0, sizeof(ops));
   ops[0].op = GRPC_OP_SEND_INITIAL_METADATA;
   grpcsharp_metadata_array_move(&(ctx->send_initial_metadata),
                                 initial_metadata);
@@ -555,6 +558,7 @@ grpcsharp_call_start_client_streaming(grpc_call *call,
                                       grpc_metadata_array *initial_metadata) {
   /* TODO: don't use magic number */
   grpc_op ops[4];
+  memset(ops, 0, sizeof(ops));
   ops[0].op = GRPC_OP_SEND_INITIAL_METADATA;
   grpcsharp_metadata_array_move(&(ctx->send_initial_metadata),
                                 initial_metadata);
@@ -596,6 +600,7 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_start_server_streaming(
     size_t send_buffer_len, grpc_metadata_array *initial_metadata, uint32_t write_flags) {
   /* TODO: don't use magic number */
   grpc_op ops[4];
+  memset(ops, 0, sizeof(ops));
   ops[0].op = GRPC_OP_SEND_INITIAL_METADATA;
   grpcsharp_metadata_array_move(&(ctx->send_initial_metadata),
                                 initial_metadata);
@@ -638,6 +643,7 @@ grpcsharp_call_start_duplex_streaming(grpc_call *call,
                                       grpc_metadata_array *initial_metadata) {
   /* TODO: don't use magic number */
   grpc_op ops[2];
+  memset(ops, 0, sizeof(ops));
   ops[0].op = GRPC_OP_SEND_INITIAL_METADATA;
   grpcsharp_metadata_array_move(&(ctx->send_initial_metadata),
                                 initial_metadata);
@@ -684,6 +690,7 @@ grpcsharp_call_send_message(grpc_call *call, grpcsharp_batch_context *ctx,
                             int32_t send_empty_initial_metadata) {
   /* TODO: don't use magic number */
   grpc_op ops[2];
+  memset(ops, 0, sizeof(ops));
   size_t nops = send_empty_initial_metadata ? 2 : 1;
   ops[0].op = GRPC_OP_SEND_MESSAGE;
   ctx->send_message = string_to_byte_buffer(send_buffer, send_buffer_len);
@@ -691,8 +698,6 @@ grpcsharp_call_send_message(grpc_call *call, grpcsharp_batch_context *ctx,
   ops[0].flags = write_flags;
   ops[0].reserved = NULL;
   ops[1].op = GRPC_OP_SEND_INITIAL_METADATA;
-  ops[1].data.send_initial_metadata.count = 0;
-  ops[1].data.send_initial_metadata.metadata = NULL;
   ops[1].flags = 0;
   ops[1].reserved = NULL;
 
@@ -719,6 +724,7 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_send_status_from_server(
     size_t optional_send_buffer_len, uint32_t write_flags) {
   /* TODO: don't use magic number */
   grpc_op ops[3];
+  memset(ops, 0, sizeof(ops));
   size_t nops = 1;
   ops[0].op = GRPC_OP_SEND_STATUS_FROM_SERVER;
   ops[0].data.send_status_from_server.status = status_code;
@@ -743,8 +749,6 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_send_status_from_server(
   }
   if (send_empty_initial_metadata) {
     ops[nops].op = GRPC_OP_SEND_INITIAL_METADATA;
-    ops[nops].data.send_initial_metadata.count = 0;
-    ops[nops].data.send_initial_metadata.metadata = NULL;
     ops[nops].flags = 0;
     ops[nops].reserved = NULL;
     nops++;
@@ -784,6 +788,7 @@ grpcsharp_call_send_initial_metadata(grpc_call *call,
                                      grpc_metadata_array *initial_metadata) {
   /* TODO: don't use magic number */
   grpc_op ops[1];
+  memset(ops, 0, sizeof(ops));
   ops[0].op = GRPC_OP_SEND_INITIAL_METADATA;
   grpcsharp_metadata_array_move(&(ctx->send_initial_metadata),
                                 initial_metadata);
@@ -806,11 +811,14 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_set_credentials(
 /* Server */
 
 GPR_EXPORT grpc_server *GPR_CALLTYPE
-grpcsharp_server_create(grpc_completion_queue *cq,
-                        const grpc_channel_args *args) {
-  grpc_server *server = grpc_server_create(args, NULL);
+grpcsharp_server_create(const grpc_channel_args *args) {
+  return grpc_server_create(args, NULL);
+}
+
+GPR_EXPORT void GPR_CALLTYPE
+grpcsharp_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq) {
   grpc_server_register_completion_queue(server, cq, NULL);
-  return server;
 }
 
 GPR_EXPORT int32_t GPR_CALLTYPE
diff --git a/src/csharp/grpc.native.csharp/grpc.native.csharp.nuspec b/src/csharp/grpc.native.csharp/grpc.native.csharp.nuspec
deleted file mode 100644
index cc688e2bc7..0000000000
--- a/src/csharp/grpc.native.csharp/grpc.native.csharp.nuspec
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<package>
-  <metadata>
-    <id>grpc.native.csharp</id>
-    <version>$version$</version>
-    <authors>Google Inc.</authors>
-    <owners>grpc-packages</owners>
-    <licenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</licenseUrl>
-    <projectUrl>http://github.com/grpc/grpc</projectUrl>
-    <requireLicenseAcceptance>false</requireLicenseAcceptance>
-    <description>Native extension needed by gRPC C# library. This is not the package you are looking for, it is only meant to be used as a dependency.</description>
-    <releaseNotes>Release of gRPC C core $version$ libraries.</releaseNotes>
-    <copyright>Copyright 2015</copyright>
-    <title>gRPC C# Native Extension</title>
-    <summary>Native library required by gRPC C#</summary>
-    <tags>gRPC native</tags>
-  </metadata>
-  <files>
-    <file src="grpc.native.csharp.targets" target="\build\portable-net45+netcore45+wpa81+wp8\grpc.native.csharp.targets" />
-    <file src="windows_x86/grpc_csharp_ext.dll" target="/build/native/bin/windows_x86/grpc_csharp_ext.dll" />
-    <file src="windows_x64/grpc_csharp_ext.dll" target="/build/native/bin/windows_x64/grpc_csharp_ext.dll" />
-    <file src="linux_x86/libgrpc_csharp_ext.so" target="/build/native/bin/linux_x86/libgrpc_csharp_ext.so" />
-    <file src="linux_x64/libgrpc_csharp_ext.so" target="/build/native/bin/linux_x64/libgrpc_csharp_ext.so" />
-    <file src="macosx_x86/libgrpc_csharp_ext.dylib" target="/build/native/bin/macosx_x86/libgrpc_csharp_ext.dylib" />
-    <file src="macosx_x64/libgrpc_csharp_ext.dylib" target="/build/native/bin/macosx_x64/libgrpc_csharp_ext.dylib" />
-  </files>
-</package>
diff --git a/src/csharp/tests.json b/src/csharp/tests.json
index f6af3408d5..7e7aee1093 100644
--- a/src/csharp/tests.json
+++ b/src/csharp/tests.json
@@ -7,6 +7,7 @@
     "Grpc.Core.Internal.Tests.CompletionQueueSafeHandleTest",
     "Grpc.Core.Internal.Tests.MetadataArraySafeHandleTest",
     "Grpc.Core.Internal.Tests.TimespecTest",
+    "Grpc.Core.Tests.AppDomainUnloadTest",
     "Grpc.Core.Tests.CallCredentialsTest",
     "Grpc.Core.Tests.CallOptionsTest",
     "Grpc.Core.Tests.ChannelCredentialsTest",
@@ -25,6 +26,9 @@
     "Grpc.Core.Tests.ResponseHeadersTest",
     "Grpc.Core.Tests.SanityTest",
     "Grpc.Core.Tests.ServerTest",
+    "Grpc.Core.Tests.ShutdownHookClientTest",
+    "Grpc.Core.Tests.ShutdownHookPendingCallTest",
+    "Grpc.Core.Tests.ShutdownHookServerTest",
     "Grpc.Core.Tests.ShutdownTest",
     "Grpc.Core.Tests.TimeoutsTest",
     "Grpc.Core.Tests.UserAgentStringTest"
diff --git a/src/node/ext/byte_buffer.cc b/src/node/ext/byte_buffer.cc
index 8e0b6916e9..3479a67702 100644
--- a/src/node/ext/byte_buffer.cc
+++ b/src/node/ext/byte_buffer.cc
@@ -72,17 +72,13 @@ Local<Value> ByteBufferToBuffer(grpc_byte_buffer *buffer) {
   if (buffer == NULL) {
     return scope.Escape(Nan::Null());
   }
-  size_t length = grpc_byte_buffer_length(buffer);
-  char *result = new char[length];
-  size_t offset = 0;
   grpc_byte_buffer_reader reader;
   grpc_byte_buffer_reader_init(&reader, buffer);
-  gpr_slice next;
-  while (grpc_byte_buffer_reader_next(&reader, &next) != 0) {
-    memcpy(result + offset, GPR_SLICE_START_PTR(next), GPR_SLICE_LENGTH(next));
-    offset += GPR_SLICE_LENGTH(next);
-    gpr_slice_unref(next);
-  }
+  gpr_slice slice = grpc_byte_buffer_reader_readall(&reader);
+  size_t length = GPR_SLICE_LENGTH(slice);
+  char *result = new char[length];
+  memcpy(result, GPR_SLICE_START_PTR(slice), length);
+  gpr_slice_unref(slice);
   return scope.Escape(MakeFastBuffer(
       Nan::NewBuffer(result, length, delete_buffer, NULL).ToLocalChecked()));
 }
diff --git a/src/node/ext/node_grpc.cc b/src/node/ext/node_grpc.cc
index 6b6e42737b..745b5023d5 100644
--- a/src/node/ext/node_grpc.cc
+++ b/src/node/ext/node_grpc.cc
@@ -31,12 +31,16 @@
  *
  */
 
+#include <list>
+
 #include <node.h>
 #include <nan.h>
 #include <v8.h>
 #include "grpc/grpc.h"
 #include "grpc/grpc_security.h"
 #include "grpc/support/alloc.h"
+#include "grpc/support/log.h"
+#include "grpc/support/time.h"
 
 #include "call.h"
 #include "call_credentials.h"
@@ -45,14 +49,32 @@
 #include "server.h"
 #include "completion_queue_async_worker.h"
 #include "server_credentials.h"
+#include "timeval.h"
 
 using v8::FunctionTemplate;
 using v8::Local;
 using v8::Value;
+using v8::Number;
 using v8::Object;
 using v8::Uint32;
 using v8::String;
 
+typedef struct log_args {
+  gpr_log_func_args core_args;
+  gpr_timespec timestamp;
+} log_args;
+
+typedef struct logger_state {
+  Nan::Callback *callback;
+  std::list<log_args *> *pending_args;
+  uv_mutex_t mutex;
+  uv_async_t async;
+  // Indicates that a logger has been set
+  bool logger_set;
+} logger_state;
+
+logger_state grpc_logger_state;
+
 static char *pem_root_certs = NULL;
 
 void InitStatusConstants(Local<Object> exports) {
@@ -220,7 +242,7 @@ void InitConnectivityStateConstants(Local<Object> exports) {
   Nan::Set(channel_state, Nan::New("TRANSIENT_FAILURE").ToLocalChecked(),
            TRANSIENT_FAILURE);
   Local<Value> FATAL_FAILURE(
-      Nan::New<Uint32, uint32_t>(GRPC_CHANNEL_FATAL_FAILURE));
+      Nan::New<Uint32, uint32_t>(GRPC_CHANNEL_SHUTDOWN));
   Nan::Set(channel_state, Nan::New("FATAL_FAILURE").ToLocalChecked(),
            FATAL_FAILURE);
 }
@@ -235,6 +257,18 @@ void InitWriteFlags(Local<Object> exports) {
   Nan::Set(write_flags, Nan::New("NO_COMPRESS").ToLocalChecked(), NO_COMPRESS);
 }
 
+void InitLogConstants(Local<Object> exports) {
+  Nan::HandleScope scope;
+  Local<Object> log_verbosity = Nan::New<Object>();
+  Nan::Set(exports, Nan::New("logVerbosity").ToLocalChecked(), log_verbosity);
+  Local<Value> DEBUG(Nan::New<Uint32, uint32_t>(GPR_LOG_SEVERITY_DEBUG));
+  Nan::Set(log_verbosity, Nan::New("DEBUG").ToLocalChecked(), DEBUG);
+  Local<Value> INFO(Nan::New<Uint32, uint32_t>(GPR_LOG_SEVERITY_INFO));
+  Nan::Set(log_verbosity, Nan::New("INFO").ToLocalChecked(), INFO);
+  Local<Value> LOG_ERROR(Nan::New<Uint32, uint32_t>(GPR_LOG_SEVERITY_ERROR));
+  Nan::Set(log_verbosity, Nan::New("ERROR").ToLocalChecked(), LOG_ERROR);
+}
+
 NAN_METHOD(MetadataKeyIsLegal) {
   if (!info[0]->IsString()) {
     return Nan::ThrowTypeError(
@@ -298,16 +332,101 @@ NAN_METHOD(SetDefaultRootsPem) {
   }
 }
 
+NAUV_WORK_CB(LogMessagesCallback) {
+  Nan::HandleScope scope;
+  std::list<log_args *> args;
+  uv_mutex_lock(&grpc_logger_state.mutex);
+  args.splice(args.begin(), *grpc_logger_state.pending_args);
+  uv_mutex_unlock(&grpc_logger_state.mutex);
+  /* Call the callback with each log message */
+  while (!args.empty()) {
+    log_args *arg = args.front();
+    args.pop_front();
+    Local<Value> file = Nan::New(arg->core_args.file).ToLocalChecked();
+    Local<Value> line = Nan::New<Uint32, uint32_t>(arg->core_args.line);
+    Local<Value> severity = Nan::New(
+        gpr_log_severity_string(arg->core_args.severity)).ToLocalChecked();
+    Local<Value> message = Nan::New(arg->core_args.message).ToLocalChecked();
+    Local<Value> timestamp = Nan::New<v8::Date>(
+        grpc::node::TimespecToMilliseconds(arg->timestamp)).ToLocalChecked();
+    const int argc = 5;
+    Local<Value> argv[argc] = {file, line, severity, message, timestamp};
+    grpc_logger_state.callback->Call(argc, argv);
+    delete[] arg->core_args.message;
+    delete arg;
+  }
+}
+
+void node_log_func(gpr_log_func_args *args) {
+  // TODO(mlumish): Use the core's log formatter when it becomes available
+  log_args *args_copy = new log_args;
+  size_t message_len = strlen(args->message) + 1;
+  char *message = new char[message_len];
+  memcpy(message, args->message, message_len);
+  memcpy(&args_copy->core_args, args, sizeof(gpr_log_func_args));
+  args_copy->core_args.message = message;
+  args_copy->timestamp = gpr_now(GPR_CLOCK_REALTIME);
+
+  uv_mutex_lock(&grpc_logger_state.mutex);
+  grpc_logger_state.pending_args->push_back(args_copy);
+  uv_mutex_unlock(&grpc_logger_state.mutex);
+
+  uv_async_send(&grpc_logger_state.async);
+}
+
+void init_logger() {
+  memset(&grpc_logger_state, 0, sizeof(logger_state));
+  grpc_logger_state.pending_args = new std::list<log_args *>();
+  uv_mutex_init(&grpc_logger_state.mutex);
+  uv_async_init(uv_default_loop(),
+                &grpc_logger_state.async,
+                LogMessagesCallback);
+  uv_unref((uv_handle_t*)&grpc_logger_state.async);
+  grpc_logger_state.logger_set = false;
+
+  gpr_log_verbosity_init();
+}
+
+/* This registers a JavaScript logger for messages from the gRPC core. Because
+   that handler has to be run in the context of the JavaScript event loop, it
+   will be run asynchronously. To minimize the problems that could cause for
+   debugging, we leave core to do its default synchronous logging until a
+   JavaScript logger is set */
+NAN_METHOD(SetDefaultLoggerCallback) {
+  if (!info[0]->IsFunction()) {
+    return Nan::ThrowTypeError(
+        "setDefaultLoggerCallback's argument must be a function");
+  }
+  if (!grpc_logger_state.logger_set) {
+    gpr_set_log_function(node_log_func);
+    grpc_logger_state.logger_set = true;
+  }
+  grpc_logger_state.callback = new Nan::Callback(info[0].As<v8::Function>());
+}
+
+NAN_METHOD(SetLogVerbosity) {
+  if (!info[0]->IsUint32()) {
+    return Nan::ThrowTypeError(
+        "setLogVerbosity's argument must be a number");
+  }
+  gpr_log_severity severity = static_cast<gpr_log_severity>(
+      Nan::To<uint32_t>(info[0]).FromJust());
+  gpr_set_log_verbosity(severity);
+}
+
 void init(Local<Object> exports) {
   Nan::HandleScope scope;
   grpc_init();
   grpc_set_ssl_roots_override_callback(get_ssl_roots_override);
+  init_logger();
+
   InitStatusConstants(exports);
   InitCallErrorConstants(exports);
   InitOpTypeConstants(exports);
   InitPropagateConstants(exports);
   InitConnectivityStateConstants(exports);
   InitWriteFlags(exports);
+  InitLogConstants(exports);
 
   grpc::node::Call::Init(exports);
   grpc::node::CallCredentials::Init(exports);
@@ -333,6 +452,14 @@ void init(Local<Object> exports) {
            Nan::GetFunction(
                Nan::New<FunctionTemplate>(SetDefaultRootsPem)
                             ).ToLocalChecked());
+  Nan::Set(exports, Nan::New("setDefaultLoggerCallback").ToLocalChecked(),
+           Nan::GetFunction(
+               Nan::New<FunctionTemplate>(SetDefaultLoggerCallback)
+                            ).ToLocalChecked());
+  Nan::Set(exports, Nan::New("setLogVerbosity").ToLocalChecked(),
+           Nan::GetFunction(
+               Nan::New<FunctionTemplate>(SetLogVerbosity)
+                            ).ToLocalChecked());
 }
 
 NODE_MODULE(grpc_node, init)
diff --git a/src/node/ext/server.cc b/src/node/ext/server.cc
index b9e1fe9160..dd1b777ac8 100644
--- a/src/node/ext/server.cc
+++ b/src/node/ext/server.cc
@@ -35,15 +35,15 @@
 
 #include "server.h"
 
-#include <node.h>
 #include <nan.h>
+#include <node.h>
 
 #include <vector>
+#include "call.h"
+#include "completion_queue_async_worker.h"
 #include "grpc/grpc.h"
 #include "grpc/grpc_security.h"
 #include "grpc/support/log.h"
-#include "call.h"
-#include "completion_queue_async_worker.h"
 #include "server_credentials.h"
 #include "timeval.h"
 
@@ -100,8 +100,8 @@ class NewCallOp : public Op {
     Nan::Set(obj, Nan::New("host").ToLocalChecked(),
              Nan::New(details.host).ToLocalChecked());
     Nan::Set(obj, Nan::New("deadline").ToLocalChecked(),
-             Nan::New<Date>(
-                 TimespecToMilliseconds(details.deadline)).ToLocalChecked());
+             Nan::New<Date>(TimespecToMilliseconds(details.deadline))
+                 .ToLocalChecked());
     Nan::Set(obj, Nan::New("metadata").ToLocalChecked(),
              ParseMetadata(&request_metadata));
     return scope.Escape(obj);
@@ -117,14 +117,13 @@ class NewCallOp : public Op {
   grpc_metadata_array request_metadata;
 
  protected:
-  std::string GetTypeString() const {
-    return "new_call";
-  }
+  std::string GetTypeString() const { return "new_call"; }
 };
 
 Server::Server(grpc_server *server) : wrapped_server(server) {
   shutdown_queue = grpc_completion_queue_create(NULL);
-  grpc_server_register_completion_queue(server, shutdown_queue, NULL);
+  grpc_server_register_non_listening_completion_queue(server, shutdown_queue,
+                                                      NULL);
 }
 
 Server::~Server() {
@@ -156,8 +155,7 @@ bool Server::HasInstance(Local<Value> val) {
 }
 
 void Server::ShutdownServer() {
-  grpc_server_shutdown_and_notify(this->wrapped_server,
-                                  this->shutdown_queue,
+  grpc_server_shutdown_and_notify(this->wrapped_server, this->shutdown_queue,
                                   NULL);
   grpc_server_cancel_all_calls(this->wrapped_server);
   grpc_completion_queue_pluck(this->shutdown_queue, NULL,
@@ -170,8 +168,8 @@ NAN_METHOD(Server::New) {
   if (!info.IsConstructCall()) {
     const int argc = 1;
     Local<Value> argv[argc] = {info[0]};
-    MaybeLocal<Object> maybe_instance = constructor->GetFunction()->NewInstance(
-        argc, argv);
+    MaybeLocal<Object> maybe_instance =
+        constructor->GetFunction()->NewInstance(argc, argv);
     if (maybe_instance.IsEmpty()) {
       // There's probably a pending exception
       return;
@@ -185,8 +183,9 @@ NAN_METHOD(Server::New) {
   grpc_channel_args *channel_args;
   if (!ParseChannelArgs(info[0], &channel_args)) {
     DeallocateChannelArgs(channel_args);
-    return Nan::ThrowTypeError("Server options must be an object with "
-                               "string keys and integer or string values");
+    return Nan::ThrowTypeError(
+        "Server options must be an object with "
+        "string keys and integer or string values");
   }
   wrapped_server = grpc_server_create(channel_args, NULL);
   DeallocateChannelArgs(channel_args);
@@ -218,8 +217,7 @@ NAN_METHOD(Server::RequestCall) {
 
 NAN_METHOD(Server::AddHttp2Port) {
   if (!HasInstance(info.This())) {
-    return Nan::ThrowTypeError(
-        "addHttp2Port can only be called on a Server");
+    return Nan::ThrowTypeError("addHttp2Port can only be called on a Server");
   }
   if (!info[0]->IsString()) {
     return Nan::ThrowTypeError(
@@ -239,8 +237,7 @@ NAN_METHOD(Server::AddHttp2Port) {
                                                *Utf8String(info[0]));
   } else {
     port = grpc_server_add_secure_http2_port(server->wrapped_server,
-                                             *Utf8String(info[0]),
-                                             creds);
+                                             *Utf8String(info[0]), creds);
   }
   info.GetReturnValue().Set(Nan::New<Number>(port));
 }
@@ -262,8 +259,7 @@ NAN_METHOD(Server::TryShutdown) {
   Server *server = ObjectWrap::Unwrap<Server>(info.This());
   unique_ptr<OpVec> ops(new OpVec());
   grpc_server_shutdown_and_notify(
-      server->wrapped_server,
-      CompletionQueueAsyncWorker::GetQueue(),
+      server->wrapped_server, CompletionQueueAsyncWorker::GetQueue(),
       new struct tag(new Nan::Callback(info[0].As<Function>()), ops.release(),
                      shared_ptr<Resources>(nullptr)));
   CompletionQueueAsyncWorker::Next();
diff --git a/src/node/index.js b/src/node/index.js
index 66664d94b5..9fb6faa5d7 100644
--- a/src/node/index.js
+++ b/src/node/index.js
@@ -46,6 +46,8 @@ var client = require('./src/client.js');
 
 var server = require('./src/server.js');
 
+var common = require('./src/common.js');
+
 var Metadata = require('./src/metadata.js');
 
 var grpc = require('./src/grpc_extension');
@@ -122,6 +124,42 @@ exports.load = function load(filename, format, options) {
   return loadObject(builder.ns, options);
 };
 
+var log_template = _.template(
+    '{severity} {timestamp}\t{file}:{line}]\t{message}',
+    {interpolate: /{([\s\S]+?)}/g});
+
+/**
+ * Sets the logger function for the gRPC module. For debugging purposes, the C
+ * core will log synchronously directly to stdout unless this function is
+ * called. Note: the output format here is intended to be informational, and
+ * is not guaranteed to stay the same in the future.
+ * Logs will be directed to logger.error.
+ * @param {Console} logger A Console-like object.
+ */
+exports.setLogger = function setLogger(logger) {
+  common.logger = logger;
+  grpc.setDefaultLoggerCallback(function(file, line, severity,
+                                         message, timestamp) {
+    logger.error(log_template({
+      file: path.basename(file),
+      line: line,
+      severity: severity,
+      message: message,
+      timestamp: timestamp.toISOString()
+    }));
+  });
+};
+
+/**
+ * Sets the logger verbosity for gRPC module logging. The options are members
+ * of the grpc.logVerbosity map.
+ * @param {Number} verbosity The minimum severity to log
+ */
+exports.setLogVerbosity = function setLogVerbosity(verbosity) {
+  common.logVerbosity = verbosity;
+  grpc.setLogVerbosity(verbosity);
+};
+
 /**
  * @see module:src/server.Server
  */
@@ -153,6 +191,11 @@ exports.callError = grpc.callError;
 exports.writeFlags = grpc.writeFlags;
 
 /**
+ * Log verbosity setting name to code number mapping
+ */
+exports.logVerbosity = grpc.logVerbosity;
+
+/**
  * Credentials factories
  */
 exports.credentials = require('./src/credentials.js');
diff --git a/src/node/interop/interop_server.js b/src/node/interop/interop_server.js
index 7280762305..05f52a1083 100644
--- a/src/node/interop/interop_server.js
+++ b/src/node/interop/interop_server.js
@@ -45,9 +45,6 @@ var testProto = grpc.load({
 var ECHO_INITIAL_KEY = 'x-grpc-test-echo-initial';
 var ECHO_TRAILING_KEY = 'x-grpc-test-echo-trailing-bin';
 
-var incompressible_data = fs.readFileSync(
-    __dirname + '/../../../test/cpp/interop/rnd.dat');
-
 /**
  * Create a buffer filled with size zeroes
  * @param {number} size The length of the buffer
@@ -88,15 +85,7 @@ function getEchoTrailer(call) {
 }
 
 function getPayload(payload_type, size) {
-  if (payload_type === 'RANDOM') {
-    payload_type = ['COMPRESSABLE',
-                    'UNCOMPRESSABLE'][Math.random() < 0.5 ? 0 : 1];
-  }
-  var body;
-  switch (payload_type) {
-    case 'COMPRESSABLE': body = zeroBuffer(size); break;
-    case 'UNCOMPRESSABLE': incompressible_data.slice(size); break;
-  }
+  var body = zeroBuffer(size);
   return {type: payload_type, body: body};
 }
 
diff --git a/src/node/performance/benchmark_client.js b/src/node/performance/benchmark_client.js
index 262aa33862..5ef5260a96 100644
--- a/src/node/performance/benchmark_client.js
+++ b/src/node/performance/benchmark_client.js
@@ -42,6 +42,8 @@ var fs = require('fs');
 var path = require('path');
 var util = require('util');
 var EventEmitter = require('events');
+
+var async = require('async');
 var _ = require('lodash');
 var PoissonProcess = require('poisson-process');
 var Histogram = require('./histogram');
@@ -128,6 +130,36 @@ function BenchmarkClient(server_targets, channels, histogram_params,
 util.inherits(BenchmarkClient, EventEmitter);
 
 /**
+ * Start every client in the list of clients by waiting for each to be ready,
+ * then starting outstanding_rpcs_per_channel calls on each of them
+ * @param {Array<grpc.Client>} client_list The list of clients
+ * @param {Number} outstanding_rpcs_per_channel The number of calls to start
+ *     on each client
+ * @param {function(grpc.Client)} makeCall Function to make a single call on
+ *     a single client
+ * @param {EventEmitter} emitter The event emitter to send errors on, if
+ *     necessary
+ */
+function startAllClients(client_list, outstanding_rpcs_per_channel, makeCall,
+                         emitter) {
+  var ready_wait_funcs = _.map(client_list, function(client) {
+    return _.partial(grpc.waitForClientReady, client, Infinity);
+  });
+  async.parallel(ready_wait_funcs, function(err) {
+    if (err) {
+      emitter.emit('error', err);
+      return;
+    }
+
+    _.each(client_list, function(client) {
+      _.times(outstanding_rpcs_per_channel, function() {
+        makeCall(client);
+      });
+    });
+  });
+}
+
+/**
  * Start a closed-loop test. For each channel, start
  * outstanding_rpcs_per_channel RPCs. Then, whenever an RPC finishes, start
  * another one.
@@ -212,11 +244,7 @@ BenchmarkClient.prototype.startClosedLoop = function(
     };
   }
 
-  _.each(client_list, function(client) {
-    _.times(outstanding_rpcs_per_channel, function() {
-      makeCall(client);
-    });
-  });
+  startAllClients(client_list, outstanding_rpcs_per_channel, makeCall, self);
 };
 
 /**
@@ -310,14 +338,12 @@ BenchmarkClient.prototype.startPoisson = function(
 
   var averageIntervalMs = (1 / offered_load) * 1000;
 
-  _.each(client_list, function(client) {
-    _.times(outstanding_rpcs_per_channel, function() {
-      var p = PoissonProcess.create(averageIntervalMs, function() {
-        makeCall(client, p);
-      });
-      p.start();
+  startAllClients(client_list, outstanding_rpcs_per_channel, function(client){
+    var p = PoissonProcess.create(averageIntervalMs, function() {
+      makeCall(client, p);
     });
-  });
+    p.start();
+  }, self);
 };
 
 /**
diff --git a/src/node/src/common.js b/src/node/src/common.js
index 8cf43b7a84..22159dd39f 100644
--- a/src/node/src/common.js
+++ b/src/node/src/common.js
@@ -157,3 +157,24 @@ exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service,
     }];
   }));
 };
+
+/**
+ * The logger object for the gRPC module. Defaults to console.
+ */
+exports.logger = console;
+
+/**
+ * The current logging verbosity. 0 corresponds to logging everything
+ */
+exports.logVerbosity = 0;
+
+/**
+ * Log a message if the severity is at least as high as the current verbosity
+ * @param {Number} severity A value of the grpc.logVerbosity map
+ * @param {String} message The message to log
+ */
+exports.log = function log(severity, message) {
+  if (severity >= exports.logVerbosity) {
+    exports.logger.error(message);
+  }
+};
diff --git a/src/node/src/credentials.js b/src/node/src/credentials.js
index a12eade4e1..b746d0625d 100644
--- a/src/node/src/credentials.js
+++ b/src/node/src/credentials.js
@@ -69,6 +69,8 @@ var ChannelCredentials = grpc.ChannelCredentials;
 
 var Metadata = require('./metadata.js');
 
+var common = require('./common.js');
+
 /**
  * Create an SSL Credentials object. If using a client-side certificate, both
  * the second and third arguments must be passed.
@@ -120,7 +122,7 @@ exports.createFromGoogleCredential = function(google_credential) {
     var service_url = auth_context.service_url;
     google_credential.getRequestMetadata(service_url, function(err, header) {
       if (err) {
-        console.log('Auth error:', err);
+        common.log(grpc.logVerbosity.INFO, 'Auth error:' + err);
         callback(err);
         return;
       }
diff --git a/src/node/src/server.js b/src/node/src/server.js
index fd5153f124..b3b414969a 100644
--- a/src/node/src/server.js
+++ b/src/node/src/server.js
@@ -735,8 +735,8 @@ Server.prototype.addService = function(service, implementation) {
     }
     var impl;
     if (implementation[name] === undefined) {
-      console.warn('Method handler for %s expected but not provided',
-                   attrs.path);
+      common.log(grpc.logVerbosity.ERROR, 'Method handler for ' +
+          attrs.path + ' expected but not provided');
       impl = defaultHandler[method_type];
     } else {
       impl = _.bind(implementation[name], implementation);
diff --git a/src/node/test/credentials_test.js b/src/node/test/credentials_test.js
index 794215b246..0a21572582 100644
--- a/src/node/test/credentials_test.js
+++ b/src/node/test/credentials_test.js
@@ -318,7 +318,7 @@ describe('client credentials', function() {
       done();
     });
   });
-  it.skip('should propagate errors that the updater emits', function(done) {
+  it('should propagate errors that the updater emits', function(done) {
     var metadataUpdater = function(service_url, callback) {
       var error = new Error('Authentication error');
       error.code = grpc.status.UNAUTHENTICATED;
@@ -370,7 +370,7 @@ describe('client credentials', function() {
       done();
     });
   });
-  it.skip('should get an error from a Google credential', function(done) {
+  it('should get an error from a Google credential', function(done) {
     var creds = grpc.credentials.createFromGoogleCredential(
         fakeFailingGoogleCredentials);
     var combined_creds = grpc.credentials.combineChannelCredentials(
diff --git a/src/node/test/math/math_grpc_pb.js b/src/node/test/math/math_grpc_pb.js
index 083ed66913..17a4bf7243 100644
--- a/src/node/test/math/math_grpc_pb.js
+++ b/src/node/test/math/math_grpc_pb.js
@@ -1,94 +1,135 @@
 // GENERATED CODE -- DO NOT EDIT!
 
+// Original file comments:
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
 'use strict';
 var grpc = require('grpc');
-var math_pb = require('./math_pb.js');
+var math_math_pb = require('../math/math_pb.js');
 
 function serialize_DivArgs(arg) {
-  if (!(arg instanceof math_pb.DivArgs)) {
+  if (!(arg instanceof math_math_pb.DivArgs)) {
     throw new Error('Expected argument of type DivArgs');
   }
   return new Buffer(arg.serializeBinary());
 }
 
 function deserialize_DivArgs(buffer_arg) {
-  return math_pb.DivArgs.deserializeBinary(new Uint8Array(buffer_arg));
+  return math_math_pb.DivArgs.deserializeBinary(new Uint8Array(buffer_arg));
 }
 
 function serialize_DivReply(arg) {
-  if (!(arg instanceof math_pb.DivReply)) {
+  if (!(arg instanceof math_math_pb.DivReply)) {
     throw new Error('Expected argument of type DivReply');
   }
   return new Buffer(arg.serializeBinary());
 }
 
 function deserialize_DivReply(buffer_arg) {
-  return math_pb.DivReply.deserializeBinary(new Uint8Array(buffer_arg));
+  return math_math_pb.DivReply.deserializeBinary(new Uint8Array(buffer_arg));
 }
 
 function serialize_FibArgs(arg) {
-  if (!(arg instanceof math_pb.FibArgs)) {
+  if (!(arg instanceof math_math_pb.FibArgs)) {
     throw new Error('Expected argument of type FibArgs');
   }
   return new Buffer(arg.serializeBinary());
 }
 
 function deserialize_FibArgs(buffer_arg) {
-  return math_pb.FibArgs.deserializeBinary(new Uint8Array(buffer_arg));
+  return math_math_pb.FibArgs.deserializeBinary(new Uint8Array(buffer_arg));
 }
 
 function serialize_Num(arg) {
-  if (!(arg instanceof math_pb.Num)) {
+  if (!(arg instanceof math_math_pb.Num)) {
     throw new Error('Expected argument of type Num');
   }
   return new Buffer(arg.serializeBinary());
 }
 
 function deserialize_Num(buffer_arg) {
-  return math_pb.Num.deserializeBinary(new Uint8Array(buffer_arg));
+  return math_math_pb.Num.deserializeBinary(new Uint8Array(buffer_arg));
 }
 
 
 var MathService = exports.MathService = {
+  // Div divides args.dividend by args.divisor and returns the quotient and
+  // remainder.
   div: {
     path: '/math.Math/Div',
     requestStream: false,
     responseStream: false,
-    requestType: math_pb.DivArgs,
-    responseType: math_pb.DivReply,
+    requestType: math_math_pb.DivArgs,
+    responseType: math_math_pb.DivReply,
     requestSerialize: serialize_DivArgs,
     requestDeserialize: deserialize_DivArgs,
     responseSerialize: serialize_DivReply,
     responseDeserialize: deserialize_DivReply,
   },
+  // DivMany accepts an arbitrary number of division args from the client stream
+  // and sends back the results in the reply stream.  The stream continues until
+  // the client closes its end; the server does the same after sending all the
+  // replies.  The stream ends immediately if either end aborts.
   divMany: {
     path: '/math.Math/DivMany',
     requestStream: true,
     responseStream: true,
-    requestType: math_pb.DivArgs,
-    responseType: math_pb.DivReply,
+    requestType: math_math_pb.DivArgs,
+    responseType: math_math_pb.DivReply,
     requestSerialize: serialize_DivArgs,
     requestDeserialize: deserialize_DivArgs,
     responseSerialize: serialize_DivReply,
     responseDeserialize: deserialize_DivReply,
   },
+  // Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+  // generates up to limit numbers; otherwise it continues until the call is
+  // canceled.  Unlike Fib above, Fib has no final FibReply.
   fib: {
     path: '/math.Math/Fib',
     requestStream: false,
     responseStream: true,
-    requestType: math_pb.FibArgs,
-    responseType: math_pb.Num,
+    requestType: math_math_pb.FibArgs,
+    responseType: math_math_pb.Num,
     requestSerialize: serialize_FibArgs,
     requestDeserialize: deserialize_FibArgs,
     responseSerialize: serialize_Num,
     responseDeserialize: deserialize_Num,
   },
+  // Sum sums a stream of numbers, returning the final result once the stream
+  // is closed.
   sum: {
     path: '/math.Math/Sum',
     requestStream: true,
     responseStream: false,
-    requestType: math_pb.Num,
-    responseType: math_pb.Num,
+    requestType: math_math_pb.Num,
+    responseType: math_math_pb.Num,
     requestSerialize: serialize_Num,
     requestDeserialize: deserialize_Num,
     responseSerialize: serialize_Num,
diff --git a/src/node/test/math/math_pb.js b/src/node/test/math/math_pb.js
index 3489143bec..ccc05c6e06 100644
--- a/src/node/test/math/math_pb.js
+++ b/src/node/test/math/math_pb.js
@@ -65,7 +65,7 @@ proto.math.DivArgs.toObject = function(includeInstance, msg) {
   };
 
   if (includeInstance) {
-    obj.$jspbMessageInstance = msg
+    obj.$jspbMessageInstance = msg;
   }
   return obj;
 };
@@ -251,7 +251,7 @@ proto.math.DivReply.toObject = function(includeInstance, msg) {
   };
 
   if (includeInstance) {
-    obj.$jspbMessageInstance = msg
+    obj.$jspbMessageInstance = msg;
   }
   return obj;
 };
@@ -436,7 +436,7 @@ proto.math.FibArgs.toObject = function(includeInstance, msg) {
   };
 
   if (includeInstance) {
-    obj.$jspbMessageInstance = msg
+    obj.$jspbMessageInstance = msg;
   }
   return obj;
 };
@@ -595,7 +595,7 @@ proto.math.Num.toObject = function(includeInstance, msg) {
   };
 
   if (includeInstance) {
-    obj.$jspbMessageInstance = msg
+    obj.$jspbMessageInstance = msg;
   }
   return obj;
 };
@@ -754,7 +754,7 @@ proto.math.FibReply.toObject = function(includeInstance, msg) {
   };
 
   if (includeInstance) {
-    obj.$jspbMessageInstance = msg
+    obj.$jspbMessageInstance = msg;
   }
   return obj;
 };
diff --git a/src/node/test/math/math_server.js b/src/node/test/math/math_server.js
index fa05ed0165..5e3d1dd864 100644
--- a/src/node/test/math/math_server.js
+++ b/src/node/test/math/math_server.js
@@ -68,7 +68,7 @@ function mathDiv(call, cb) {
 function mathFib(stream) {
   // Here, call is a standard writable Node object Stream
   var previous = 0, current = 1;
-  for (var i = 0; i < stream.request.limit; i++) {
+  for (var i = 0; i < stream.request.getLimit(); i++) {
     var response = new math.Num();
     response.setNum(current);
     stream.write(response);
diff --git a/src/node/tools/package.json b/src/node/tools/package.json
index 321e3c3e7a..7c256d7ba0 100644
--- a/src/node/tools/package.json
+++ b/src/node/tools/package.json
@@ -1,6 +1,6 @@
 {
   "name": "grpc-tools",
-  "version": "0.14.2-pre1",
+  "version": "0.16.0-dev",
   "author": "Google Inc.",
   "description": "Tools for developing with gRPC on Node.js",
   "homepage": "http://www.grpc.io/",
@@ -34,6 +34,7 @@
     "index.js",
     "bin/protoc.js",
     "bin/protoc_plugin.js",
+    "bin/google/protobuf",
     "LICENSE"
   ],
   "main": "index.js"
diff --git a/src/ruby/lib/grpc/signals.rb b/src/objective-c/CronetFramework.podspec
index 2ab85c8bb1..20af7647f7 100644
--- a/src/ruby/lib/grpc/signals.rb
+++ b/src/objective-c/CronetFramework.podspec
@@ -27,43 +27,17 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-require 'thread'
-require_relative 'grpc'
 
-# GRPC contains the General RPC module.
-module GRPC
-  # Signals contains gRPC functions related to signal handling
-  module Signals
-    @interpreter_exiting = false
-    @signal_handlers = []
-    @handlers_mutex = Mutex.new
-
-    def register_handler(&handler)
-      @handlers_mutex.synchronize do
-        @signal_handlers.push(handler)
-        handler.call if @exit_signal_received
-      end
-      # Returns a function to remove the handler
-      lambda do
-        @handlers_mutex.synchronize { @signal_handlers.delete(handler) }
-      end
-    end
-    module_function :register_handler
-
-    def wait_for_signals
-      t = Thread.new do
-        sleep 0.1 until GRPC::Core.signal_received? || @interpreter_exiting
-        unless @interpreter_exiting
-          @handlers_mutex.synchronize do
-            @signal_handlers.each(&:call)
-          end
-        end
-      end
-      at_exit do
-        @interpreter_exiting = true
-        t.join
-      end
-    end
-    module_function :wait_for_signals
-  end
+Pod::Spec.new do |s|
+  s.name         = "CronetFramework"
+  s.version      = "0.0.2"
+  s.summary      = "Cronet, precompiled and used as a framework."
+  s.homepage     = "http://chromium.org"
+  s.license      = { :type => 'BSD' }
+  s.vendored_framework = "Cronet.framework"
+  s.author             = "The Chromium Authors"
+  s.ios.deployment_target = "8.0"
+  s.source       = { :http => 'https://storage.googleapis.com/grpc-precompiled-binaries/cronet/Cronet.framework.zip' }
+  s.preserve_paths = "Cronet.framework"
+  s.public_header_files = "Cronet.framework/Headers/**/*{.h}"
 end
diff --git a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h
index bd6b064f16..646bf43b54 100644
--- a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h
+++ b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h
@@ -32,6 +32,8 @@
  */
 #import "GRPCCall.h"
 
+#include <AvailabilityMacros.h>
+
 /**
  * Methods to configure GRPC channel options.
  */
@@ -43,4 +45,7 @@
  */
 + (void)setUserAgentPrefix:(NSString *)userAgentPrefix forHost:(NSString *)host;
 
++ (void)closeOpenConnections DEPRECATED_MSG_ATTRIBUTE("The API for this feature is experimental, "
+                                                      "and might be removed or modified at any "
+                                                      "time.");
 @end
diff --git a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m
index 5f9932d86d..bcc3b91507 100644
--- a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m
+++ b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m
@@ -46,4 +46,8 @@
   hostConfig.userAgentPrefix = userAgentPrefix;
 }
 
++ (void)closeOpenConnections {
+  [GRPCHost flushChannelCache];
+}
+
 @end
diff --git a/src/objective-c/GRPCClient/GRPCCall+ChannelCredentials.h b/src/objective-c/GRPCClient/GRPCCall+ChannelCredentials.h
index 343dd48a14..ac2a37d75f 100644
--- a/src/objective-c/GRPCClient/GRPCCall+ChannelCredentials.h
+++ b/src/objective-c/GRPCClient/GRPCCall+ChannelCredentials.h
@@ -41,7 +41,7 @@
  */
 + (BOOL)setTLSPEMRootCerts:(nullable NSString *)pemRootCert
                    forHost:(nonnull NSString *)host
-                     error:(NSError **)errorPtr;
+                     error:(NSError * _Nullable * _Nullable)errorPtr;
 /**
  * Configures @c host with TLS/SSL Client Credentials and optionally trusted root Certificate
  * Authorities. If @c pemRootCerts is nil, the default CA Certificates bundled with gRPC will be
@@ -51,6 +51,6 @@
             withPrivateKey:(nullable NSString *)pemPrivateKey
              withCertChain:(nullable NSString *)pemCertChain
                    forHost:(nonnull NSString *)host
-                     error:(NSError **)errorPtr;
+                     error:(NSError * _Nullable * _Nullable)errorPtr;
 
 @end
diff --git a/src/objective-c/GRPCClient/GRPCCall+Cronet.h b/src/objective-c/GRPCClient/GRPCCall+Cronet.h
new file mode 100644
index 0000000000..2d8f7ac8fb
--- /dev/null
+++ b/src/objective-c/GRPCClient/GRPCCall+Cronet.h
@@ -0,0 +1,57 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifdef GRPC_COMPILE_WITH_CRONET
+#import <Cronet/Cronet.h>
+
+#import "GRPCCall.h"
+
+/**
+ * Methods for using cronet transport.
+ */
+@interface GRPCCall (Cronet)
+
+/**
+ * This method should be called before issuing the first RPC. It should be
+ * called only once. Create an instance of Cronet engine in your app elsewhere
+ * and pass the instance pointer in the cronet_engine parameter. Once set,
+ * all subsequent RPCs will use Cronet transport. The method is not thread
+ * safe.
+ */
++(void)useCronetWithEngine:(cronet_engine *)engine;
+
++(cronet_engine *)cronetEngine;
+
++(BOOL)isUsingCronet;
+
+@end
+#endif
diff --git a/src/objective-c/GRPCClient/GRPCCall+Cronet.m b/src/objective-c/GRPCClient/GRPCCall+Cronet.m
new file mode 100644
index 0000000000..76ca1a2537
--- /dev/null
+++ b/src/objective-c/GRPCClient/GRPCCall+Cronet.m
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#import "GRPCCall+Cronet.h"
+
+#ifdef GRPC_COMPILE_WITH_CRONET
+static BOOL useCronet = NO;
+static cronet_engine *globalCronetEngine;
+
+@implementation GRPCCall (Cronet)
+
++ (void)useCronetWithEngine:(cronet_engine *)engine {
+  useCronet = YES;
+  globalCronetEngine = engine;
+}
+
++ (cronet_engine *)cronetEngine {
+  return globalCronetEngine;
+}
+
++ (BOOL)isUsingCronet {
+  return useCronet;
+}
+
+@end
+#endif
diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h
index 7a77ae60b6..b9e741dfa8 100644
--- a/src/objective-c/GRPCClient/GRPCCall.h
+++ b/src/objective-c/GRPCClient/GRPCCall.h
@@ -220,6 +220,8 @@ extern id const kGRPCTrailersKey;
  * messages to the response side of the call indefinitely (depending on the semantics of the
  * specific remote method called).
  * To finish a call right away, invoke cancel.
+ * host parameter should not contain the scheme (http:// or https://), only the name or IP addr
+ * and the port number, for example @"localhost:5050".
  */
 - (instancetype)initWithHost:(NSString *)host
                         path:(NSString *)path
diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m
index 63553c0242..da9473f9a2 100644
--- a/src/objective-c/GRPCClient/GRPCCall.m
+++ b/src/objective-c/GRPCClient/GRPCCall.m
@@ -76,7 +76,6 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
   NSString *_host;
   NSString *_path;
   GRPCWrappedCall *_wrappedCall;
-  dispatch_once_t _callAlreadyInvoked;
   GRPCConnectivityMonitor *_connectivityMonitor;
 
   // The C gRPC library has less guarantees on the ordering of events than we
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h
index 70d1a9bd2f..40e78a92d6 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.h
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.h
@@ -56,6 +56,13 @@ struct grpc_channel_credentials;
 + (nullable GRPCChannel *)secureChannelWithHost:(nonnull NSString *)host;
 
 /**
+ * Creates a secure channel to the specified @c host using Cronet as a transport mechanism.
+ */
+#ifdef GRPC_COMPILE_WITH_CRONET
++ (nullable GRPCChannel *)secureCronetChannelWithHost:(NSString *)host
+                                          channelArgs:(NSDictionary *)channelArgs;
+#endif
+/**
  * Creates a secure channel to the specified @c host using the specified @c credentials and
  * @c channelArgs. Only in tests should @c GRPC_SSL_TARGET_NAME_OVERRIDE_ARG channel arg be set.
  */
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m
index 203ef58c0d..7b7b79e1c6 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.m
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.m
@@ -34,10 +34,17 @@
 #import "GRPCChannel.h"
 
 #include <grpc/grpc_security.h>
+#ifdef GRPC_COMPILE_WITH_CRONET
+#include <grpc/grpc_cronet.h>
+#endif
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#ifdef GRPC_COMPILE_WITH_CRONET
+#import <Cronet/Cronet.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
+#endif
 #import "GRPCCompletionQueue.h"
 
 void freeChannelArgs(grpc_channel_args *channel_args) {
@@ -99,6 +106,24 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
   grpc_channel_args *_channelArgs;
 }
 
+#ifdef GRPC_COMPILE_WITH_CRONET
+- (instancetype)initWithHost:(NSString *)host
+                cronetEngine:(cronet_engine *)cronetEngine
+                 channelArgs:(NSDictionary *)channelArgs {
+  if (!host) {
+    [NSException raise:NSInvalidArgumentException format:@"host argument missing"];
+  }
+
+  if (self = [super init]) {
+    _channelArgs = buildChannelArgs(channelArgs);
+    _host = [host copy];
+    _unmanagedChannel = grpc_cronet_secure_channel_create(cronetEngine, _host.UTF8String, _channelArgs,
+                                                     NULL);
+  }
+
+  return self;
+}
+#endif
 
 - (instancetype)initWithHost:(NSString *)host
                       secure:(BOOL)secure
@@ -133,6 +158,19 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
   freeChannelArgs(_channelArgs);
 }
 
+#ifdef GRPC_COMPILE_WITH_CRONET
++ (GRPCChannel *)secureCronetChannelWithHost:(NSString *)host
+                                 channelArgs:(NSDictionary *)channelArgs {
+  cronet_engine *engine = [GRPCCall cronetEngine];
+  if (!engine) {
+    [NSException raise:NSInvalidArgumentException
+                format:@"cronet_engine is NULL. Set it first."];
+    return nil;
+  }
+  return [[GRPCChannel alloc] initWithHost:host cronetEngine:engine channelArgs:channelArgs];
+}
+#endif
+
 + (GRPCChannel *)secureChannelWithHost:(NSString *)host {
   return [[GRPCChannel alloc] initWithHost:host secure:YES credentials:NULL channelArgs:NULL];
 }
@@ -161,9 +199,7 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
                                   NULL, GRPC_PROPAGATE_DEFAULTS,
                                   queue.unmanagedQueue,
                                   path.UTF8String,
-                                  // Get "host" from "host:port"
-                                  // TODO(jcanizales): Use NSURLs throughout, to clarify these.
-                                  [_host componentsSeparatedByString:@":"][0].UTF8String,
+                                  NULL, // Passing NULL for host
                                   gpr_inf_future(GPR_CLOCK_REALTIME), NULL);
 }
 
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h
index 9220e2a33d..350c69bf8e 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.h
+++ b/src/objective-c/GRPCClient/private/GRPCHost.h
@@ -41,6 +41,8 @@ struct grpc_channel_credentials;
 
 @interface GRPCHost : NSObject
 
++ (void)flushChannelCache;
+
 @property(nonatomic, readonly) NSString *address;
 @property(nonatomic, copy, nullable) NSString *userAgentPrefix;
 @property(nonatomic, nullable) struct grpc_channel_credentials *channelCreds;
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m
index 43166cbb52..08c699f99e 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.m
+++ b/src/objective-c/GRPCClient/private/GRPCHost.m
@@ -36,7 +36,10 @@
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
 #import <GRPCClient/GRPCCall.h>
+#ifdef GRPC_COMPILE_WITH_CRONET
 #import <GRPCClient/GRPCCall+ChannelArg.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
+#endif
 
 #import "GRPCChannel.h"
 #import "GRPCCompletionQueue.h"
@@ -48,6 +51,8 @@ NS_ASSUME_NONNULL_BEGIN
 // templates/src/core/surface/version.c.template .
 #define GRPC_OBJC_VERSION_STRING @"0.13.0"
 
+static NSMutableDictionary *kHostCache;
+
 @implementation GRPCHost {
   // TODO(mlumish): Investigate whether caching channels with strong links is a good idea.
   GRPCChannel *_channel;
@@ -79,13 +84,12 @@ NS_ASSUME_NONNULL_BEGIN
   }
 
   // Look up the GRPCHost in the cache.
-  static NSMutableDictionary *hostCache;
   static dispatch_once_t cacheInitialization;
   dispatch_once(&cacheInitialization, ^{
-    hostCache = [NSMutableDictionary dictionary];
+    kHostCache = [NSMutableDictionary dictionary];
   });
-  @synchronized(hostCache) {
-    GRPCHost *cachedHost = hostCache[address];
+  @synchronized(kHostCache) {
+    GRPCHost *cachedHost = kHostCache[address];
     if (cachedHost) {
       return cachedHost;
     }
@@ -93,12 +97,22 @@ NS_ASSUME_NONNULL_BEGIN
     if ((self = [super init])) {
       _address = address;
       _secure = YES;
-      hostCache[address] = self;
+      kHostCache[address] = self;
     }
   }
   return self;
 }
 
++ (void)flushChannelCache {
+  @synchronized(kHostCache) {
+    [kHostCache enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key,
+                                                    GRPCHost * _Nonnull host,
+                                                    BOOL * _Nonnull stop) {
+      [host disconnect];
+    }];
+  }
+}
+
 - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path
                               completionQueue:(GRPCCompletionQueue *)queue {
   GRPCChannel *channel;
@@ -200,15 +214,26 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (GRPCChannel *)newChannel {
   NSDictionary *args = [self channelArgs];
+#ifdef GRPC_COMPILE_WITH_CRONET
+  BOOL useCronet = [GRPCCall isUsingCronet];
+#endif
   if (_secure) {
       GRPCChannel *channel;
       @synchronized(self) {
         if (_channelCreds == nil) {
           [self setTLSPEMRootCerts:nil withPrivateKey:nil withCertChain:nil error:nil];
         }
-        channel = [GRPCChannel secureChannelWithHost:_address
-                                          credentials:_channelCreds
-                                          channelArgs:args];
+#ifdef GRPC_COMPILE_WITH_CRONET
+        if (useCronet) {
+          channel = [GRPCChannel secureCronetChannelWithHost:_address
+                                                 channelArgs:args];
+        } else
+#endif
+        {
+          channel = [GRPCChannel secureChannelWithHost:_address
+                                            credentials:_channelCreds
+                                            channelArgs:args];
+        }
       }
       return channel;
   } else {
diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
index 160344a7a6..97f6b89340 100644
--- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
+++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
@@ -72,6 +72,8 @@
     _op.op = GRPC_OP_SEND_INITIAL_METADATA;
     _op.data.send_initial_metadata.count = metadata.count;
     _op.data.send_initial_metadata.metadata = metadata.grpc_metadataArray;
+    _op.data.send_initial_metadata.maybe_compression_level.is_set = false;
+    _op.data.send_initial_metadata.maybe_compression_level.level = 0;
     _handler = handler;
   }
   return self;
diff --git a/src/objective-c/GRPCClient/private/NSData+GRPC.m b/src/objective-c/GRPCClient/private/NSData+GRPC.m
index e6a6c3c605..1238374af3 100644
--- a/src/objective-c/GRPCClient/private/NSData+GRPC.m
+++ b/src/objective-c/GRPCClient/private/NSData+GRPC.m
@@ -39,17 +39,21 @@
 
 // TODO(jcanizales): Move these two incantations to the C library.
 
-static void CopyByteBufferToCharArray(grpc_byte_buffer *buffer, char *array) {
-  size_t offset = 0;
+static void MallocAndCopyByteBufferToCharArray(grpc_byte_buffer *buffer,
+                                               size_t *length, char **array) {
   grpc_byte_buffer_reader reader;
   grpc_byte_buffer_reader_init(&reader, buffer);
-  gpr_slice next;
-  while (grpc_byte_buffer_reader_next(&reader, &next) != 0){
-    memcpy(array + offset, GPR_SLICE_START_PTR(next),
-           (size_t)GPR_SLICE_LENGTH(next));
-    offset += GPR_SLICE_LENGTH(next);
-    gpr_slice_unref(next);
+  // The slice contains uncompressed data even if compressed data was received
+  // because the reader takes care of automatically decompressing it
+  gpr_slice slice = grpc_byte_buffer_reader_readall(&reader);
+  size_t uncompressed_length = GPR_SLICE_LENGTH(slice);
+  char *result = malloc(uncompressed_length);
+  if (result) {
+    memcpy(result, GPR_SLICE_START_PTR(slice), uncompressed_length);
   }
+  gpr_slice_unref(slice);
+  *array = result;
+  *length = uncompressed_length;
 }
 
 static grpc_byte_buffer *CopyCharArrayToNewByteBuffer(const char *array,
@@ -65,8 +69,9 @@ static grpc_byte_buffer *CopyCharArrayToNewByteBuffer(const char *array,
   if (buffer == NULL) {
     return nil;
   }
-  NSUInteger length = grpc_byte_buffer_length(buffer);
-  char *array = malloc(length * sizeof(*array));
+  char *array;
+  size_t length;
+  MallocAndCopyByteBufferToCharArray(buffer, &length, &array);
   if (!array) {
     // TODO(jcanizales): grpc_byte_buffer is reference-counted, so we can
     // prevent this memory problem by implementing a subclass of NSData
@@ -74,8 +79,9 @@ static grpc_byte_buffer *CopyCharArrayToNewByteBuffer(const char *array,
     // can be implemented using a grpc_byte_buffer_reader.
     return nil;
   }
-  CopyByteBufferToCharArray(buffer, array);
-  return [self dataWithBytesNoCopy:array length:length freeWhenDone:YES];
+  // Not depending upon size assumption of NSUInteger
+  NSUInteger length_max = MIN(length, UINT_MAX);
+  return [self dataWithBytesNoCopy:array length:length_max freeWhenDone:YES];
 }
 
 - (grpc_byte_buffer *)grpc_byteBuffer {
@@ -85,8 +91,10 @@ static grpc_byte_buffer *CopyCharArrayToNewByteBuffer(const char *array,
   // The following implementation is thus not optimal, sometimes requiring two
   // copies (one by self.bytes and another by gpr_slice_from_copied_buffer).
   // If it turns out to be an issue, we can use enumerateByteRangesUsingblock:
-  // to create an array of gpr_slice objects to pass to grpc_raw_byte_buffer_create.
+  // to create an array of gpr_slice objects to pass to
+  // grpc_raw_byte_buffer_create.
   // That would make it do exactly one copy, always.
-  return CopyCharArrayToNewByteBuffer((const char *)self.bytes, (size_t)self.length);
+  return CopyCharArrayToNewByteBuffer((const char *)self.bytes,
+                                      (size_t)self.length);
 }
 @end
diff --git a/src/objective-c/ProtoRPC/ProtoMethod.h b/src/objective-c/ProtoRPC/ProtoMethod.h
index a0ed2cf98a..ae3a2723fc 100644
--- a/src/objective-c/ProtoRPC/ProtoMethod.h
+++ b/src/objective-c/ProtoRPC/ProtoMethod.h
@@ -37,6 +37,7 @@
  * A fully-qualified proto service method name. Full qualification is needed because a gRPC endpoint
  * can implement multiple services.
  */
+__attribute__((deprecated("Please use GRPCProtoMethod.")))
 @interface ProtoMethod : NSObject
 @property(nonatomic, readonly) NSString *package;
 @property(nonatomic, readonly) NSString *service;
@@ -48,3 +49,14 @@
                         service:(NSString *)service
                          method:(NSString *)method;
 @end
+
+/**
+ * This subclass is empty now. Eventually we'll remove ProtoMethod class
+ * to avoid potential naming conflict
+ */
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+@interface GRPCProtoMethod : ProtoMethod
+#pragma clang diagnostic pop
+
+@end
diff --git a/src/objective-c/ProtoRPC/ProtoMethod.m b/src/objective-c/ProtoRPC/ProtoMethod.m
index 4b7ed63123..e9978f38af 100644
--- a/src/objective-c/ProtoRPC/ProtoMethod.m
+++ b/src/objective-c/ProtoRPC/ProtoMethod.m
@@ -53,3 +53,7 @@
   }
 }
 @end
+
+@implementation GRPCProtoMethod
+
+@end
diff --git a/src/objective-c/ProtoRPC/ProtoRPC.h b/src/objective-c/ProtoRPC/ProtoRPC.h
index bd926b7328..04fc1e45f1 100644
--- a/src/objective-c/ProtoRPC/ProtoRPC.h
+++ b/src/objective-c/ProtoRPC/ProtoRPC.h
@@ -36,13 +36,29 @@
 
 #import "ProtoMethod.h"
 
+__attribute__((deprecated("Please use GRPCProtoCall.")))
 @interface ProtoRPC : GRPCCall
 
+/**
+ * host parameter should not contain the scheme (http:// or https://), only the name or IP addr
+ * and the port number, for example @"localhost:5050".
+ */
 - (instancetype)initWithHost:(NSString *)host
-                      method:(ProtoMethod *)method
+                      method:(GRPCProtoMethod *)method
               requestsWriter:(GRXWriter *)requestsWriter
                responseClass:(Class)responseClass
           responsesWriteable:(id<GRXWriteable>)responsesWriteable NS_DESIGNATED_INITIALIZER;
 
 - (void)start;
 @end
+
+/**
+ * This subclass is empty now. Eventually we'll remove ProtoRPC class
+ * to avoid potential naming conflict
+ */
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+@interface GRPCProtoCall : ProtoRPC
+#pragma clang diagnostic pop
+
+@end
diff --git a/src/objective-c/ProtoRPC/ProtoRPC.m b/src/objective-c/ProtoRPC/ProtoRPC.m
index b526708c58..e7232f2683 100644
--- a/src/objective-c/ProtoRPC/ProtoRPC.m
+++ b/src/objective-c/ProtoRPC/ProtoRPC.m
@@ -70,7 +70,7 @@ static NSError *ErrorForBadProto(id proto, Class expectedClass, NSError *parsing
 
 // Designated initializer
 - (instancetype)initWithHost:(NSString *)host
-                      method:(ProtoMethod *)method
+                      method:(GRPCProtoMethod *)method
               requestsWriter:(GRXWriter *)requestsWriter
                responseClass:(Class)responseClass
           responsesWriteable:(id<GRXWriteable>)responsesWriteable {
@@ -117,3 +117,7 @@ static NSError *ErrorForBadProto(id proto, Class expectedClass, NSError *parsing
   _responseWriteable = nil;
 }
 @end
+
+@implementation GRPCProtoCall
+
+@end
diff --git a/src/objective-c/ProtoRPC/ProtoService.h b/src/objective-c/ProtoRPC/ProtoService.h
index 2e8cb33696..7faae1b49c 100644
--- a/src/objective-c/ProtoRPC/ProtoService.h
+++ b/src/objective-c/ProtoRPC/ProtoService.h
@@ -33,17 +33,31 @@
 
 #import <Foundation/Foundation.h>
 
-@class ProtoRPC;
+@class GRPCProtoCall;
 @protocol GRXWriteable;
 @class GRXWriter;
 
+
+__attribute__((deprecated("Please use GRPCProtoService.")))
 @interface ProtoService : NSObject
 - (instancetype)initWithHost:(NSString *)host
                  packageName:(NSString *)packageName
                  serviceName:(NSString *)serviceName NS_DESIGNATED_INITIALIZER;
 
-- (ProtoRPC *)RPCToMethod:(NSString *)method
+- (GRPCProtoCall *)RPCToMethod:(NSString *)method
            requestsWriter:(GRXWriter *)requestsWriter
   	        responseClass:(Class)responseClass
   	   responsesWriteable:(id<GRXWriteable>)responsesWriteable;
 @end
+
+
+/**
+ * This subclass is empty now. Eventually we'll remove ProtoService class
+ * to avoid potential naming conflict
+ */
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+@interface GRPCProtoService : ProtoService
+#pragma clang diagnostic pop
+
+@end
diff --git a/src/objective-c/ProtoRPC/ProtoService.m b/src/objective-c/ProtoRPC/ProtoService.m
index fccc6aadc9..3487fac59d 100644
--- a/src/objective-c/ProtoRPC/ProtoService.m
+++ b/src/objective-c/ProtoRPC/ProtoService.m
@@ -65,17 +65,21 @@
   return self;
 }
 
-- (ProtoRPC *)RPCToMethod:(NSString *)method
-           requestsWriter:(GRXWriter *)requestsWriter
-            responseClass:(Class)responseClass
-       responsesWriteable:(id<GRXWriteable>)responsesWriteable {
-  ProtoMethod *methodName = [[ProtoMethod alloc] initWithPackage:_packageName
-                                                         service:_serviceName
-                                                          method:method];
-  return [[ProtoRPC alloc] initWithHost:_host
-                                 method:methodName
-                         requestsWriter:requestsWriter
-                          responseClass:responseClass
-                     responsesWriteable:responsesWriteable];
+- (GRPCProtoCall *)RPCToMethod:(NSString *)method
+                requestsWriter:(GRXWriter *)requestsWriter
+                 responseClass:(Class)responseClass
+            responsesWriteable:(id<GRXWriteable>)responsesWriteable {
+  GRPCProtoMethod *methodName = [[GRPCProtoMethod alloc] initWithPackage:_packageName
+                                                                 service:_serviceName
+                                                                  method:method];
+  return [[GRPCProtoCall alloc] initWithHost:_host
+                                      method:methodName
+                              requestsWriter:requestsWriter
+                               responseClass:responseClass
+                          responsesWriteable:responsesWriteable];
 }
 @end
+
+@implementation GRPCProtoService
+
+@end
diff --git a/src/objective-c/examples/Sample/Sample.xcodeproj/project.pbxproj b/src/objective-c/examples/Sample/Sample.xcodeproj/project.pbxproj
index 067ff3a13b..5c2a6d14f9 100644
--- a/src/objective-c/examples/Sample/Sample.xcodeproj/project.pbxproj
+++ b/src/objective-c/examples/Sample/Sample.xcodeproj/project.pbxproj
@@ -7,17 +7,16 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		426A5020E0E158A101BCA1D9 /* libPods-Sample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C20055928615A6F8434E26B4 /* libPods-Sample.a */; };
 		6369A2701A9322E20015FC5C /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 6369A26F1A9322E20015FC5C /* main.m */; };
 		6369A2731A9322E20015FC5C /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 6369A2721A9322E20015FC5C /* AppDelegate.m */; };
 		6369A2761A9322E20015FC5C /* ViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 6369A2751A9322E20015FC5C /* ViewController.m */; };
 		6369A2791A9322E20015FC5C /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 6369A2771A9322E20015FC5C /* Main.storyboard */; };
 		6369A27B1A9322E20015FC5C /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 6369A27A1A9322E20015FC5C /* Images.xcassets */; };
-		AF4A3AE4267AAFB18431B287 /* libPods-Sample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 0E797A3F68012317421BA87E /* libPods-Sample.a */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
-		0E797A3F68012317421BA87E /* libPods-Sample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-Sample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
-		2DC7B7C4C0410F43B9621631 /* libPods.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libPods.a; sourceTree = BUILT_PRODUCTS_DIR; };
+		5A8C9F4B28733B249DE4AB6D /* Pods-Sample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Sample.release.xcconfig"; path = "Pods/Target Support Files/Pods-Sample/Pods-Sample.release.xcconfig"; sourceTree = "<group>"; };
 		6369A26A1A9322E20015FC5C /* Sample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Sample.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		6369A26E1A9322E20015FC5C /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		6369A26F1A9322E20015FC5C /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = "<group>"; };
@@ -27,10 +26,8 @@
 		6369A2751A9322E20015FC5C /* ViewController.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ViewController.m; sourceTree = "<group>"; };
 		6369A2781A9322E20015FC5C /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
 		6369A27A1A9322E20015FC5C /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Images.xcassets; sourceTree = "<group>"; };
-		7F4A2C3C7ABEB427FB51922C /* Pods-Sample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Sample.release.xcconfig"; path = "Pods/Target Support Files/Pods-Sample/Pods-Sample.release.xcconfig"; sourceTree = "<group>"; };
-		7FB00EC776A4A5F68401AE2B /* Pods-Sample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Sample.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Sample/Pods-Sample.debug.xcconfig"; sourceTree = "<group>"; };
-		AC29DD6FCDF962F519FEBB0D /* Pods.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = Pods.debug.xcconfig; path = "Pods/Target Support Files/Pods/Pods.debug.xcconfig"; sourceTree = "<group>"; };
-		C68330F8D451CC6ACEABA09F /* Pods.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = Pods.release.xcconfig; path = "Pods/Target Support Files/Pods/Pods.release.xcconfig"; sourceTree = "<group>"; };
+		C20055928615A6F8434E26B4 /* libPods-Sample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-Sample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
+		E3C01DF315C4E7433BCEC6E6 /* Pods-Sample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Sample.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Sample/Pods-Sample.debug.xcconfig"; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -38,7 +35,7 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				AF4A3AE4267AAFB18431B287 /* libPods-Sample.a in Frameworks */,
+				426A5020E0E158A101BCA1D9 /* libPods-Sample.a in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -89,10 +86,8 @@
 		AB3331C9AE6488E61B2B094E /* Pods */ = {
 			isa = PBXGroup;
 			children = (
-				AC29DD6FCDF962F519FEBB0D /* Pods.debug.xcconfig */,
-				C68330F8D451CC6ACEABA09F /* Pods.release.xcconfig */,
-				7FB00EC776A4A5F68401AE2B /* Pods-Sample.debug.xcconfig */,
-				7F4A2C3C7ABEB427FB51922C /* Pods-Sample.release.xcconfig */,
+				E3C01DF315C4E7433BCEC6E6 /* Pods-Sample.debug.xcconfig */,
+				5A8C9F4B28733B249DE4AB6D /* Pods-Sample.release.xcconfig */,
 			);
 			name = Pods;
 			sourceTree = "<group>";
@@ -100,8 +95,7 @@
 		C4C2C5219053E079C9EFB930 /* Frameworks */ = {
 			isa = PBXGroup;
 			children = (
-				2DC7B7C4C0410F43B9621631 /* libPods.a */,
-				0E797A3F68012317421BA87E /* libPods-Sample.a */,
+				C20055928615A6F8434E26B4 /* libPods-Sample.a */,
 			);
 			name = Frameworks;
 			sourceTree = "<group>";
@@ -113,12 +107,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 6369A28D1A9322E20015FC5C /* Build configuration list for PBXNativeTarget "Sample" */;
 			buildPhases = (
-				41F7486D8F66994B0BFB84AF /* 📦 Check Pods Manifest.lock */,
+				41F7486D8F66994B0BFB84AF /* [CP] Check Pods Manifest.lock */,
 				6369A2661A9322E20015FC5C /* Sources */,
 				6369A2671A9322E20015FC5C /* Frameworks */,
 				6369A2681A9322E20015FC5C /* Resources */,
-				04554623324BE4A838846086 /* 📦 Copy Pods Resources */,
-				D2E32169A6ED4D88151F0046 /* 📦 Embed Pods Frameworks */,
+				04554623324BE4A838846086 /* [CP] Copy Pods Resources */,
+				C7FAD018D05AB5F0B0FE81E2 /* [CP] Embed Pods Frameworks */,
 			);
 			buildRules = (
 			);
@@ -174,14 +168,14 @@
 /* End PBXResourcesBuildPhase section */
 
 /* Begin PBXShellScriptBuildPhase section */
-		04554623324BE4A838846086 /* 📦 Copy Pods Resources */ = {
+		04554623324BE4A838846086 /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -189,14 +183,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-Sample/Pods-Sample-resources.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		41F7486D8F66994B0BFB84AF /* 📦 Check Pods Manifest.lock */ = {
+		41F7486D8F66994B0BFB84AF /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -204,14 +198,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		D2E32169A6ED4D88151F0046 /* 📦 Embed Pods Frameworks */ = {
+		C7FAD018D05AB5F0B0FE81E2 /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -326,7 +320,7 @@
 		};
 		6369A28E1A9322E20015FC5C /* Debug */ = {
 			isa = XCBuildConfiguration;
-			baseConfigurationReference = 7FB00EC776A4A5F68401AE2B /* Pods-Sample.debug.xcconfig */;
+			baseConfigurationReference = E3C01DF315C4E7433BCEC6E6 /* Pods-Sample.debug.xcconfig */;
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				INFOPLIST_FILE = Sample/Info.plist;
@@ -337,7 +331,7 @@
 		};
 		6369A28F1A9322E20015FC5C /* Release */ = {
 			isa = XCBuildConfiguration;
-			baseConfigurationReference = 7F4A2C3C7ABEB427FB51922C /* Pods-Sample.release.xcconfig */;
+			baseConfigurationReference = 5A8C9F4B28733B249DE4AB6D /* Pods-Sample.release.xcconfig */;
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				INFOPLIST_FILE = Sample/Info.plist;
diff --git a/src/objective-c/examples/Sample/Sample.xcodeproj/xcshareddata/xcschemes/Sample.xcscheme b/src/objective-c/examples/Sample/Sample.xcodeproj/xcshareddata/xcschemes/Sample.xcscheme
new file mode 100644
index 0000000000..d399e22e46
--- /dev/null
+++ b/src/objective-c/examples/Sample/Sample.xcodeproj/xcshareddata/xcschemes/Sample.xcscheme
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "0730"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "6369A2691A9322E20015FC5C"
+               BuildableName = "Sample.app"
+               BlueprintName = "Sample"
+               ReferencedContainer = "container:Sample.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+      </Testables>
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "6369A2691A9322E20015FC5C"
+            BuildableName = "Sample.app"
+            BlueprintName = "Sample"
+            ReferencedContainer = "container:Sample.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "6369A2691A9322E20015FC5C"
+            BuildableName = "Sample.app"
+            BlueprintName = "Sample"
+            ReferencedContainer = "container:Sample.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "6369A2691A9322E20015FC5C"
+            BuildableName = "Sample.app"
+            BlueprintName = "Sample"
+            ReferencedContainer = "container:Sample.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>
diff --git a/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/project.pbxproj b/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/project.pbxproj
index 2f9a41875d..2a1b30f2cf 100644
--- a/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/project.pbxproj
+++ b/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/project.pbxproj
@@ -11,12 +11,10 @@
 		633BFFCA1B950B210007E424 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 633BFFC91B950B210007E424 /* ViewController.swift */; };
 		633BFFCD1B950B210007E424 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 633BFFCB1B950B210007E424 /* Main.storyboard */; };
 		633BFFCF1B950B210007E424 /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 633BFFCE1B950B210007E424 /* Images.xcassets */; };
-		AE76C196CEB7CF33421129CD /* libPods-SwiftSample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FBDB042A015267195A988FFB /* libPods-SwiftSample.a */; };
+		92EDB1408A1E1E7DDAB25D9C /* libPods-SwiftSample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 69BB5C6CA3C1F97E007AC527 /* libPods-SwiftSample.a */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
-		12C7B447AA80E624D93B5C54 /* Pods.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = Pods.debug.xcconfig; path = "Pods/Target Support Files/Pods/Pods.debug.xcconfig"; sourceTree = "<group>"; };
-		5ABC04686A90EA11D0BD35A1 /* Pods-SwiftSample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SwiftSample.release.xcconfig"; path = "Pods/Target Support Files/Pods-SwiftSample/Pods-SwiftSample.release.xcconfig"; sourceTree = "<group>"; };
 		633BFFC21B950B210007E424 /* SwiftSample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SwiftSample.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		633BFFC61B950B210007E424 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		633BFFC71B950B210007E424 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
@@ -24,10 +22,9 @@
 		633BFFCC1B950B210007E424 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
 		633BFFCE1B950B210007E424 /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Images.xcassets; sourceTree = "<group>"; };
 		6367AD231B951655007FD3A4 /* Bridging-Header.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "Bridging-Header.h"; sourceTree = "<group>"; };
-		C335CBC4C160E0D9EDEE646B /* Pods.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = Pods.release.xcconfig; path = "Pods/Target Support Files/Pods/Pods.release.xcconfig"; sourceTree = "<group>"; };
-		C4278D3EA95326A34DF5D15F /* Pods-SwiftSample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SwiftSample.debug.xcconfig"; path = "Pods/Target Support Files/Pods-SwiftSample/Pods-SwiftSample.debug.xcconfig"; sourceTree = "<group>"; };
-		DC58ACA18DCCB1553531B885 /* libPods.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libPods.a; sourceTree = BUILT_PRODUCTS_DIR; };
-		FBDB042A015267195A988FFB /* libPods-SwiftSample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-SwiftSample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
+		69BB5C6CA3C1F97E007AC527 /* libPods-SwiftSample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-SwiftSample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
+		A7E614A494D89D01BB395761 /* Pods-SwiftSample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SwiftSample.debug.xcconfig"; path = "Pods/Target Support Files/Pods-SwiftSample/Pods-SwiftSample.debug.xcconfig"; sourceTree = "<group>"; };
+		C314E3E246AF23AC29B38FCF /* Pods-SwiftSample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SwiftSample.release.xcconfig"; path = "Pods/Target Support Files/Pods-SwiftSample/Pods-SwiftSample.release.xcconfig"; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -35,7 +32,7 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				AE76C196CEB7CF33421129CD /* libPods-SwiftSample.a in Frameworks */,
+				92EDB1408A1E1E7DDAB25D9C /* libPods-SwiftSample.a in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -45,10 +42,8 @@
 		31F283C976AE97586C17CCD9 /* Pods */ = {
 			isa = PBXGroup;
 			children = (
-				12C7B447AA80E624D93B5C54 /* Pods.debug.xcconfig */,
-				C335CBC4C160E0D9EDEE646B /* Pods.release.xcconfig */,
-				C4278D3EA95326A34DF5D15F /* Pods-SwiftSample.debug.xcconfig */,
-				5ABC04686A90EA11D0BD35A1 /* Pods-SwiftSample.release.xcconfig */,
+				A7E614A494D89D01BB395761 /* Pods-SwiftSample.debug.xcconfig */,
+				C314E3E246AF23AC29B38FCF /* Pods-SwiftSample.release.xcconfig */,
 			);
 			name = Pods;
 			sourceTree = "<group>";
@@ -95,8 +90,7 @@
 		9D63A7F6423989BA306810CA /* Frameworks */ = {
 			isa = PBXGroup;
 			children = (
-				DC58ACA18DCCB1553531B885 /* libPods.a */,
-				FBDB042A015267195A988FFB /* libPods-SwiftSample.a */,
+				69BB5C6CA3C1F97E007AC527 /* libPods-SwiftSample.a */,
 			);
 			name = Frameworks;
 			sourceTree = "<group>";
@@ -108,12 +102,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 633BFFE11B950B210007E424 /* Build configuration list for PBXNativeTarget "SwiftSample" */;
 			buildPhases = (
-				6BEEB33CA2705D7D2F2210E6 /* 📦 Check Pods Manifest.lock */,
+				6BEEB33CA2705D7D2F2210E6 /* [CP] Check Pods Manifest.lock */,
 				633BFFBE1B950B210007E424 /* Sources */,
 				633BFFBF1B950B210007E424 /* Frameworks */,
 				633BFFC01B950B210007E424 /* Resources */,
-				AC2F6F9AB1C090BB0BEE6E4D /* 📦 Copy Pods Resources */,
-				A1738A987353B0BF2C64F0F7 /* 📦 Embed Pods Frameworks */,
+				AC2F6F9AB1C090BB0BEE6E4D /* [CP] Copy Pods Resources */,
+				A1738A987353B0BF2C64F0F7 /* [CP] Embed Pods Frameworks */,
 			);
 			buildRules = (
 			);
@@ -170,14 +164,14 @@
 /* End PBXResourcesBuildPhase section */
 
 /* Begin PBXShellScriptBuildPhase section */
-		6BEEB33CA2705D7D2F2210E6 /* 📦 Check Pods Manifest.lock */ = {
+		6BEEB33CA2705D7D2F2210E6 /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -185,14 +179,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		A1738A987353B0BF2C64F0F7 /* 📦 Embed Pods Frameworks */ = {
+		A1738A987353B0BF2C64F0F7 /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -200,14 +194,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-SwiftSample/Pods-SwiftSample-frameworks.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		AC2F6F9AB1C090BB0BEE6E4D /* 📦 Copy Pods Resources */ = {
+		AC2F6F9AB1C090BB0BEE6E4D /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -326,7 +320,7 @@
 		};
 		633BFFE21B950B210007E424 /* Debug */ = {
 			isa = XCBuildConfiguration;
-			baseConfigurationReference = C4278D3EA95326A34DF5D15F /* Pods-SwiftSample.debug.xcconfig */;
+			baseConfigurationReference = A7E614A494D89D01BB395761 /* Pods-SwiftSample.debug.xcconfig */;
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				INFOPLIST_FILE = Info.plist;
@@ -339,7 +333,7 @@
 		};
 		633BFFE31B950B210007E424 /* Release */ = {
 			isa = XCBuildConfiguration;
-			baseConfigurationReference = 5ABC04686A90EA11D0BD35A1 /* Pods-SwiftSample.release.xcconfig */;
+			baseConfigurationReference = C314E3E246AF23AC29B38FCF /* Pods-SwiftSample.release.xcconfig */;
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				INFOPLIST_FILE = Info.plist;
diff --git a/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/xcshareddata/xcschemes/SwiftSample.xcscheme b/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/xcshareddata/xcschemes/SwiftSample.xcscheme
new file mode 100644
index 0000000000..bba6a02b2b
--- /dev/null
+++ b/src/objective-c/examples/SwiftSample/SwiftSample.xcodeproj/xcshareddata/xcschemes/SwiftSample.xcscheme
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "0730"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "633BFFC11B950B210007E424"
+               BuildableName = "SwiftSample.app"
+               BlueprintName = "SwiftSample"
+               ReferencedContainer = "container:SwiftSample.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+      </Testables>
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "633BFFC11B950B210007E424"
+            BuildableName = "SwiftSample.app"
+            BlueprintName = "SwiftSample"
+            ReferencedContainer = "container:SwiftSample.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "633BFFC11B950B210007E424"
+            BuildableName = "SwiftSample.app"
+            BlueprintName = "SwiftSample"
+            ReferencedContainer = "container:SwiftSample.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "633BFFC11B950B210007E424"
+            BuildableName = "SwiftSample.app"
+            BlueprintName = "SwiftSample"
+            ReferencedContainer = "container:SwiftSample.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>
diff --git a/src/objective-c/examples/SwiftSample/ViewController.swift b/src/objective-c/examples/SwiftSample/ViewController.swift
index a21ce07978..2a95d2de51 100644
--- a/src/objective-c/examples/SwiftSample/ViewController.swift
+++ b/src/objective-c/examples/SwiftSample/ViewController.swift
@@ -71,7 +71,8 @@ class ViewController: UIViewController {
       NSLog("2. Response trailers: \(RPC.responseTrailers)")
     }
 
-    RPC.requestHeaders["My-Header"] = "My value"
+    // TODO(jcanizales): Revert to using subscript syntax once XCode 8 is released.
+    RPC.requestHeaders.setObject("My value", forKey: "My-Header")
 
     RPC.start()
 
@@ -84,7 +85,7 @@ class ViewController: UIViewController {
 
     let call = GRPCCall(host: RemoteHost, path: method.HTTPPath, requestsWriter: requestsWriter)
 
-    call.requestHeaders["My-Header"] = "My value"
+    call.requestHeaders.setObject("My value", forKey: "My-Header")
 
     call.startWithWriteable(GRXWriteable { response, error in
       if let response = response as? NSData {
diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m
index 9a8d425324..1167a715bb 100644
--- a/src/objective-c/tests/GRPCClientTests.m
+++ b/src/objective-c/tests/GRPCClientTests.m
@@ -48,9 +48,9 @@ static NSString * const kPackage = @"grpc.testing";
 static NSString * const kService = @"TestService";
 static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com";
 
-static ProtoMethod *kInexistentMethod;
-static ProtoMethod *kEmptyCallMethod;
-static ProtoMethod *kUnaryCallMethod;
+static GRPCProtoMethod *kInexistentMethod;
+static GRPCProtoMethod *kEmptyCallMethod;
+static GRPCProtoMethod *kUnaryCallMethod;
 
 /** Observer class for testing that responseMetadata is KVO-compliant */
 @interface PassthroughObserver : NSObject
@@ -109,15 +109,15 @@ static ProtoMethod *kUnaryCallMethod;
   [GRPCCall useInsecureConnectionsForHost:kHostAddress];
 
   // This method isn't implemented by the remote server.
-  kInexistentMethod = [[ProtoMethod alloc] initWithPackage:kPackage
-                                                   service:kService
-                                                    method:@"Inexistent"];
-  kEmptyCallMethod = [[ProtoMethod alloc] initWithPackage:kPackage
-                                                  service:kService
-                                                   method:@"EmptyCall"];
-  kUnaryCallMethod = [[ProtoMethod alloc] initWithPackage:kPackage
-                                                  service:kService
-                                                   method:@"UnaryCall"];
+  kInexistentMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage
+                                                       service:kService
+                                                        method:@"Inexistent"];
+  kEmptyCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage
+                                                      service:kService
+                                                       method:@"EmptyCall"];
+  kUnaryCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage
+                                                      service:kService
+                                                       method:@"UnaryCall"];
 }
 
 - (void)testConnectionToRemoteServer {
@@ -303,9 +303,9 @@ static ProtoMethod *kUnaryCallMethod;
 
   // Try to set parameters to nil for GRPCCall. This should cause an exception
   @try {
-    GRPCCall *call = [[GRPCCall alloc] initWithHost:nil
-                                               path:nil
-                                     requestsWriter:nil];
+    (void)[[GRPCCall alloc] initWithHost:nil
+                                    path:nil
+                          requestsWriter:nil];
     XCTFail(@"Did not receive an exception when parameters are nil");
   } @catch(NSException *theException) {
     NSLog(@"Received exception as expected: %@", theException.name);
@@ -316,9 +316,9 @@ static ProtoMethod *kUnaryCallMethod;
   GRXWriter *requestsWriter = [GRXWriter emptyWriter];
   [requestsWriter finishWithError:nil];
   @try {
-    GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress
-                                               path:kUnaryCallMethod.HTTPPath
-                                     requestsWriter:requestsWriter];
+    (void)[[GRPCCall alloc] initWithHost:kHostAddress
+                                    path:kUnaryCallMethod.HTTPPath
+                          requestsWriter:requestsWriter];
     XCTFail(@"Did not receive an exception when GRXWriter has incorrect state.");
   } @catch(NSException *theException) {
     NSLog(@"Received exception as expected: %@", theException.name);
diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m
index 4f096b9efa..a503f02059 100644
--- a/src/objective-c/tests/InteropTests.m
+++ b/src/objective-c/tests/InteropTests.m
@@ -35,7 +35,10 @@
 
 #include <grpc/status.h>
 
+#import <Cronet/Cronet.h>
+#import <GRPCClient/GRPCCall+ChannelArg.h>
 #import <GRPCClient/GRPCCall+Tests.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 #import <ProtoRPC/ProtoRPC.h>
 #import <RemoteTest/Empty.pbobjc.h>
 #import <RemoteTest/Messages.pbobjc.h>
@@ -56,7 +59,7 @@
                  requestedResponseSize:(NSNumber *)responseSize {
   RMTStreamingOutputCallRequest *request = [self message];
   RMTResponseParameters *parameters = [RMTResponseParameters message];
-  parameters.size = responseSize.integerValue;
+  parameters.size = responseSize.intValue;
   [request.responseParametersArray addObject:parameters];
   request.payload.body = [NSMutableData dataWithLength:payloadSize.unsignedIntegerValue];
   return request;
@@ -78,6 +81,10 @@
 
 #pragma mark Tests
 
+#ifdef GRPC_COMPILE_WITH_CRONET
+static cronet_engine *cronetEngine = NULL;
+#endif
+
 @implementation InteropTests {
   RMTTestService *_service;
 }
@@ -88,6 +95,15 @@
 
 - (void)setUp {
   _service = self.class.host ? [RMTTestService serviceWithHost:self.class.host] : nil;
+#ifdef GRPC_COMPILE_WITH_CRONET
+  if (cronetEngine == NULL) {
+    // Cronet setup
+    [Cronet setHttp2Enabled:YES];
+    [Cronet start];
+    cronetEngine = [Cronet getGlobalEngine];
+    [GRPCCall useCronetWithEngine:cronetEngine];
+  }
+#endif
 }
 
 - (void)testEmptyUnaryRPC {
@@ -173,7 +189,7 @@
   RMTStreamingOutputCallRequest *request = [RMTStreamingOutputCallRequest message];
   for (NSNumber *size in expectedSizes) {
     RMTResponseParameters *parameters = [RMTResponseParameters message];
-    parameters.size = [size integerValue];
+    parameters.size = [size intValue];
     [request.responseParametersArray addObject:parameters];
   }
 
@@ -245,6 +261,8 @@
   [self waitForExpectationsWithTimeout:4 handler:nil];
 }
 
+#ifndef GRPC_COMPILE_WITH_CRONET
+// TODO(makdharma@): Fix this test
 - (void)testEmptyStreamRPC {
   XCTAssertNotNil(self.class.host);
   __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyStream"];
@@ -258,6 +276,7 @@
   }];
   [self waitForExpectationsWithTimeout:2 handler:nil];
 }
+#endif
 
 - (void)testCancelAfterBeginRPC {
   XCTAssertNotNil(self.class.host);
@@ -266,9 +285,10 @@
   // A buffered pipe to which we never write any value acts as a writer that just hangs.
   GRXBufferedPipe *requestsBuffer = [[GRXBufferedPipe alloc] init];
 
-  ProtoRPC *call = [_service RPCToStreamingInputCallWithRequestsWriter:requestsBuffer
-                                                               handler:^(RMTStreamingInputCallResponse *response,
-                                                                         NSError *error) {
+  GRPCProtoCall *call =
+      [_service RPCToStreamingInputCallWithRequestsWriter:requestsBuffer
+                                                  handler:^(RMTStreamingInputCallResponse *response,
+                                                            NSError *error) {
     XCTAssertEqual(error.code, GRPC_STATUS_CANCELLED);
     [expectation fulfill];
   }];
@@ -297,7 +317,7 @@
 
   [requestsBuffer writeValue:request];
 
-  __block ProtoRPC *call =
+  __block GRPCProtoCall *call =
       [_service RPCToFullDuplexCallWithRequestsWriter:requestsBuffer
                                          eventHandler:^(BOOL done,
                                                         RMTStreamingOutputCallResponse *response,
@@ -318,4 +338,28 @@
   [self waitForExpectationsWithTimeout:8 handler:nil];
 }
 
+- (void)testRPCAfterClosingOpenConnections {
+  XCTAssertNotNil(self.class.host);
+  __weak XCTestExpectation *expectation =
+      [self expectationWithDescription:@"RPC after closing connection"];
+
+  RMTEmpty *request = [RMTEmpty message];
+
+  [_service emptyCallWithRequest:request handler:^(RMTEmpty *response, NSError *error) {
+    XCTAssertNil(error, @"First RPC finished with unexpected error: %@", error);
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+    [GRPCCall closeOpenConnections];
+#pragma clang diagnostic pop
+
+    [_service emptyCallWithRequest:request handler:^(RMTEmpty *response, NSError *error) {
+      XCTAssertNil(error, @"Second RPC finished with unexpected error: %@", error);
+      [expectation fulfill];
+    }];
+  }];
+
+  [self waitForExpectationsWithTimeout:4 handler:nil];
+}
+
 @end
diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile
index b627a92e4f..64b0009eb1 100644
--- a/src/objective-c/tests/Podfile
+++ b/src/objective-c/tests/Podfile
@@ -16,8 +16,9 @@ GRPC_LOCAL_SRC = '../../..'
   InteropTestsLocalCleartext
 ).each do |target_name|
   target target_name do
-    pod 'Protobuf', :path => "#{GRPC_LOCAL_SRC}/third_party/protobuf"
-    pod 'BoringSSL', :podspec => "#{GRPC_LOCAL_SRC}/src/objective-c"
+    pod 'Protobuf', :path => "#{GRPC_LOCAL_SRC}/third_party/protobuf", :inhibit_warnings => true
+    pod 'BoringSSL', :podspec => "#{GRPC_LOCAL_SRC}/src/objective-c", :inhibit_warnings => true
+    pod 'CronetFramework', :podspec => "#{GRPC_LOCAL_SRC}/src/objective-c"
     pod 'gRPC', :path => GRPC_LOCAL_SRC
     pod 'gRPC-Core', :path => GRPC_LOCAL_SRC
     pod 'gRPC-RxLibrary', :path => GRPC_LOCAL_SRC
@@ -56,3 +57,19 @@ pre_install do |installer|
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
   }
 end
+
+post_install do |installer|
+  installer.pods_project.targets.each do |target|
+    target.build_configurations.each do |config|
+      config.build_settings['GCC_TREAT_WARNINGS_AS_ERRORS'] = 'YES'
+    end
+    if target.name == 'gRPC'
+      target.build_configurations.each do |config|
+        # TODO(zyc) Remove this setting after the issue is resolved
+        # GPR_UNREACHABLE_CODE causes "Control may reach end of non-void
+        # function" warning
+        config.build_settings['GCC_WARN_ABOUT_RETURN_TYPE'] = 'NO'
+      end
+    end
+  end
+end
diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj
index 89e0ea60b9..f9389a4977 100644
--- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj
+++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj
@@ -274,12 +274,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 63423F4D1B150A5F006CF63C /* Build configuration list for PBXNativeTarget "AllTests" */;
 			buildPhases = (
-				914ADDD7106BA9BB8A7E569F /* 📦 Check Pods Manifest.lock */,
+				914ADDD7106BA9BB8A7E569F /* [CP] Check Pods Manifest.lock */,
 				63423F401B150A5F006CF63C /* Sources */,
 				63423F411B150A5F006CF63C /* Frameworks */,
 				63423F421B150A5F006CF63C /* Resources */,
-				A441F71824DCB9D0CA297748 /* 📦 Copy Pods Resources */,
-				5F14F59509E10C2852014F9E /* 📦 Embed Pods Frameworks */,
+				A441F71824DCB9D0CA297748 /* [CP] Copy Pods Resources */,
+				5F14F59509E10C2852014F9E /* [CP] Embed Pods Frameworks */,
 			);
 			buildRules = (
 			);
@@ -295,11 +295,11 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 635697DB1B14FC11007A7283 /* Build configuration list for PBXNativeTarget "Tests" */;
 			buildPhases = (
-				796680C7599CB4ED736DD62A /* 📦 Check Pods Manifest.lock */,
+				796680C7599CB4ED736DD62A /* [CP] Check Pods Manifest.lock */,
 				635697C31B14FC11007A7283 /* Sources */,
 				635697C41B14FC11007A7283 /* Frameworks */,
 				635697C51B14FC11007A7283 /* CopyFiles */,
-				AEEBFC914CBAEE347382E8C4 /* 📦 Copy Pods Resources */,
+				AEEBFC914CBAEE347382E8C4 /* [CP] Copy Pods Resources */,
 			);
 			buildRules = (
 			);
@@ -314,12 +314,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 63DC841B1BE15179000708E8 /* Build configuration list for PBXNativeTarget "RxLibraryUnitTests" */;
 			buildPhases = (
-				B2986CEEE8CDD4901C97598B /* 📦 Check Pods Manifest.lock */,
+				B2986CEEE8CDD4901C97598B /* [CP] Check Pods Manifest.lock */,
 				63DC840F1BE15179000708E8 /* Sources */,
 				63DC84101BE15179000708E8 /* Frameworks */,
 				63DC84111BE15179000708E8 /* Resources */,
-				4F5690DC0E6AD6663FE78B8B /* 📦 Embed Pods Frameworks */,
-				C977426A8727267BBAC7D48E /* 📦 Copy Pods Resources */,
+				4F5690DC0E6AD6663FE78B8B /* [CP] Embed Pods Frameworks */,
+				C977426A8727267BBAC7D48E /* [CP] Copy Pods Resources */,
 			);
 			buildRules = (
 			);
@@ -335,12 +335,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 63DC842B1BE15267000708E8 /* Build configuration list for PBXNativeTarget "InteropTestsRemote" */;
 			buildPhases = (
-				4C406327D3907A5E5FBA8AC9 /* 📦 Check Pods Manifest.lock */,
+				4C406327D3907A5E5FBA8AC9 /* [CP] Check Pods Manifest.lock */,
 				63DC841F1BE15267000708E8 /* Sources */,
 				63DC84201BE15267000708E8 /* Frameworks */,
 				63DC84211BE15267000708E8 /* Resources */,
-				900B6EDD4D16BE7D765C3885 /* 📦 Embed Pods Frameworks */,
-				C2E09DC4BD239F71160F0CC1 /* 📦 Copy Pods Resources */,
+				900B6EDD4D16BE7D765C3885 /* [CP] Embed Pods Frameworks */,
+				C2E09DC4BD239F71160F0CC1 /* [CP] Copy Pods Resources */,
 			);
 			buildRules = (
 			);
@@ -356,12 +356,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 63DC843C1BE15294000708E8 /* Build configuration list for PBXNativeTarget "InteropTestsLocalSSL" */;
 			buildPhases = (
-				5C20DCCB71C3991E6FE78C22 /* 📦 Check Pods Manifest.lock */,
+				5C20DCCB71C3991E6FE78C22 /* [CP] Check Pods Manifest.lock */,
 				63DC84301BE15294000708E8 /* Sources */,
 				63DC84311BE15294000708E8 /* Frameworks */,
 				63DC84321BE15294000708E8 /* Resources */,
-				C591129ACE9F6CC5EE03FCDE /* 📦 Embed Pods Frameworks */,
-				693DD0B453431D64EA24FD66 /* 📦 Copy Pods Resources */,
+				C591129ACE9F6CC5EE03FCDE /* [CP] Embed Pods Frameworks */,
+				693DD0B453431D64EA24FD66 /* [CP] Copy Pods Resources */,
 			);
 			buildRules = (
 			);
@@ -377,12 +377,12 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 63DC844B1BE152B5000708E8 /* Build configuration list for PBXNativeTarget "InteropTestsLocalCleartext" */;
 			buildPhases = (
-				7418AC7B3844B29E48D24FC7 /* 📦 Check Pods Manifest.lock */,
+				7418AC7B3844B29E48D24FC7 /* [CP] Check Pods Manifest.lock */,
 				63DC843F1BE152B5000708E8 /* Sources */,
 				63DC84401BE152B5000708E8 /* Frameworks */,
 				63DC84411BE152B5000708E8 /* Resources */,
-				A8E3AC66DF770B774114A30E /* 📦 Embed Pods Frameworks */,
-				8AD3130D3C58A0FB32FF2A36 /* 📦 Copy Pods Resources */,
+				A8E3AC66DF770B774114A30E /* [CP] Embed Pods Frameworks */,
+				8AD3130D3C58A0FB32FF2A36 /* [CP] Copy Pods Resources */,
 			);
 			buildRules = (
 			);
@@ -486,14 +486,14 @@
 /* End PBXResourcesBuildPhase section */
 
 /* Begin PBXShellScriptBuildPhase section */
-		4C406327D3907A5E5FBA8AC9 /* 📦 Check Pods Manifest.lock */ = {
+		4C406327D3907A5E5FBA8AC9 /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -501,14 +501,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		4F5690DC0E6AD6663FE78B8B /* 📦 Embed Pods Frameworks */ = {
+		4F5690DC0E6AD6663FE78B8B /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -516,14 +516,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-RxLibraryUnitTests/Pods-RxLibraryUnitTests-frameworks.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		5C20DCCB71C3991E6FE78C22 /* 📦 Check Pods Manifest.lock */ = {
+		5C20DCCB71C3991E6FE78C22 /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -531,14 +531,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		5F14F59509E10C2852014F9E /* 📦 Embed Pods Frameworks */ = {
+		5F14F59509E10C2852014F9E /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -546,14 +546,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-AllTests/Pods-AllTests-frameworks.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		693DD0B453431D64EA24FD66 /* 📦 Copy Pods Resources */ = {
+		693DD0B453431D64EA24FD66 /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -561,14 +561,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsLocalSSL/Pods-InteropTestsLocalSSL-resources.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		7418AC7B3844B29E48D24FC7 /* 📦 Check Pods Manifest.lock */ = {
+		7418AC7B3844B29E48D24FC7 /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -576,14 +576,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		796680C7599CB4ED736DD62A /* 📦 Check Pods Manifest.lock */ = {
+		796680C7599CB4ED736DD62A /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -591,14 +591,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		8AD3130D3C58A0FB32FF2A36 /* 📦 Copy Pods Resources */ = {
+		8AD3130D3C58A0FB32FF2A36 /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -606,14 +606,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsLocalCleartext/Pods-InteropTestsLocalCleartext-resources.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		900B6EDD4D16BE7D765C3885 /* 📦 Embed Pods Frameworks */ = {
+		900B6EDD4D16BE7D765C3885 /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -621,14 +621,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsRemote/Pods-InteropTestsRemote-frameworks.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		914ADDD7106BA9BB8A7E569F /* 📦 Check Pods Manifest.lock */ = {
+		914ADDD7106BA9BB8A7E569F /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -636,14 +636,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		A441F71824DCB9D0CA297748 /* 📦 Copy Pods Resources */ = {
+		A441F71824DCB9D0CA297748 /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -651,14 +651,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-AllTests/Pods-AllTests-resources.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		A8E3AC66DF770B774114A30E /* 📦 Embed Pods Frameworks */ = {
+		A8E3AC66DF770B774114A30E /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -666,14 +666,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsLocalCleartext/Pods-InteropTestsLocalCleartext-frameworks.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		AEEBFC914CBAEE347382E8C4 /* 📦 Copy Pods Resources */ = {
+		AEEBFC914CBAEE347382E8C4 /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -681,14 +681,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-Tests/Pods-Tests-resources.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		B2986CEEE8CDD4901C97598B /* 📦 Check Pods Manifest.lock */ = {
+		B2986CEEE8CDD4901C97598B /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Check Pods Manifest.lock";
+			name = "[CP] Check Pods Manifest.lock";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -696,14 +696,14 @@
 			shellScript = "diff \"${PODS_ROOT}/../Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [[ $? != 0 ]] ; then\n    cat << EOM\nerror: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\nEOM\n    exit 1\nfi\n";
 			showEnvVarsInLog = 0;
 		};
-		C2E09DC4BD239F71160F0CC1 /* 📦 Copy Pods Resources */ = {
+		C2E09DC4BD239F71160F0CC1 /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -711,14 +711,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsRemote/Pods-InteropTestsRemote-resources.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		C591129ACE9F6CC5EE03FCDE /* 📦 Embed Pods Frameworks */ = {
+		C591129ACE9F6CC5EE03FCDE /* [CP] Embed Pods Frameworks */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Embed Pods Frameworks";
+			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -726,14 +726,14 @@
 			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsLocalSSL/Pods-InteropTestsLocalSSL-frameworks.sh\"\n";
 			showEnvVarsInLog = 0;
 		};
-		C977426A8727267BBAC7D48E /* 📦 Copy Pods Resources */ = {
+		C977426A8727267BBAC7D48E /* [CP] Copy Pods Resources */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
 			);
 			inputPaths = (
 			);
-			name = "📦 Copy Pods Resources";
+			name = "[CP] Copy Pods Resources";
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -893,6 +893,7 @@
 					"$(inherited)",
 				);
 				GCC_SYMBOLS_PRIVATE_EXTERN = NO;
+				GCC_TREAT_WARNINGS_AS_ERRORS = YES;
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
 				GCC_WARN_UNDECLARED_SELECTOR = YES;
@@ -929,6 +930,7 @@
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				GCC_C_LANGUAGE_STANDARD = gnu99;
 				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_TREAT_WARNINGS_AS_ERRORS = YES;
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
 				GCC_WARN_UNDECLARED_SELECTOR = YES;
@@ -946,6 +948,7 @@
 			isa = XCBuildConfiguration;
 			baseConfigurationReference = 060EF32D7EC0DF67ED617507 /* Pods-Tests.debug.xcconfig */;
 			buildSettings = {
+				GCC_TREAT_WARNINGS_AS_ERRORS = YES;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				SKIP_INSTALL = YES;
 			};
@@ -955,6 +958,7 @@
 			isa = XCBuildConfiguration;
 			baseConfigurationReference = E6733B838B28453434B556E2 /* Pods-Tests.release.xcconfig */;
 			buildSettings = {
+				GCC_TREAT_WARNINGS_AS_ERRORS = YES;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				SKIP_INSTALL = YES;
 			};
diff --git a/src/objective-c/tests/build_tests.sh b/src/objective-c/tests/build_tests.sh
index e7ad31e403..8547bfd3a8 100755
--- a/src/objective-c/tests/build_tests.sh
+++ b/src/objective-c/tests/build_tests.sh
@@ -33,6 +33,9 @@
 
 set -e
 
+# CocoaPods requires the terminal to be using UTF-8 encoding.
+export LANG=en_US.UTF-8
+
 cd $(dirname $0)
 
 hash pod 2>/dev/null || { echo >&2 "Cocoapods needs to be installed."; exit 1; }
diff --git a/src/php/README.md b/src/php/README.md
index cf8f2c11b0..6cc1ba4d46 100644
--- a/src/php/README.md
+++ b/src/php/README.md
@@ -58,7 +58,7 @@ To run tests with generated stub code from `.proto` files, you will also need th
 Clone this repository
 
 ```sh
-$ git clone https://github.com/grpc/grpc.git
+$ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc
 ```
 
 Build and install the gRPC C core library
@@ -101,7 +101,7 @@ extension=grpc.so
 You will need the source code to run tests
 
 ```sh
-$ git clone https://github.com/grpc/grpc.git
+$ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc
 $ cd grpc
 $ git pull --recurse-submodules && git submodule update --init --recursive
 ```
diff --git a/src/python/grpcio/grpc/_adapter/_implementations.py b/src/php/bin/stress_client.sh
index b85f228bf6..8c49b7aa4b 100644..100755
--- a/src/python/grpcio/grpc/_adapter/_implementations.py
+++ b/src/php/bin/stress_client.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -27,22 +28,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import collections
-
-from grpc.beta import interfaces
-
-class AuthMetadataContext(collections.namedtuple(
-    'AuthMetadataContext', [
-        'service_url',
-        'method_name'
-    ]), interfaces.GRPCAuthMetadataContext):
-  pass
-
-
-class AuthMetadataPluginCallback(interfaces.GRPCAuthMetadataContext):
-
-  def __init__(self, callback):
-    self._callback = callback
-
-  def __call__(self, metadata, error):
-    self._callback(metadata, error)
+set -e
+cd $(dirname $0)
+source ./determine_extension_dir.sh
+php $extension_dir -d max_execution_time=300 \
+  ../tests/interop/stress_client.php $@ 1>&2
diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c
index 8031f26768..2cd45f10dc 100644
--- a/src/php/ext/grpc/call.c
+++ b/src/php/ext/grpc/call.c
@@ -96,6 +96,7 @@ zval *grpc_php_wrap_call(grpc_call *wrapped, bool owned TSRMLS_DC) {
   wrapped_grpc_call *call =
       (wrapped_grpc_call *)zend_object_store_get_object(call_object TSRMLS_CC);
   call->wrapped = wrapped;
+  call->owned = owned;
   return call_object;
 }
 
@@ -247,6 +248,7 @@ PHP_METHOD(Call, __construct) {
   call->wrapped = grpc_channel_create_call(
       channel->wrapped, NULL, GRPC_PROPAGATE_DEFAULTS, completion_queue, method,
       host_override, deadline->wrapped, NULL);
+  call->owned = true;
 }
 
 /**
@@ -291,6 +293,7 @@ PHP_METHOD(Call, startBatch) {
   grpc_metadata_array_init(&recv_trailing_metadata);
   MAKE_STD_ZVAL(result);
   object_init(result);
+  memset(ops, 0, sizeof(ops));
   /* "a" == 1 array */
   if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "a", &array) ==
       FAILURE) {
diff --git a/src/php/ext/grpc/channel_credentials.c b/src/php/ext/grpc/channel_credentials.c
index 5c537378a6..b76fb105f3 100644
--- a/src/php/ext/grpc/channel_credentials.c
+++ b/src/php/ext/grpc/channel_credentials.c
@@ -47,11 +47,23 @@
 #include <zend_exceptions.h>
 #include <zend_hash.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
 
 zend_class_entry *grpc_ce_channel_credentials;
 
+static char *default_pem_root_certs = NULL;
+
+static grpc_ssl_roots_override_result get_ssl_roots_override(
+    char **pem_root_certs) {
+  *pem_root_certs = default_pem_root_certs;
+  if (default_pem_root_certs == NULL) {
+    return GRPC_SSL_ROOTS_OVERRIDE_FAIL;
+  }
+  return GRPC_SSL_ROOTS_OVERRIDE_OK;
+}
+
 /* Frees and destroys an instance of wrapped_grpc_channel_credentials */
 void free_wrapped_grpc_channel_credentials(void *object TSRMLS_DC) {
   wrapped_grpc_channel_credentials *creds =
@@ -94,6 +106,24 @@ zval *grpc_php_wrap_channel_credentials(grpc_channel_credentials *wrapped TSRMLS
 }
 
 /**
+ * Set default roots pem.
+ * @param string pem_roots PEM encoding of the server root certificates
+ * @return void
+ */
+PHP_METHOD(ChannelCredentials, setDefaultRootsPem) {
+  char *pem_roots;
+  int pem_roots_length;
+  if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s", &pem_roots,
+                            &pem_roots_length) == FAILURE) {
+    zend_throw_exception(spl_ce_InvalidArgumentException,
+                         "setDefaultRootsPem expects 1 string", 1 TSRMLS_CC);
+    return;
+  }
+  default_pem_root_certs = gpr_malloc((pem_roots_length + 1) * sizeof(char));
+  memcpy(default_pem_root_certs, pem_roots, pem_roots_length + 1);
+}
+
+/**
  * Create a default channel credentials object.
  * @return ChannelCredentials The new default channel credentials object
  */
@@ -178,6 +208,8 @@ PHP_METHOD(ChannelCredentials, createInsecure) {
 }
 
 static zend_function_entry channel_credentials_methods[] = {
+  PHP_ME(ChannelCredentials, setDefaultRootsPem, NULL,
+         ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
   PHP_ME(ChannelCredentials, createDefault, NULL,
          ZEND_ACC_PUBLIC | ZEND_ACC_STATIC)
   PHP_ME(ChannelCredentials, createSsl, NULL,
@@ -192,6 +224,7 @@ void grpc_init_channel_credentials(TSRMLS_D) {
   zend_class_entry ce;
   INIT_CLASS_ENTRY(ce, "Grpc\\ChannelCredentials",
                    channel_credentials_methods);
+  grpc_set_ssl_roots_override_callback(get_ssl_roots_override);
   ce.create_object = create_wrapped_grpc_channel_credentials;
   grpc_ce_channel_credentials = zend_register_internal_class(&ce TSRMLS_CC);
 }
diff --git a/src/php/ext/grpc/php_grpc.c b/src/php/ext/grpc/php_grpc.c
index 762c01385c..f4cb5b28cc 100644
--- a/src/php/ext/grpc/php_grpc.c
+++ b/src/php/ext/grpc/php_grpc.c
@@ -227,7 +227,7 @@ PHP_MINIT_FUNCTION(grpc) {
                          GRPC_CHANNEL_TRANSIENT_FAILURE,
                          CONST_CS | CONST_PERSISTENT);
   REGISTER_LONG_CONSTANT("Grpc\\CHANNEL_FATAL_FAILURE",
-                         GRPC_CHANNEL_FATAL_FAILURE,
+                         GRPC_CHANNEL_SHUTDOWN,
                          CONST_CS | CONST_PERSISTENT);
 
   grpc_init_call(TSRMLS_C);
diff --git a/src/php/lib/Grpc/AbstractCall.php b/src/php/lib/Grpc/AbstractCall.php
index 712af91eb2..c86d298805 100644
--- a/src/php/lib/Grpc/AbstractCall.php
+++ b/src/php/lib/Grpc/AbstractCall.php
@@ -39,6 +39,7 @@ abstract class AbstractCall
     protected $call;
     protected $deserialize;
     protected $metadata;
+    protected $trailing_metadata;
 
     /**
      * Create a new Call wrapper object.
@@ -66,6 +67,7 @@ abstract class AbstractCall
         $this->call = new Call($channel, $method, $deadline);
         $this->deserialize = $deserialize;
         $this->metadata = null;
+        $this->trailing_metadata = null;
         if (isset($options['call_credentials_callback']) &&
             is_callable($call_credentials_callback =
                         $options['call_credentials_callback'])) {
@@ -84,6 +86,14 @@ abstract class AbstractCall
     }
 
     /**
+     * @return The trailing metadata sent by the server.
+     */
+    public function getTrailingMetadata()
+    {
+        return $this->trailing_metadata;
+    }
+
+    /**
      * @return string The URI of the endpoint.
      */
     public function getPeer()
diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php
index 2de1b337e5..df3fe85d44 100755
--- a/src/php/lib/Grpc/BaseStub.php
+++ b/src/php/lib/Grpc/BaseStub.php
@@ -52,9 +52,14 @@ class BaseStub
      *  - 'update_metadata': (optional) a callback function which takes in a
      * metadata array, and returns an updated metadata array
      *  - 'grpc.primary_user_agent': (optional) a user-agent string
+     * @param $channel Channel An already created Channel object
      */
-    public function __construct($hostname, $opts)
+    public function __construct($hostname, $opts, $channel = null)
     {
+        $ssl_roots = file_get_contents(
+            dirname(__FILE__).'/../../../../etc/roots.pem');
+        ChannelCredentials::setDefaultRootsPem($ssl_roots);
+
         $this->hostname = $hostname;
         $this->update_metadata = null;
         if (isset($opts['update_metadata'])) {
@@ -77,7 +82,15 @@ class BaseStub
                                  'required. Please see one of the '.
                                  'ChannelCredentials::create methods');
         }
-        $this->channel = new Channel($hostname, $opts);
+        if ($channel) {
+            if (!is_a($channel, 'Channel')) {
+                throw new \Exception("The channel argument is not a".
+                                     "Channel object");
+            }
+            $this->channel = $channel;
+        } else {
+            $this->channel = new Channel($hostname, $opts);
+        }
     }
 
     /**
diff --git a/src/php/lib/Grpc/BidiStreamingCall.php b/src/php/lib/Grpc/BidiStreamingCall.php
index bf813c12e7..95e51c5088 100644
--- a/src/php/lib/Grpc/BidiStreamingCall.php
+++ b/src/php/lib/Grpc/BidiStreamingCall.php
@@ -112,6 +112,7 @@ class BidiStreamingCall extends AbstractCall
             OP_RECV_STATUS_ON_CLIENT => true,
         ]);
 
+        $this->trailing_metadata = $status_event->status->metadata;
         return $status_event->status;
     }
 }
diff --git a/src/php/lib/Grpc/ClientStreamingCall.php b/src/php/lib/Grpc/ClientStreamingCall.php
index 500cfe0d7a..315a406735 100644
--- a/src/php/lib/Grpc/ClientStreamingCall.php
+++ b/src/php/lib/Grpc/ClientStreamingCall.php
@@ -86,6 +86,8 @@ class ClientStreamingCall extends AbstractCall
         ]);
         $this->metadata = $event->metadata;
 
-        return [$this->deserializeResponse($event->message), $event->status];
+        $status = $event->status;
+        $this->trailing_metadata = $status->metadata;
+        return [$this->deserializeResponse($event->message), $status];
     }
 }
diff --git a/src/php/lib/Grpc/ServerStreamingCall.php b/src/php/lib/Grpc/ServerStreamingCall.php
index da48523717..53599fe4ae 100644
--- a/src/php/lib/Grpc/ServerStreamingCall.php
+++ b/src/php/lib/Grpc/ServerStreamingCall.php
@@ -91,6 +91,7 @@ class ServerStreamingCall extends AbstractCall
             OP_RECV_STATUS_ON_CLIENT => true,
         ]);
 
+        $this->trailing_metadata = $status_event->status->metadata;
         return $status_event->status;
     }
 }
diff --git a/src/php/lib/Grpc/UnaryCall.php b/src/php/lib/Grpc/UnaryCall.php
index b57903d6d0..b114b771b8 100644
--- a/src/php/lib/Grpc/UnaryCall.php
+++ b/src/php/lib/Grpc/UnaryCall.php
@@ -75,6 +75,8 @@ class UnaryCall extends AbstractCall
             OP_RECV_STATUS_ON_CLIENT => true,
         ]);
 
-        return [$this->deserializeResponse($event->message), $event->status];
+        $status = $event->status;
+        $this->trailing_metadata = $status->metadata;
+        return [$this->deserializeResponse($event->message), $status];
     }
 }
diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php
index aebf80f6bf..43b3199d92 100755
--- a/src/php/tests/interop/interop_client.php
+++ b/src/php/tests/interop/interop_client.php
@@ -1,7 +1,7 @@
 <?php
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -388,141 +388,269 @@ function timeoutOnSleepingServer($stub)
              'Call status was not DEADLINE_EXCEEDED');
 }
 
-$args = getopt('', ['server_host:', 'server_port:', 'test_case:',
-                    'use_tls::', 'use_test_ca::',
-                    'server_host_override:', 'oauth_scope:',
-                    'default_service_account:', ]);
-if (!array_key_exists('server_host', $args)) {
-    throw new Exception('Missing argument: --server_host is required');
-}
-if (!array_key_exists('server_port', $args)) {
-    throw new Exception('Missing argument: --server_port is required');
-}
-if (!array_key_exists('test_case', $args)) {
-    throw new Exception('Missing argument: --test_case is required');
-}
+function customMetadata($stub)
+{
+    $ECHO_INITIAL_KEY = 'x-grpc-test-echo-initial';
+    $ECHO_INITIAL_VALUE = 'test_initial_metadata_value';
+    $ECHO_TRAILING_KEY = 'x-grpc-test-echo-trailing-bin';
+    $ECHO_TRAILING_VALUE = 'ababab';
+    $request_len = 271828;
+    $response_len = 314159;
 
-if ($args['server_port'] == 443) {
-    $server_address = $args['server_host'];
-} else {
-    $server_address = $args['server_host'].':'.$args['server_port'];
-}
+    $request = new grpc\testing\SimpleRequest();
+    $request->setResponseType(grpc\testing\PayloadType::COMPRESSABLE);
+    $request->setResponseSize($response_len);
+    $payload = new grpc\testing\Payload();
+    $payload->setType(grpc\testing\PayloadType::COMPRESSABLE);
+    $payload->setBody(str_repeat("\0", $request_len));
+    $request->setPayload($payload);
 
-$test_case = $args['test_case'];
+    $metadata = [
+        $ECHO_INITIAL_KEY => [$ECHO_INITIAL_VALUE],
+        $ECHO_TRAILING_KEY => [$ECHO_TRAILING_VALUE],
+    ];
+    $call = $stub->UnaryCall($request, $metadata);
 
-$host_override = 'foo.test.google.fr';
-if (array_key_exists('server_host_override', $args)) {
-    $host_override = $args['server_host_override'];
+    $initial_metadata = $call->getMetadata();
+    hardAssert(array_key_exists($ECHO_INITIAL_KEY, $initial_metadata),
+               'Initial metadata does not contain expected key');
+    hardAssert($initial_metadata[$ECHO_INITIAL_KEY][0] ==
+               $ECHO_INITIAL_VALUE,
+               'Incorrect initial metadata value');
+
+    list($result, $status) = $call->wait();
+    hardAssert($status->code === Grpc\STATUS_OK,
+               'Call did not complete successfully');
+
+    $trailing_metadata = $call->getTrailingMetadata();
+    hardAssert(array_key_exists($ECHO_TRAILING_KEY, $trailing_metadata),
+               'Trailing metadata does not contain expected key');
+    hardAssert($trailing_metadata[$ECHO_TRAILING_KEY][0] ==
+               $ECHO_TRAILING_VALUE, 'Incorrect trailing metadata value');
+
+    $streaming_call = $stub->FullDuplexCall($metadata);
+
+    $streaming_request = new grpc\testing\StreamingOutputCallRequest();
+    $streaming_request->setPayload($payload);
+    $streaming_call->write($streaming_request);
+    $streaming_call->writesDone();
+
+    hardAssert($streaming_call->getStatus()->code === Grpc\STATUS_OK,
+               'Call did not complete successfully');
+
+    $streaming_trailing_metadata = $streaming_call->getTrailingMetadata();
+    hardAssert(array_key_exists($ECHO_TRAILING_KEY,
+                                $streaming_trailing_metadata),
+               'Trailing metadata does not contain expected key');
+    hardAssert($streaming_trailing_metadata[$ECHO_TRAILING_KEY][0] ==
+               $ECHO_TRAILING_VALUE, 'Incorrect trailing metadata value');
 }
 
-$use_tls = false;
-if (array_key_exists('use_tls', $args) &&
-    $args['use_tls'] != 'false') {
-    $use_tls = true;
+function statusCodeAndMessage($stub)
+{
+    $echo_status = new grpc\testing\EchoStatus();
+    $echo_status->setCode(2);
+    $echo_status->setMessage("test status message");
+
+    $request = new grpc\testing\SimpleRequest();
+    $request->setResponseStatus($echo_status);
+
+    $call = $stub->UnaryCall($request);
+    list($result, $status) = $call->wait();
+
+    hardAssert($status->code === 2,
+               'Received unexpected status code');
+    hardAssert($status->details === "test status message",
+               'Received unexpected status details');
+
+    $streaming_call = $stub->FullDuplexCall();
+
+    $streaming_request = new grpc\testing\StreamingOutputCallRequest();
+    $streaming_request->setResponseStatus($echo_status);
+    $streaming_call->write($streaming_request);
+    $streaming_call->writesDone();
+
+    $status = $streaming_call->getStatus();
+    hardAssert($status->code === 2,
+               'Received unexpected status code');
+    hardAssert($status->details === "test status message",
+               'Received unexpected status details');
 }
 
-$use_test_ca = false;
-if (array_key_exists('use_test_ca', $args) &&
-    $args['use_test_ca'] != 'false') {
-    $use_test_ca = true;
+function unimplementedMethod($stub)
+{
+    $call = $stub->UnimplementedCall(new grpc\testing\EmptyMessage());
+    list($result, $status) = $call->wait();
+    hardAssert($status->code === Grpc\STATUS_UNIMPLEMENTED,
+               'Received unexpected status code');
 }
 
-$opts = [];
+function _makeStub($args)
+{
+    if (!array_key_exists('server_host', $args)) {
+        throw new Exception('Missing argument: --server_host is required');
+    }
+    if (!array_key_exists('server_port', $args)) {
+        throw new Exception('Missing argument: --server_port is required');
+    }
+    if (!array_key_exists('test_case', $args)) {
+        throw new Exception('Missing argument: --test_case is required');
+    }
 
-if ($use_tls) {
-    if ($use_test_ca) {
-        $ssl_credentials = Grpc\ChannelCredentials::createSsl(
-            file_get_contents(dirname(__FILE__).'/../data/ca.pem'));
+    if ($args['server_port'] == 443) {
+        $server_address = $args['server_host'];
     } else {
-        $ssl_credentials = Grpc\ChannelCredentials::createSsl();
+        $server_address = $args['server_host'].':'.$args['server_port'];
     }
-    $opts['credentials'] = $ssl_credentials;
-    $opts['grpc.ssl_target_name_override'] = $host_override;
-} else {
-    $opts['credentials'] = Grpc\ChannelCredentials::createInsecure();
-}
 
-if (in_array($test_case, ['service_account_creds',
-    'compute_engine_creds', 'jwt_token_creds', ])) {
-    if ($test_case == 'jwt_token_creds') {
-        $auth_credentials = ApplicationDefaultCredentials::getCredentials();
+    $test_case = $args['test_case'];
+
+    $host_override = 'foo.test.google.fr';
+    if (array_key_exists('server_host_override', $args)) {
+        $host_override = $args['server_host_override'];
+    }
+
+    $use_tls = false;
+    if (array_key_exists('use_tls', $args) &&
+        $args['use_tls'] != 'false') {
+        $use_tls = true;
+    }
+
+    $use_test_ca = false;
+    if (array_key_exists('use_test_ca', $args) &&
+        $args['use_test_ca'] != 'false') {
+        $use_test_ca = true;
+    }
+
+    $opts = [];
+
+    if ($use_tls) {
+        if ($use_test_ca) {
+            $ssl_credentials = Grpc\ChannelCredentials::createSsl(
+                file_get_contents(dirname(__FILE__).'/../data/ca.pem'));
+        } else {
+            $ssl_credentials = Grpc\ChannelCredentials::createSsl();
+        }
+        $opts['credentials'] = $ssl_credentials;
+        $opts['grpc.ssl_target_name_override'] = $host_override;
     } else {
+        $opts['credentials'] = Grpc\ChannelCredentials::createInsecure();
+    }
+
+    if (in_array($test_case, ['service_account_creds',
+                              'compute_engine_creds', 'jwt_token_creds', ])) {
+        if ($test_case == 'jwt_token_creds') {
+            $auth_credentials = ApplicationDefaultCredentials::getCredentials();
+        } else {
+            $auth_credentials = ApplicationDefaultCredentials::getCredentials(
+                $args['oauth_scope']
+            );
+        }
+        $opts['update_metadata'] = $auth_credentials->getUpdateMetadataFunc();
+    }
+
+    if ($test_case == 'oauth2_auth_token') {
         $auth_credentials = ApplicationDefaultCredentials::getCredentials(
             $args['oauth_scope']
         );
+        $token = $auth_credentials->fetchAuthToken();
+        $update_metadata =
+            function ($metadata,
+                      $authUri = null,
+                      ClientInterface $client = null) use ($token) {
+                $metadata_copy = $metadata;
+                $metadata_copy[CredentialsLoader::AUTH_METADATA_KEY] =
+                    [sprintf('%s %s',
+                             $token['token_type'],
+                             $token['access_token'])];
+
+                return $metadata_copy;
+            };
+        $opts['update_metadata'] = $update_metadata;
     }
-    $opts['update_metadata'] = $auth_credentials->getUpdateMetadataFunc();
+
+    if ($test_case == 'unimplemented_method') {
+      $stub = new grpc\testing\UnimplementedServiceClient($server_address, $opts);
+    } else {
+      $stub = new grpc\testing\TestServiceClient($server_address, $opts);
+    }
+
+    return $stub;
 }
 
-if ($test_case == 'oauth2_auth_token') {
-    $auth_credentials = ApplicationDefaultCredentials::getCredentials(
-        $args['oauth_scope']
-    );
-    $token = $auth_credentials->fetchAuthToken();
-    $update_metadata =
-        function ($metadata,
-                  $authUri = null,
-                  ClientInterface $client = null) use ($token) {
-            $metadata_copy = $metadata;
-            $metadata_copy[CredentialsLoader::AUTH_METADATA_KEY] =
-                [sprintf('%s %s',
-                         $token['token_type'],
-                         $token['access_token'])];
-
-            return $metadata_copy;
-        };
-    $opts['update_metadata'] = $update_metadata;
+function interop_main($args, $stub = false)
+{
+    if (!$stub) {
+        $stub = _makeStub($args);
+    }
+
+    $test_case = $args['test_case'];
+    echo "Running test case $test_case\n";
+
+    switch ($test_case) {
+        case 'empty_unary':
+            emptyUnary($stub);
+            break;
+        case 'large_unary':
+            largeUnary($stub);
+            break;
+        case 'client_streaming':
+            clientStreaming($stub);
+            break;
+        case 'server_streaming':
+            serverStreaming($stub);
+            break;
+        case 'ping_pong':
+            pingPong($stub);
+            break;
+        case 'empty_stream':
+            emptyStream($stub);
+            break;
+        case 'cancel_after_begin':
+            cancelAfterBegin($stub);
+            break;
+        case 'cancel_after_first_response':
+            cancelAfterFirstResponse($stub);
+            break;
+        case 'timeout_on_sleeping_server':
+            timeoutOnSleepingServer($stub);
+            break;
+        case 'custom_metadata':
+            customMetadata($stub);
+            break;
+        case 'status_code_and_message':
+            statusCodeAndMessage($stub);
+            break;
+        case 'unimplemented_method':
+            unimplementedMethod($stub);
+            break;
+        case 'service_account_creds':
+            serviceAccountCreds($stub, $args);
+            break;
+        case 'compute_engine_creds':
+            computeEngineCreds($stub, $args);
+            break;
+        case 'jwt_token_creds':
+            jwtTokenCreds($stub, $args);
+            break;
+        case 'oauth2_auth_token':
+            oauth2AuthToken($stub, $args);
+            break;
+        case 'per_rpc_creds':
+            perRpcCreds($stub, $args);
+            break;
+        default:
+            echo "Unsupported test case $test_case\n";
+            exit(1);
+    }
+
+    return $stub;
 }
 
-$stub = new grpc\testing\TestServiceClient($server_address, $opts);
-
-echo "Connecting to $server_address\n";
-echo "Running test case $test_case\n";
-
-switch ($test_case) {
-    case 'empty_unary':
-        emptyUnary($stub);
-        break;
-    case 'large_unary':
-        largeUnary($stub);
-        break;
-    case 'client_streaming':
-        clientStreaming($stub);
-        break;
-    case 'server_streaming':
-        serverStreaming($stub);
-        break;
-    case 'ping_pong':
-        pingPong($stub);
-        break;
-    case 'empty_stream':
-        emptyStream($stub);
-        break;
-    case 'cancel_after_begin':
-        cancelAfterBegin($stub);
-        break;
-    case 'cancel_after_first_response':
-        cancelAfterFirstResponse($stub);
-        break;
-    case 'timeout_on_sleeping_server':
-        timeoutOnSleepingServer($stub);
-        break;
-    case 'service_account_creds':
-        serviceAccountCreds($stub, $args);
-        break;
-    case 'compute_engine_creds':
-        computeEngineCreds($stub, $args);
-        break;
-    case 'jwt_token_creds':
-        jwtTokenCreds($stub, $args);
-        break;
-    case 'oauth2_auth_token':
-        oauth2AuthToken($stub, $args);
-        break;
-    case 'per_rpc_creds':
-        perRpcCreds($stub, $args);
-        break;
-    default:
-        echo "Unsupported test case $test_case\n";
-        exit(1);
+if (isset($_SERVER['PHP_SELF']) && preg_match('/interop_client/', $_SERVER['PHP_SELF'])) {
+    $args = getopt('', ['server_host:', 'server_port:', 'test_case:',
+                        'use_tls::', 'use_test_ca::',
+                        'server_host_override:', 'oauth_scope:',
+                        'default_service_account:', ]);
+    interop_main($args);
 }
diff --git a/src/php/tests/interop/messages.proto b/src/php/tests/interop/messages.proto
index de0b1a2320..44e3c3b8f9 100644
--- a/src/php/tests/interop/messages.proto
+++ b/src/php/tests/interop/messages.proto
@@ -1,5 +1,5 @@
 
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -41,9 +41,6 @@ enum PayloadType {
 
   // Uncompressable binary format.
   UNCOMPRESSABLE = 1;
-
-  // Randomly chosen from all other formats defined in this enum.
-  RANDOM = 2;
 }
 
 // A block of data, to simply increase gRPC message size.
@@ -54,6 +51,13 @@ message Payload {
   optional bytes body = 2;
 }
 
+// A protobuf representation for grpc status. This is used by test
+// clients to specify a status that the server should attempt to return.
+message EchoStatus {
+  optional int32 code = 1;
+  optional string message = 2;
+}
+
 // Unary request.
 message SimpleRequest {
   // Desired payload type in the response from the server.
@@ -72,6 +76,12 @@ message SimpleRequest {
 
   // Whether SimpleResponse should include OAuth scope.
   optional bool fill_oauth_scope = 5;
+
+  // Whether to request the server to compress the response.
+  optional bool request_compressed_response = 6;
+
+  // Whether server should return a given status
+  optional EchoStatus response_status = 7;
 }
 
 // Unary response, as configured by the request.
@@ -123,6 +133,12 @@ message StreamingOutputCallRequest {
 
   // Optional input payload sent along with the request.
   optional Payload payload = 3;
+
+  // Whether to request the server to compress the response.
+  optional bool request_compressed_response = 6;
+
+  // Whether server should return a given status
+  optional EchoStatus response_status = 7;
 }
 
 // Server-streaming response, as configured by the request and parameters.
@@ -130,3 +146,17 @@ message StreamingOutputCallResponse {
   // Payload to increase response size.
   optional Payload payload = 1;
 }
+
+// For reconnect interop test only.
+// Client tells server what reconnection parameters it used.
+message ReconnectParams {
+  optional int32 max_reconnect_backoff_ms = 1;
+}
+
+// For reconnect interop test only.
+// Server tells client whether its reconnects are following the spec and the
+// reconnect backoffs it saw.
+message ReconnectInfo {
+  optional bool passed = 1;
+  repeated int32 backoff_ms = 2;
+}
diff --git a/src/php/tests/interop/metrics_client.php b/src/php/tests/interop/metrics_client.php
new file mode 100644
index 0000000000..19510dc5d8
--- /dev/null
+++ b/src/php/tests/interop/metrics_client.php
@@ -0,0 +1,49 @@
+<?php
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+$args = getopt('', ['metrics_server_address:', 'total_only::']);
+$parts = explode(':', $args['metrics_server_address']);
+$server_host = $parts[0];
+$server_port = (count($parts) == 2) ? $parts[1] : '';
+
+$socket = socket_create(AF_INET, SOCK_STREAM, 0);
+if (@!socket_connect($socket, $server_host, $server_port)) {
+    echo "Cannot connect to merics server...\n";
+    exit(1);
+}
+socket_write($socket, 'qps');
+while ($out = socket_read($socket, 1024)) {
+    echo "$out\n";
+}
+socket_close($socket);
diff --git a/src/php/tests/interop/stress_client.php b/src/php/tests/interop/stress_client.php
new file mode 100644
index 0000000000..f9cfe8aba5
--- /dev/null
+++ b/src/php/tests/interop/stress_client.php
@@ -0,0 +1,119 @@
+<?php
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+include_once 'interop_client.php';
+
+function stress_main($args)
+{
+    mt_srand();
+    set_time_limit(0);
+
+    // open socket to listen as metrics server
+    $socket = socket_create(AF_INET, SOCK_STREAM, 0);
+    socket_set_option($socket, SOL_SOCKET, SO_REUSEADDR, 1);
+    if (@!socket_bind($socket, 'localhost', $args['metrics_port'])) {
+        echo "Cannot create socket for metrics server...\n";
+        exit(1);
+    }
+    socket_listen($socket);
+    socket_set_nonblock($socket);
+
+    $start_time = microtime(true);
+    $count = 0;
+    $deadline = $args['test_duration_secs'] ?
+                ($start_time + $args['test_duration_secs']) : false;
+    $num_test_cases = count($args['test_cases']);
+    $stub = false;
+
+    while (true) {
+        $current_time = microtime(true);
+        if ($deadline && $current_time > $deadline) {
+            break;
+        }
+        if ($client_connection = socket_accept($socket)) {
+            // there is an incoming request, respond with qps metrics
+            $input = socket_read($client_connection, 1024);
+            $qps = round($count / ($current_time - $start_time));
+            socket_write($client_connection, "qps: $qps");
+            socket_close($client_connection);
+        } else {
+            // do actual work, run one interop test case
+            $args['test_case'] =
+                $args['test_cases'][mt_rand(0, $num_test_cases - 1)];
+            $stub = @interop_main($args, $stub);
+            ++$count;
+        }
+    }
+    socket_close($socket);
+    echo "Number of interop tests run in $args[test_duration_secs] ".
+        "seconds: $count.\n";
+}
+
+// process command line arguments
+$raw_args = getopt('',
+  ['server_addresses::',
+   'test_cases:',
+   'metrics_port::',
+   'test_duration_secs::',
+   'num_channels_per_server::',
+   'num_stubs_per_channel::',
+  ]);
+
+$args = [];
+
+if (empty($raw_args['server_addresses'])) {
+    $args['server_host'] = 'localhost';
+    $args['server_port'] = '8080';
+} else {
+    $parts = explode(':', $raw_args['server_addresses']);
+    $args['server_host'] = $parts[0];
+    $args['server_port'] = (count($parts) == 2) ? $parts[1] : '';
+}
+
+$args['metrics_port'] = empty($raw_args['metrics_port']) ?
+    '8081' : $raw_args['metrics_port'];
+
+$args['test_duration_secs'] = empty($raw_args['test_duration_secs']) ||
+    $raw_args['test_duration_secs'] == -1 ?
+    false : $raw_args['test_duration_secs'];
+
+$test_cases = [];
+$test_case_strs = explode(',', $raw_args['test_cases']);
+foreach ($test_case_strs as $test_case_str) {
+    $parts = explode(':', $test_case_str);
+    $test_cases = array_merge($test_cases, array_fill(0, $parts[1], $parts[0]));
+}
+$args['test_cases'] = $test_cases;
+
+stress_main($args);
diff --git a/src/php/tests/interop/test.proto b/src/php/tests/interop/test.proto
index 0d169e7f64..57ef30ee1c 100644
--- a/src/php/tests/interop/test.proto
+++ b/src/php/tests/interop/test.proto
@@ -1,5 +1,5 @@
 
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -69,3 +69,10 @@ service TestService {
   rpc HalfDuplexCall(stream StreamingOutputCallRequest)
       returns (stream StreamingOutputCallResponse);
 }
+
+// A simple service NOT implemented at servers so clients can test for
+// that case.
+service UnimplementedService {
+  // A call that no server should implement
+  rpc UnimplementedCall(grpc.testing.EmptyMessage) returns (grpc.testing.EmptyMessage);
+}
diff --git a/src/proto/census/census.options b/src/proto/census/census.options
new file mode 100644
index 0000000000..a1f80395c7
--- /dev/null
+++ b/src/proto/census/census.options
@@ -0,0 +1,3 @@
+google.census.Tag.key max_size:255
+google.census.Tag.value max_size:255
+google.census.View.tag_key max_count:15
diff --git a/src/proto/census/census.proto b/src/proto/census/census.proto
new file mode 100644
index 0000000000..c869d851ff
--- /dev/null
+++ b/src/proto/census/census.proto
@@ -0,0 +1,313 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package google.census;
+
+// All the census protos.
+//
+// Nomenclature note: capitalized names below (like Metric) are protos.
+//
+// Census lets you define a Metric - something which can be measured, like the
+// latency of an RPC, the number of CPU cycles spent on an operation, or
+// anything else you care to measure. You can record individual instances of
+// measurements (a double value) for every metric of interest. These
+// individual measurements are aggregated together into an Aggregation. There
+// are two Aggregation types available: Distribution (describes the
+// distribution of all measurements, possibly with a histogram) and
+// IntervalStats (the count and mean of measurements across specified time
+// periods). An Aggregation is described by an AggregationDescriptor.
+//
+// You can define how your stats are broken down by Tag values and which
+// Aggregations to use through a View. The corresponding combination of
+// Metric/View/Aggregation which is available to census clients is called a
+// ViewAggregation.
+
+
+// The following two types are copied from
+// google/protobuf/{duration,timestamp}.proto. Ideally, we would be able to
+// import them, but this causes compilation issues on C-based systems
+// (e.g. https://koti.kapsi.fi/jpa/nanopb/), which cannot process the C++
+// headers generated from the standard protobuf distribution. See the relevant
+// proto files for full documentation of these types.
+
+message Duration {
+  // Signed seconds of the span of time. Must be from -315,576,000,000
+  // to +315,576,000,000 inclusive.
+  int64 seconds = 1;
+
+  // Signed fractions of a second at nanosecond resolution of the span
+  // of time. Durations less than one second are represented with a 0
+  // `seconds` field and a positive or negative `nanos` field. For durations
+  // of one second or more, a non-zero value for the `nanos` field must be
+  // of the same sign as the `seconds` field. Must be from -999,999,999
+  // to +999,999,999 inclusive.
+  int32 nanos = 2;
+}
+
+message Timestamp {
+  // Represents seconds of UTC time since Unix epoch
+  // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to
+  // 9999-12-31T23:59:59Z inclusive.
+  int64 seconds = 1;
+
+  // Non-negative fractions of a second at nanosecond resolution. Negative
+  // second values with fractions must still have non-negative nanos values
+  // that count forward in time. Must be from 0 to 999,999,999
+  // inclusive.
+  int32 nanos = 2;
+}
+
+// Describes a metric
+message Metric {
+  // name of metric, e.g. rpc_latency, cpu.
+  string name = 1;
+
+  // More detailed description of the metric, used in documentation.
+  string description = 2;
+
+  // Fundamental units of measurement supported by Census
+  // TODO(aveitch): expand this to include other S.I. units?
+  message BasicUnit {
+    enum Measure {
+      UNKNOWN = 0;
+      BITS = 1;
+      BYTES = 2;
+      SECS = 3;
+      CORES = 4;
+      MAX_UNITS = 5;
+    }
+    Measure type = 1;
+  }
+
+  // MeasurementUnit lets you build compound units of the form
+  //   10^n * (A * B * ...) / (X * Y * ...),
+  // where the elements in the numerator and denominator are all BasicUnits.  A
+  // MeasurementUnit must have at least one BasicUnit in its numerator.
+  //
+  // To specify multiplication in the numerator or denominator, simply specify
+  // multiple numerator or denominator fields.  For example:
+  //
+  // - byte-seconds (i.e. bytes * seconds):
+  //     numerator: BYTES
+  //     numerator: SECS
+  //
+  // - events/sec^2 (i.e. rate of change of events/sec):
+  //     numerator: COUNT
+  //     denominator: SECS
+  //     denominator: SECS
+  //
+  // To specify multiples (in power of 10) units, specify a non-zero prefix
+  // value, for example:
+  //
+  // - MB/s (i.e. megabytes / s):
+  //     prefix: 6
+  //     numerator: BYTES
+  //     denominator: SECS
+  //
+  // - nanoseconds
+  //     prefix: -9
+  //     numerator: SECS
+  message MeasurementUnit {
+    int32 prefix = 1;
+    repeated BasicUnit numerator = 2;
+    repeated BasicUnit denominator = 3;
+  }
+
+  // The units in which the Metric value is reported.
+  MeasurementUnit unit = 3;
+
+  // Metrics will be assigned an ID when registered. Invalid if <= 0.
+  int32 id = 4;
+}
+
+// An Aggregation summarizes a series of individual Metric measurements, an
+// AggregationDescriptor describes an Aggregation.
+message AggregationDescriptor {
+  // At most one set of options. If neither option is set, a default type
+  // of Distribution (without a histogram component) will be used.
+  oneof options {
+    // Defines the histogram bucket boundaries for Distributions.
+    BucketBoundaries bucket_boundaries = 1;
+    // Defines the time windows to record for IntervalStats.
+    IntervalBoundaries interval_boundaries = 2;
+  }
+
+  // A Distribution may optionally contain a histogram of the values in the
+  // population. The bucket boundaries for that histogram is described by
+  // `bucket_boundaries`.
+  //
+  // Describes histogram bucket boundaries. Defines `size(bounds) + 1` (= N)
+  // buckets (for size(bounds) >= 1; if size(bounds) == 0, then no histogram
+  // will be defined. The boundaries for bucket index i are:
+  //
+  // [-infinity, bounds[i]) for i == 0
+  // [bounds[i-1], bounds[i]) for 0 < i < N-2
+  // [bounds[i-1], +infinity) for i == N-1
+  //
+  // i.e. an underflow bucket (number 0), zero or more finite buckets (1
+  // through N - 2, and an overflow bucket (N - 1), with inclusive lower
+  // bounds and exclusive upper bounds.
+  //
+  // There must be at least one element in `bounds`.  If `bounds` has only one
+  // element, there are no finite buckets, and that single element is the
+  // common boundary of the overflow and underflow buckets.
+  message BucketBoundaries {
+    // The values must be monotonically increasing.
+    repeated double bounds = 1;
+  }
+
+  // For Interval stats, describe the size of each window.
+  message IntervalBoundaries {
+    // For each time window, specify a duration in seconds.
+    repeated double window_size = 1;
+  }
+}
+
+// Distribution contains summary statistics for a population of values and,
+// optionally, a histogram representing the distribution of those values across
+// a specified set of histogram buckets, as defined in
+// Aggregation.bucket_options.
+//
+// The summary statistics are the count, mean, sum of the squared deviation from
+// the mean, the minimum, and the maximum of the set of population of values.
+//
+// Although it is not forbidden, it is generally a bad idea to include
+// non-finite values (infinities or NaNs) in the population of values, as this
+// will render the `mean` field meaningless.
+message Distribution {
+  // The number of values in the population. Must be non-negative.
+  int64 count = 1;
+
+  // The arithmetic mean of the values in the population. If `count` is zero
+  // then this field must be zero.
+  double mean = 2;
+
+  // Describes a range of population values.
+  message Range {
+    // The minimum of the population values.
+    double min = 1;
+    // The maximum of the population values.
+    double max = 2;
+  }
+
+  // The range of the population values. If `count` is zero, this field will not
+  // be defined.
+  Range range = 3;
+
+  // A Distribution may optionally contain a histogram of the values in the
+  // population.  The histogram is given in `bucket_count` as counts of values
+  // that fall into one of a sequence of non-overlapping buckets, as described
+  // by `AggregationDescriptor.options.bucket_boundaries`.
+  // The sum of the values in `bucket_counts` must equal the value in `count`.
+  //
+  // Bucket counts are given in order under the numbering scheme described
+  // above (the underflow bucket has number 0; the finite buckets, if any,
+  // have numbers 1 through N-2; the overflow bucket has number N-1).
+  //
+  // The size of `bucket_count` must be no greater than N as defined in
+  // `bucket_boundaries`.
+  //
+  // Any suffix of trailing zero bucket_count fields may be omitted.
+  repeated int64 bucket_count = 4;
+}
+
+// Record summary stats over various time windows.
+message IntervalStats {
+  // Summary statistic over a single time window.
+  message Window {
+    // The window duration.
+    Duration window_size = 1;
+    // The number of measurements in this window.
+    int64 count = 2;
+    // The arithmetic mean of all measurements in the window.
+    double mean = 3;
+  }
+
+  // Full set of windows for this metric.
+  repeated Window window = 1;
+}
+
+// A Tag: key-value pair.
+message Tag {
+  string key = 1;
+  string value = 2;
+}
+
+// A View specifies an Aggregation and a set of tag keys. The Aggregation will
+// be broken down by the unique set of matching tag values for each measurement.
+message View {
+  // Name of view.
+  string name = 1;
+
+  // More detailed description, for documentation purposes.
+  string description = 2;
+
+  // ID of Metric to associate with this View.
+  int32 metric_id = 3;
+
+  // Aggregation type to associate with this View.
+  AggregationDescriptor aggregation = 4;
+
+  // Tag keys to match with a given Metric. If no keys are specified, then all
+  // stats for the Metric are recorded. Keys must be unique.
+  repeated string tag_key = 5;
+}
+
+// An Aggregation summarizes a series of individual Metric measures.
+message Aggregation {
+  // Name of this aggregation.
+  string name = 1;
+
+  // More detailed description, for documentation purposes.
+  string description = 2;
+
+  // The data for this Aggregation.
+  oneof data {
+    Distribution distribution = 3;
+    IntervalStats interval_stats = 4;
+  }
+
+  // Tags associated with this Aggregation.
+  repeated Tag tag = 5;
+}
+
+// A ViewAggregations represents all the Aggregations for a particular view.
+message ViewAggregations {
+  // Aggregations - each will have a unique set of tag values for the tag_keys
+  // associated with the corresponding View.
+  repeated Aggregation aggregation = 1;
+
+  // Start and end timestamps over which the value was accumulated. These
+  // values are not relevant/defined for IntervalStats aggregations, which are
+  // always accumulated over a fixed time period.
+  Timestamp start = 2;
+  Timestamp end = 3;
+}
diff --git a/src/proto/grpc/binary_log/v1alpha/log.proto b/src/proto/grpc/binary_log/v1alpha/log.proto
index 83166cd410..46656109bc 100644
--- a/src/proto/grpc/binary_log/v1alpha/log.proto
+++ b/src/proto/grpc/binary_log/v1alpha/log.proto
@@ -29,20 +29,20 @@
 
 syntax = "proto3";
 
-import "google/protobuf/timestamp.proto"
+import "google/protobuf/timestamp.proto";
 
 package grpc.binary_log.v1alpha;
 
 enum Direction {
-  SERVER_SEND;
-  SERVER_RECV;
-  CLIENT_SEND;
-  CLIENT_RECV;
+  SERVER_SEND = 0;
+  SERVER_RECV = 1;
+  CLIENT_SEND = 2;
+  CLIENT_RECV = 3;
 }
 
 message KeyValuePair {
-  string key;
-  string value;
+  string key = 1;
+  string value = 2;
 }
 
 // Any sort of metadata that may be sent in either direction during a call
diff --git a/src/proto/grpc/lb/v0/load_balancer.options b/src/proto/grpc/lb/v0/load_balancer.options
deleted file mode 100644
index 6d4528f838..0000000000
--- a/src/proto/grpc/lb/v0/load_balancer.options
+++ /dev/null
@@ -1,6 +0,0 @@
-grpc.lb.v0.InitialLoadBalanceRequest.name max_size:128
-grpc.lb.v0.InitialLoadBalanceResponse.client_config max_size:64
-grpc.lb.v0.InitialLoadBalanceResponse.load_balancer_delegate max_size:64
-grpc.lb.v0.Server.ip_address max_size:46
-grpc.lb.v0.Server.load_balance_token max_size:64
-load_balancer.proto no_unions:true
diff --git a/src/proto/grpc/lb/v1/load_balancer.options b/src/proto/grpc/lb/v1/load_balancer.options
new file mode 100644
index 0000000000..d90366996e
--- /dev/null
+++ b/src/proto/grpc/lb/v1/load_balancer.options
@@ -0,0 +1,6 @@
+grpc.lb.v1.InitialLoadBalanceRequest.name max_size:128
+grpc.lb.v1.InitialLoadBalanceResponse.client_config max_size:64
+grpc.lb.v1.InitialLoadBalanceResponse.load_balancer_delegate max_size:64
+grpc.lb.v1.Server.ip_address max_size:46
+grpc.lb.v1.Server.load_balance_token max_size:64
+load_balancer.proto no_unions:true
diff --git a/src/proto/grpc/lb/v0/load_balancer.proto b/src/proto/grpc/lb/v1/load_balancer.proto
index e88a4f8c4a..1bcad0b1d4 100644
--- a/src/proto/grpc/lb/v0/load_balancer.proto
+++ b/src/proto/grpc/lb/v1/load_balancer.proto
@@ -29,7 +29,7 @@
 
 syntax = "proto3";
 
-package grpc.lb.v0;
+package grpc.lb.v1;
 
 message Duration {
 
@@ -94,9 +94,8 @@ message LoadBalanceResponse {
 
 message InitialLoadBalanceResponse {
   oneof initial_response_type {
-    // Contains gRPC config options like RPC deadline or flow control.
-    // TODO(yetianx): Change to ClientConfig after it is defined.
-    string client_config = 1;
+    // TODO(zhangkun83): ClientConfig not yet defined
+    //ClientConfig client_config = 1;
 
     // This is an application layer redirect that indicates the client should
     // use the specified server for load balancing. When this field is set in
@@ -134,9 +133,7 @@ message Server {
   // An opaque token that is passed from the client to the server in metadata.
   // The server may expect this token to indicate that the request from the
   // client was load balanced.
-  // TODO(yetianx): Not used right now, and will be used after implementing
-  // load report.
-  bytes load_balance_token = 3;
+  string load_balance_token = 3;
 
   // Indicates whether this particular request should be dropped by the client
   // when this server is chosen from the list.
diff --git a/src/proto/grpc/reflection/v1alpha/reflection.proto b/src/proto/grpc/reflection/v1alpha/reflection.proto
new file mode 100644
index 0000000000..276ff0e255
--- /dev/null
+++ b/src/proto/grpc/reflection/v1alpha/reflection.proto
@@ -0,0 +1,151 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Service exported by server reflection
+
+syntax = "proto3";
+
+package grpc.reflection.v1alpha;
+
+service ServerReflection {
+  // The reflection service is structured as a bidirectional stream, ensuring
+  // all related requests go to a single server.
+  rpc ServerReflectionInfo(stream ServerReflectionRequest)
+      returns (stream ServerReflectionResponse);
+}
+
+// The message sent by the client when calling ServerReflectionInfo method.
+message ServerReflectionRequest {
+  string host = 1;
+  // To use reflection service, the client should set one of the following
+  // fields in message_request. The server distinguishes requests by their
+  // defined field and then handles them using corresponding methods.
+  oneof message_request {
+    // Find a proto file by the file name.
+    string file_by_filename = 3;
+
+    // Find the proto file that declares the given fully-qualified symbol name.
+    // This field should be a fully-qualified symbol name
+    // (e.g. <package>.<service>[.<method>] or <package>.<type>).
+    string file_containing_symbol = 4;
+
+    // Find the proto file which defines an extension extending the given
+    // message type with the given field number.
+    ExtensionRequest file_containing_extension = 5;
+
+    // Finds the tag numbers used by all known extensions of extendee_type, and
+    // appends them to ExtensionNumberResponse in an undefined order.
+    // Its corresponding method is best-effort: it's not guaranteed that the
+    // reflection service will implement this method, and it's not guaranteed
+    // that this method will provide all extensions. Returns
+    // StatusCode::UNIMPLEMENTED if it's not implemented.
+    // This field should be a fully-qualified type name. The format is
+    // <package>.<type>
+    string all_extension_numbers_of_type = 6;
+
+    // List the full names of registered services. The content will not be
+    // checked.
+    string list_services = 7;
+  }
+}
+
+// The type name and extension number sent by the client when requesting
+// file_containing_extension.
+message ExtensionRequest {
+  // Fully-qualified type name. The format should be <package>.<type>
+  string containing_type = 1;
+  int32 extension_number = 2;
+}
+
+// The message sent by the server to answer ServerReflectionInfo method.
+message ServerReflectionResponse {
+  string valid_host = 1;
+  ServerReflectionRequest original_request = 2;
+  // The server set one of the following fields accroding to the message_request
+  // in the request.
+  oneof message_response {
+    // This message is used to answer file_by_filename, file_containing_symbol,
+    // file_containing_extension requests with transitive dependencies. As
+    // the repeated label is not allowed in oneof fields, we use a
+    // FileDescriptorResponse message to encapsulate the repeated fields.
+    // The reflection service is allowed to avoid sending FileDescriptorProtos
+    // that were previously sent in response to earlier requests in the stream.
+    FileDescriptorResponse file_descriptor_response = 4;
+
+    // This message is used to answer all_extension_numbers_of_type requst.
+    ExtensionNumberResponse all_extension_numbers_response = 5;
+
+    // This message is used to answer list_services request.
+    ListServiceResponse list_services_response = 6;
+
+    // This message is used when an error occurs.
+    ErrorResponse error_response = 7;
+  }
+}
+
+// Serialized FileDescriptorProto messages sent by the server answering
+// a file_by_filename, file_containing_symbol, or file_containing_extension
+// request.
+message FileDescriptorResponse {
+  // Serialized FileDescriptorProto messages. We avoid taking a dependency on
+  // descriptor.proto, which uses proto2 only features, by making them opaque
+  // bytes instead.
+  repeated bytes file_descriptor_proto = 1;
+}
+
+// A list of extension numbers sent by the server answering
+// all_extension_numbers_of_type request.
+message ExtensionNumberResponse {
+  // Full name of the base type, including the package name. The format
+  // is <package>.<type>
+  string base_type_name = 1;
+  repeated int32 extension_number = 2;
+}
+
+// A list of ServiceResponse sent by the server answering list_services request.
+message ListServiceResponse {
+  // The information of each service may be expanded in the future, so we use
+  // ServiceResponse message to encapsulate it.
+  repeated ServiceResponse service = 1;
+}
+
+// The information of a single service used by ListServiceResponse to answer
+// list_services request.
+message ServiceResponse {
+  // Full name of a registered service, including its package name. The format
+  // is <package>.<service>
+  string name = 1;
+}
+
+// The error code and error message sent by the server when an error occurs.
+message ErrorResponse {
+  // This field uses the error codes defined in grpc::StatusCode.
+  int32 error_code = 1;
+  string error_message = 2;
+}
diff --git a/src/proto/grpc/testing/echo_messages.proto b/src/proto/grpc/testing/echo_messages.proto
index 1be1966f10..b405acf043 100644
--- a/src/proto/grpc/testing/echo_messages.proto
+++ b/src/proto/grpc/testing/echo_messages.proto
@@ -32,6 +32,12 @@ syntax = "proto3";
 
 package grpc.testing;
 
+// Message to be echoed back serialized in trailer.
+message DebugInfo {
+  repeated string stack_entries = 1;
+  string detail = 2;
+}
+
 message RequestParams {
   bool echo_deadline = 1;
   int32 client_cancel_after_us = 2;
@@ -43,6 +49,7 @@ message RequestParams {
   string expected_client_identity = 8; // will force check_auth_context.
   bool skip_cancelled_check = 9;
   string expected_transport_security_type = 10;
+  DebugInfo debug_info = 11;
 }
 
 message EchoRequest {
diff --git a/src/proto/grpc/testing/messages.proto b/src/proto/grpc/testing/messages.proto
index a063b470c7..64998c2f23 100644
--- a/src/proto/grpc/testing/messages.proto
+++ b/src/proto/grpc/testing/messages.proto
@@ -34,28 +34,24 @@ syntax = "proto3";
 
 package grpc.testing;
 
+// TODO(dgq): Go back to using well-known types once
+// https://github.com/grpc/grpc/issues/6980 has been fixed.
+// import "google/protobuf/wrappers.proto";
+message BoolValue {
+  // The bool value.
+  bool value = 1;
+}
+
+// DEPRECATED, don't use. To be removed shortly.
 // The type of payload that should be returned.
 enum PayloadType {
   // Compressable text format.
   COMPRESSABLE = 0;
-
-  // Uncompressable binary format.
-  UNCOMPRESSABLE = 1;
-
-  // Randomly chosen from all other formats defined in this enum.
-  RANDOM = 2;
-}
-
-// Compression algorithms
-enum CompressionType {
-  // No compression
-  NONE = 0;
-  GZIP = 1;
-  DEFLATE = 2;
 }
 
 // A block of data, to simply increase gRPC message size.
 message Payload {
+  // DEPRECATED, don't use. To be removed shortly.
   // The type of data in body.
   PayloadType type = 1;
   // Primary contents of payload.
@@ -71,12 +67,12 @@ message EchoStatus {
 
 // Unary request.
 message SimpleRequest {
+  // DEPRECATED, don't use. To be removed shortly.
   // Desired payload type in the response from the server.
   // If response_type is RANDOM, server randomly chooses one from other formats.
   PayloadType response_type = 1;
 
   // Desired payload size in the response from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
   int32 response_size = 2;
 
   // Optional input payload sent along with the request.
@@ -88,11 +84,17 @@ message SimpleRequest {
   // Whether SimpleResponse should include OAuth scope.
   bool fill_oauth_scope = 5;
 
-  // Compression algorithm to be used by the server for the response (stream)
-  CompressionType response_compression = 6;
+  // Whether to request the server to compress the response. This field is
+  // "nullable" in order to interoperate seamlessly with clients not able to
+  // implement the full compression tests by introspecting the call to verify
+  // the response's compression status.
+  BoolValue response_compressed = 6;
 
   // Whether server should return a given status
   EchoStatus response_status = 7;
+
+  // Whether the server should expect this request to be compressed.
+  BoolValue expect_compressed = 8;
 }
 
 // Unary response, as configured by the request.
@@ -111,6 +113,12 @@ message StreamingInputCallRequest {
   // Optional input payload sent along with the request.
   Payload payload = 1;
 
+  // Whether the server should expect this request to be compressed. This field
+  // is "nullable" in order to interoperate seamlessly with servers not able to
+  // implement the full compression tests by introspecting the call to verify
+  // the request's compression status.
+  BoolValue expect_compressed = 2;
+
   // Not expecting any payload from the response.
 }
 
@@ -123,16 +131,22 @@ message StreamingInputCallResponse {
 // Configuration for a particular response.
 message ResponseParameters {
   // Desired payload sizes in responses from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
   int32 size = 1;
 
   // Desired interval between consecutive responses in the response stream in
   // microseconds.
   int32 interval_us = 2;
+
+  // Whether to request the server to compress the response. This field is
+  // "nullable" in order to interoperate seamlessly with clients not able to
+  // implement the full compression tests by introspecting the call to verify
+  // the response's compression status.
+  BoolValue compressed = 3;
 }
 
 // Server-streaming request.
 message StreamingOutputCallRequest {
+  // DEPRECATED, don't use. To be removed shortly.
   // Desired payload type in the response from the server.
   // If response_type is RANDOM, the payload from each response in the stream
   // might be of different types. This is to simulate a mixed type of payload
@@ -145,9 +159,6 @@ message StreamingOutputCallRequest {
   // Optional input payload sent along with the request.
   Payload payload = 3;
 
-  // Compression algorithm to be used by the server for the response (stream)
-  CompressionType response_compression = 6;
-
   // Whether server should return a given status
   EchoStatus response_status = 7;
 }
diff --git a/src/proto/math/math.proto b/src/proto/math/math.proto
index 311e148c02..269c60bde8 100644
--- a/src/proto/math/math.proto
+++ b/src/proto/math/math.proto
@@ -55,8 +55,8 @@ message FibReply {
 }
 
 service Math {
-  // Div divides args.dividend by args.divisor and returns the quotient and
-  // remainder.
+  // Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+  // and remainder.
   rpc Div (DivArgs) returns (DivReply) {
   }
 
@@ -67,7 +67,7 @@ service Math {
   rpc DivMany (stream DivArgs) returns (stream DivReply) {
   }
 
-  // Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+  // Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
   // generates up to limit numbers; otherwise it continues until the call is
   // canceled.  Unlike Fib above, Fib has no final FibReply.
   rpc Fib (FibArgs) returns (stream Num) {
diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst
index cb3f6b87fe..3fc318539e 100644
--- a/src/python/grpcio/README.rst
+++ b/src/python/grpcio/README.rst
@@ -46,8 +46,9 @@ package named :code:`python-dev`).
 ::
 
   $ export REPO_ROOT=grpc  # REPO_ROOT can be any directory of your choice
-  $ git clone https://github.com/grpc/grpc.git $REPO_ROOT
+  $ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc $REPO_ROOT
   $ cd $REPO_ROOT
+  $ git submodule update --init
 
   # For the next two commands do `sudo pip install` if you get permission-denied errors
   $ pip install -rrequirements.txt
diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py
index 295dab2d27..f498ed4190 100644
--- a/src/python/grpcio/commands.py
+++ b/src/python/grpcio/commands.py
@@ -182,6 +182,8 @@ class BuildProtoModules(setuptools.Command):
           '--plugin=protoc-gen-python-grpc={}'.format(
               self.grpc_python_plugin_command),
           '-I {}'.format(GRPC_STEM),
+          '-I .',
+          '-I {}/third_party/protobuf/src'.format(GRPC_STEM),
           '--python_out={}'.format(PROTO_GEN_STEM),
           '--python-grpc_out={}'.format(PROTO_GEN_STEM),
       ] + [path]
@@ -191,7 +193,7 @@ class BuildProtoModules(setuptools.Command):
       except subprocess.CalledProcessError as e:
         sys.stderr.write(
             'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
-                command, e.message, e.output))
+                command, str(e), e.output))
 
     # Generated proto directories dont include __init__.py, but
     # these are needed for python package resolution
diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py
index b844a14c48..b3eeaad1f7 100644
--- a/src/python/grpcio/grpc/__init__.py
+++ b/src/python/grpcio/grpc/__init__.py
@@ -27,5 +27,1248 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-__import__('pkg_resources').declare_namespace(__name__)
+"""gRPC's Python API."""
 
+import abc
+import enum
+
+import six
+
+from grpc._cython import cygrpc as _cygrpc
+
+
+############################## Future Interface  ###############################
+
+
+class FutureTimeoutError(Exception):
+  """Indicates that a method call on a Future timed out."""
+
+
+class FutureCancelledError(Exception):
+  """Indicates that the computation underlying a Future was cancelled."""
+
+
+class Future(six.with_metaclass(abc.ABCMeta)):
+  """A representation of a computation in another control flow.
+
+  Computations represented by a Future may be yet to be begun, may be ongoing,
+  or may have already completed.
+  """
+
+  @abc.abstractmethod
+  def cancel(self):
+    """Attempts to cancel the computation.
+
+    This method does not block.
+
+    Returns:
+      True if the computation has not yet begun, will not be allowed to take
+        place, and determination of both was possible without blocking. False
+        under all other circumstances including but not limited to the
+        computation's already having begun, the computation's already having
+        finished, and the computation's having been scheduled for execution on a
+        remote system for which a determination of whether or not it commenced
+        before being cancelled cannot be made without blocking.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def cancelled(self):
+    """Describes whether the computation was cancelled.
+
+    This method does not block.
+
+    Returns:
+      True if the computation was cancelled any time before its result became
+        immediately available. False under all other circumstances including but
+        not limited to this object's cancel method not having been called and
+        the computation's result having become immediately available.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def running(self):
+    """Describes whether the computation is taking place.
+
+    This method does not block.
+
+    Returns:
+      True if the computation is scheduled to take place in the future or is
+        taking place now, or False if the computation took place in the past or
+        was cancelled.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def done(self):
+    """Describes whether the computation has taken place.
+
+    This method does not block.
+
+    Returns:
+      True if the computation is known to have either completed or have been
+        unscheduled or interrupted. False if the computation may possibly be
+        executing or scheduled to execute later.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def result(self, timeout=None):
+    """Accesses the outcome of the computation or raises its exception.
+
+    This method may return immediately or may block.
+
+    Args:
+      timeout: The length of time in seconds to wait for the computation to
+        finish or be cancelled, or None if this method should block until the
+        computation has finished or is cancelled no matter how long that takes.
+
+    Returns:
+      The return value of the computation.
+
+    Raises:
+      FutureTimeoutError: If a timeout value is passed and the computation does
+        not terminate within the allotted time.
+      FutureCancelledError: If the computation was cancelled.
+      Exception: If the computation raised an exception, this call will raise
+        the same exception.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def exception(self, timeout=None):
+    """Return the exception raised by the computation.
+
+    This method may return immediately or may block.
+
+    Args:
+      timeout: The length of time in seconds to wait for the computation to
+        terminate or be cancelled, or None if this method should block until
+        the computation is terminated or is cancelled no matter how long that
+        takes.
+
+    Returns:
+      The exception raised by the computation, or None if the computation did
+        not raise an exception.
+
+    Raises:
+      FutureTimeoutError: If a timeout value is passed and the computation does
+        not terminate within the allotted time.
+      FutureCancelledError: If the computation was cancelled.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def traceback(self, timeout=None):
+    """Access the traceback of the exception raised by the computation.
+
+    This method may return immediately or may block.
+
+    Args:
+      timeout: The length of time in seconds to wait for the computation to
+        terminate or be cancelled, or None if this method should block until
+        the computation is terminated or is cancelled no matter how long that
+        takes.
+
+    Returns:
+      The traceback of the exception raised by the computation, or None if the
+        computation did not raise an exception.
+
+    Raises:
+      FutureTimeoutError: If a timeout value is passed and the computation does
+        not terminate within the allotted time.
+      FutureCancelledError: If the computation was cancelled.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_done_callback(self, fn):
+    """Adds a function to be called at completion of the computation.
+
+    The callback will be passed this Future object describing the outcome of
+    the computation.
+
+    If the computation has already completed, the callback will be called
+    immediately.
+
+    Args:
+      fn: A callable taking this Future object as its single parameter.
+    """
+    raise NotImplementedError()
+
+
+################################  gRPC Enums  ##################################
+
+
+@enum.unique
+class ChannelConnectivity(enum.Enum):
+  """Mirrors grpc_connectivity_state in the gRPC Core.
+
+  Attributes:
+    IDLE: The channel is idle.
+    CONNECTING: The channel is connecting.
+    READY: The channel is ready to conduct RPCs.
+    TRANSIENT_FAILURE: The channel has seen a failure from which it expects to
+      recover.
+    SHUTDOWN: The channel has seen a failure from which it cannot recover.
+  """
+  IDLE              = (_cygrpc.ConnectivityState.idle, 'idle')
+  CONNECTING        = (_cygrpc.ConnectivityState.connecting, 'connecting')
+  READY             = (_cygrpc.ConnectivityState.ready, 'ready')
+  TRANSIENT_FAILURE = (
+      _cygrpc.ConnectivityState.transient_failure, 'transient failure')
+  SHUTDOWN          = (_cygrpc.ConnectivityState.shutdown, 'shutdown')
+
+
+@enum.unique
+class StatusCode(enum.Enum):
+  """Mirrors grpc_status_code in the gRPC Core."""
+  OK                  = (_cygrpc.StatusCode.ok, 'ok')
+  CANCELLED           = (_cygrpc.StatusCode.cancelled, 'cancelled')
+  UNKNOWN             = (_cygrpc.StatusCode.unknown, 'unknown')
+  INVALID_ARGUMENT    = (
+      _cygrpc.StatusCode.invalid_argument, 'invalid argument')
+  DEADLINE_EXCEEDED   = (
+      _cygrpc.StatusCode.deadline_exceeded, 'deadline exceeded')
+  NOT_FOUND           = (_cygrpc.StatusCode.not_found, 'not found')
+  ALREADY_EXISTS      = (_cygrpc.StatusCode.already_exists, 'already exists')
+  PERMISSION_DENIED   = (
+      _cygrpc.StatusCode.permission_denied, 'permission denied')
+  RESOURCE_EXHAUSTED  = (
+      _cygrpc.StatusCode.resource_exhausted, 'resource exhausted')
+  FAILED_PRECONDITION = (
+      _cygrpc.StatusCode.failed_precondition, 'failed precondition')
+  ABORTED             = (_cygrpc.StatusCode.aborted, 'aborted')
+  OUT_OF_RANGE        = (_cygrpc.StatusCode.out_of_range, 'out of range')
+  UNIMPLEMENTED       = (_cygrpc.StatusCode.unimplemented, 'unimplemented')
+  INTERNAL            = (_cygrpc.StatusCode.internal, 'internal')
+  UNAVAILABLE         = (_cygrpc.StatusCode.unavailable, 'unavailable')
+  DATA_LOSS           = (_cygrpc.StatusCode.data_loss, 'data loss')
+  UNAUTHENTICATED     = (_cygrpc.StatusCode.unauthenticated, 'unauthenticated')
+
+
+#############################  gRPC Exceptions  ################################
+
+
+class RpcError(Exception):
+  """Raised by the gRPC library to indicate non-OK-status RPC termination."""
+
+
+##############################  Shared Context  ################################
+
+
+class RpcContext(six.with_metaclass(abc.ABCMeta)):
+  """Provides RPC-related information and action."""
+
+  @abc.abstractmethod
+  def is_active(self):
+    """Describes whether the RPC is active or has terminated."""
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def time_remaining(self):
+    """Describes the length of allowed time remaining for the RPC.
+
+    Returns:
+      A nonnegative float indicating the length of allowed time in seconds
+      remaining for the RPC to complete before it is considered to have timed
+      out, or None if no deadline was specified for the RPC.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def cancel(self):
+    """Cancels the RPC.
+
+    Idempotent and has no effect if the RPC has already terminated.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_callback(self, callback):
+    """Registers a callback to be called on RPC termination.
+
+    Args:
+      callback: A no-parameter callable to be called on RPC termination.
+
+    Returns:
+      True if the callback was added and will be called later; False if the
+        callback was not added and will not later be called (because the RPC
+        already terminated or some other reason).
+    """
+    raise NotImplementedError()
+
+
+#########################  Invocation-Side Context  ############################
+
+
+class Call(six.with_metaclass(abc.ABCMeta, RpcContext)):
+  """Invocation-side utility object for an RPC."""
+
+  @abc.abstractmethod
+  def initial_metadata(self):
+    """Accesses the initial metadata from the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The initial metadata as a sequence of pairs of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def trailing_metadata(self):
+    """Accesses the trailing metadata from the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The trailing metadata as a sequence of pairs of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def code(self):
+    """Accesses the status code emitted by the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The StatusCode value for the RPC.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def details(self):
+    """Accesses the details value emitted by the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The bytes of the details of the RPC.
+    """
+    raise NotImplementedError()
+
+
+############  Authentication & Authorization Interfaces & Classes  #############
+
+
+class ChannelCredentials(object):
+  """A value encapsulating the data required to create a secure Channel.
+
+  This class has no supported interface - it exists to define the type of its
+  instances and its instances exist to be passed to other functions.
+  """
+
+  def __init__(self, credentials):
+    self._credentials = credentials
+
+
+class CallCredentials(object):
+  """A value encapsulating data asserting an identity over a channel.
+
+  A CallCredentials may be composed with ChannelCredentials to always assert
+  identity for every call over that Channel.
+
+  This class has no supported interface - it exists to define the type of its
+  instances and its instances exist to be passed to other functions.
+  """
+
+  def __init__(self, credentials):
+    self._credentials = credentials
+
+
+class AuthMetadataContext(six.with_metaclass(abc.ABCMeta)):
+  """Provides information to call credentials metadata plugins.
+
+  Attributes:
+    service_url: A string URL of the service being called into.
+    method_name: A string of the fully qualified method name being called.
+  """
+
+
+class AuthMetadataPluginCallback(six.with_metaclass(abc.ABCMeta)):
+  """Callback object received by a metadata plugin."""
+
+  def __call__(self, metadata, error):
+    """Inform the gRPC runtime of the metadata to construct a CallCredentials.
+
+    Args:
+      metadata: An iterable of 2-sequences (e.g. tuples) of metadata key/value
+        pairs.
+      error: An Exception to indicate error or None to indicate success.
+    """
+    raise NotImplementedError()
+
+
+class AuthMetadataPlugin(six.with_metaclass(abc.ABCMeta)):
+  """A specification for custom authentication."""
+
+  def __call__(self, context, callback):
+    """Implements authentication by passing metadata to a callback.
+
+    Implementations of this method must not block.
+
+    Args:
+      context: An AuthMetadataContext providing information on the RPC that the
+        plugin is being called to authenticate.
+      callback: An AuthMetadataPluginCallback to be invoked either synchronously
+        or asynchronously.
+    """
+    raise NotImplementedError()
+
+
+class ServerCredentials(object):
+  """A value encapsulating the data required to open a secure port on a Server.
+
+  This class has no supported interface - it exists to define the type of its
+  instances and its instances exist to be passed to other functions.
+  """
+
+  def __init__(self, credentials):
+    self._credentials = credentials
+
+
+########################  Multi-Callable Interfaces  ###########################
+
+
+class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a unary-unary RPC."""
+
+  @abc.abstractmethod
+  def __call__(self, request, timeout=None, metadata=None, credentials=None):
+    """Synchronously invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      The response value for the RPC.
+
+    Raises:
+      RpcError: Indicating that the RPC terminated with non-OK status. The
+        raised RpcError will also be a Call for the RPC affording the RPC's
+        metadata, status code, and details.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def with_call(self, request, timeout=None, metadata=None, credentials=None):
+    """Synchronously invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional durating of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      The response value for the RPC and a Call value for the RPC.
+
+    Raises:
+      RpcError: Indicating that the RPC terminated with non-OK status. The
+        raised RpcError will also be a Call for the RPC affording the RPC's
+        metadata, status code, and details.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def future(self, request, timeout=None, metadata=None, credentials=None):
+    """Asynchronously invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and a Future. In the event of
+        RPC completion, the return Future's result value will be the response
+        message of the RPC. Should the event terminate with non-OK status, the
+        returned Future's exception value will be an RpcError.
+    """
+    raise NotImplementedError()
+
+
+class UnaryStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a unary-stream RPC."""
+
+  @abc.abstractmethod
+  def __call__(self, request, timeout=None, metadata=None, credentials=None):
+    """Invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and an iterator of response
+        values. Drawing response values from the returned iterator may raise
+        RpcError indicating termination of the RPC with non-OK status.
+    """
+    raise NotImplementedError()
+
+
+class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a stream-unary RPC in any call style."""
+
+  @abc.abstractmethod
+  def __call__(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    """Synchronously invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      The response value for the RPC, and a Call for the RPC if with_call was
+        set to True at invocation.
+
+    Raises:
+      RpcError: Indicating that the RPC terminated with non-OK status. The
+        raised RpcError will also be a Call for the RPC affording the RPC's
+        metadata, status code, and details.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def with_call(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    """Synchronously invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      The response value for the RPC and a Call for the RPC.
+
+    Raises:
+      RpcError: Indicating that the RPC terminated with non-OK status. The
+        raised RpcError will also be a Call for the RPC affording the RPC's
+        metadata, status code, and details.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def future(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    """Asynchronously invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and a Future. In the event of
+        RPC completion, the return Future's result value will be the response
+        message of the RPC. Should the event terminate with non-OK status, the
+        returned Future's exception value will be an RpcError.
+    """
+    raise NotImplementedError()
+
+
+class StreamStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a stream-stream RPC in any call style."""
+
+  @abc.abstractmethod
+  def __call__(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    """Invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      credentials: An optional CallCredentials for the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and an iterator of response
+        values. Drawing response values from the returned iterator may raise
+        RpcError indicating termination of the RPC with non-OK status.
+    """
+    raise NotImplementedError()
+
+
+#############################  Channel Interface  ##############################
+
+
+class Channel(six.with_metaclass(abc.ABCMeta)):
+  """Affords RPC invocation via generic methods."""
+
+  @abc.abstractmethod
+  def subscribe(self, callback, try_to_connect=False):
+    """Subscribes to this Channel's connectivity.
+
+    Args:
+      callback: A callable to be invoked and passed a ChannelConnectivity value
+        describing this Channel's connectivity. The callable will be invoked
+        immediately upon subscription and again for every change to this
+        Channel's connectivity thereafter until it is unsubscribed or this
+        Channel object goes out of scope.
+      try_to_connect: A boolean indicating whether or not this Channel should
+        attempt to connect if it is not already connected and ready to conduct
+        RPCs.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def unsubscribe(self, callback):
+    """Unsubscribes a callback from this Channel's connectivity.
+
+    Args:
+      callback: A callable previously registered with this Channel from having
+        been passed to its "subscribe" method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def unary_unary(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a UnaryUnaryMultiCallable for a unary-unary method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A UnaryUnaryMultiCallable value for the named unary-unary method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def unary_stream(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a UnaryStreamMultiCallable for a unary-stream method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A UnaryStreamMultiCallable value for the name unary-stream method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def stream_unary(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a StreamUnaryMultiCallable for a stream-unary method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A StreamUnaryMultiCallable value for the named stream-unary method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def stream_stream(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a StreamStreamMultiCallable for a stream-stream method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A StreamStreamMultiCallable value for the named stream-stream method.
+    """
+    raise NotImplementedError()
+
+
+##########################  Service-Side Context  ##############################
+
+
+class ServicerContext(six.with_metaclass(abc.ABCMeta, RpcContext)):
+  """A context object passed to method implementations."""
+
+  @abc.abstractmethod
+  def invocation_metadata(self):
+    """Accesses the metadata from the invocation-side of the RPC.
+
+    Returns:
+      The invocation metadata object as a sequence of pairs of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def peer(self):
+    """Identifies the peer that invoked the RPC being serviced.
+
+    Returns:
+      A string identifying the peer that invoked the RPC being serviced.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def send_initial_metadata(self, initial_metadata):
+    """Sends the initial metadata value to the invocation-side of the RPC.
+
+    This method need not be called by method implementations if they have no
+    service-side initial metadata to transmit.
+
+    Args:
+      initial_metadata: The initial metadata of the RPC as a sequence of pairs
+        of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set_trailing_metadata(self, trailing_metadata):
+    """Accepts the trailing metadata value of the RPC.
+
+    This method need not be called by method implementations if they have no
+    service-side trailing metadata to transmit.
+
+    Args:
+      trailing_metadata: The trailing metadata of the RPC as a sequence of pairs
+        of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set_code(self, code):
+    """Accepts the status code of the RPC.
+
+    This method need not be called by method implementations if they wish the
+    gRPC runtime to determine the status code of the RPC.
+
+    Args:
+      code: The integer status code of the RPC to be transmitted to the
+        invocation side of the RPC.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set_details(self, details):
+    """Accepts the service-side details of the RPC.
+
+    This method need not be called by method implementations if they have no
+    details to transmit.
+
+    Args:
+      details: The details bytes of the RPC to be transmitted to
+        the invocation side of the RPC.
+    """
+    raise NotImplementedError()
+
+
+#####################  Service-Side Handler Interfaces  ########################
+
+
+class RpcMethodHandler(six.with_metaclass(abc.ABCMeta)):
+  """An implementation of a single RPC method.
+
+  Attributes:
+    request_streaming: Whether the RPC supports exactly one request message or
+      any arbitrary number of request messages.
+    response_streaming: Whether the RPC supports exactly one response message or
+      any arbitrary number of response messages.
+    request_deserializer: A callable behavior that accepts a byte string and
+      returns an object suitable to be passed to this object's business logic,
+      or None to indicate that this object's business logic should be passed the
+      raw request bytes.
+    response_serializer: A callable behavior that accepts an object produced by
+      this object's business logic and returns a byte string, or None to
+      indicate that the byte strings produced by this object's business logic
+      should be transmitted on the wire as they are.
+    unary_unary: This object's application-specific business logic as a callable
+      value that takes a request value and a ServicerContext object and returns
+      a response value. Only non-None if both request_streaming and
+      response_streaming are False.
+    unary_stream: This object's application-specific business logic as a
+      callable value that takes a request value and a ServicerContext object and
+      returns an iterator of response values. Only non-None if request_streaming
+      is False and response_streaming is True.
+    stream_unary: This object's application-specific business logic as a
+      callable value that takes an iterator of request values and a
+      ServicerContext object and returns a response value. Only non-None if
+      request_streaming is True and response_streaming is False.
+    stream_stream: This object's application-specific business logic as a
+      callable value that takes an iterator of request values and a
+      ServicerContext object and returns an iterator of response values. Only
+      non-None if request_streaming and response_streaming are both True.
+  """
+
+
+class HandlerCallDetails(six.with_metaclass(abc.ABCMeta)):
+  """Describes an RPC that has just arrived for service.
+  Attributes:
+    method: The method name of the RPC.
+    invocation_metadata: The metadata from the invocation side of the RPC.
+  """
+
+
+class GenericRpcHandler(six.with_metaclass(abc.ABCMeta)):
+  """An implementation of arbitrarily many RPC methods."""
+
+  @abc.abstractmethod
+  def service(self, handler_call_details):
+    """Services an RPC (or not).
+
+    Args:
+      handler_call_details: A HandlerCallDetails describing the RPC.
+
+    Returns:
+      An RpcMethodHandler with which the RPC may be serviced, or None to
+        indicate that this object will not be servicing the RPC.
+    """
+    raise NotImplementedError()
+
+
+#############################  Server Interface  ###############################
+
+
+class Server(six.with_metaclass(abc.ABCMeta)):
+  """Services RPCs."""
+
+  @abc.abstractmethod
+  def add_generic_rpc_handlers(self, generic_rpc_handlers):
+    """Registers GenericRpcHandlers with this Server.
+
+    This method is only safe to call before the server is started.
+
+    Args:
+      generic_rpc_handlers: An iterable of GenericRpcHandlers that will be used
+        to service RPCs after this Server is started.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_insecure_port(self, address):
+    """Reserves a port for insecure RPC service once this Server becomes active.
+
+    This method may only be called before calling this Server's start method is
+    called.
+
+    Args:
+      address: The address for which to open a port.
+
+    Returns:
+      An integer port on which RPCs will be serviced after this link has been
+        started. This is typically the same number as the port number contained
+        in the passed address, but will likely be different if the port number
+        contained in the passed address was zero.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_secure_port(self, address, server_credentials):
+    """Reserves a port for secure RPC service after this Server becomes active.
+
+    This method may only be called before calling this Server's start method is
+    called.
+
+    Args:
+      address: The address for which to open a port.
+      server_credentials: A ServerCredentials.
+
+    Returns:
+      An integer port on which RPCs will be serviced after this link has been
+        started. This is typically the same number as the port number contained
+        in the passed address, but will likely be different if the port number
+        contained in the passed address was zero.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def start(self):
+    """Starts this Server's service of RPCs.
+
+    This method may only be called while the server is not serving RPCs (i.e. it
+    is not idempotent).
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def stop(self, grace):
+    """Stops this Server's service of RPCs.
+
+    All calls to this method immediately stop service of new RPCs. When existing
+    RPCs are aborted is controlled by the grace period parameter passed to this
+    method.
+
+    This method may be called at any time and is idempotent. Passing a smaller
+    grace value than has been passed in a previous call will have the effect of
+    stopping the Server sooner. Passing a larger grace value than has been
+    passed in a previous call will not have the effect of stopping the server
+    later.
+
+    Args:
+      grace: A duration of time in seconds to allow existing RPCs to complete
+        before being aborted by this Server's stopping. If None, this method
+        will block until the server is completely stopped.
+
+    Returns:
+      A threading.Event that will be set when this Server has completely
+      stopped. The returned event may not be set until after the full grace
+      period (if some ongoing RPC continues for the full length of the period)
+      of it may be set much sooner (such as if this Server had no RPCs underway
+      at the time it was stopped or if all RPCs that it had underway completed
+      very early in the grace period).
+    """
+    raise NotImplementedError()
+
+
+#################################  Functions    ################################
+
+
+def unary_unary_rpc_method_handler(
+    behavior, request_deserializer=None, response_serializer=None):
+  """Creates an RpcMethodHandler for a unary-unary RPC method.
+
+  Args:
+    behavior: The implementation of an RPC method as a callable behavior taking
+      a single request value and returning a single response value.
+    request_deserializer: An optional request deserialization behavior.
+    response_serializer: An optional response serialization behavior.
+
+  Returns:
+    An RpcMethodHandler for a unary-unary RPC method constructed from the given
+      parameters.
+  """
+  from grpc import _utilities
+  return _utilities.RpcMethodHandler(
+      False, False, request_deserializer, response_serializer, behavior, None,
+      None, None)
+
+
+def unary_stream_rpc_method_handler(
+    behavior, request_deserializer=None, response_serializer=None):
+  """Creates an RpcMethodHandler for a unary-stream RPC method.
+
+  Args:
+    behavior: The implementation of an RPC method as a callable behavior taking
+      a single request value and returning an iterator of response values.
+    request_deserializer: An optional request deserialization behavior.
+    response_serializer: An optional response serialization behavior.
+
+  Returns:
+    An RpcMethodHandler for a unary-stream RPC method constructed from the
+      given parameters.
+  """
+  from grpc import _utilities
+  return _utilities.RpcMethodHandler(
+      False, True, request_deserializer, response_serializer, None, behavior,
+      None, None)
+
+
+def stream_unary_rpc_method_handler(
+    behavior, request_deserializer=None, response_serializer=None):
+  """Creates an RpcMethodHandler for a stream-unary RPC method.
+
+  Args:
+    behavior: The implementation of an RPC method as a callable behavior taking
+      an iterator of request values and returning a single response value.
+    request_deserializer: An optional request deserialization behavior.
+    response_serializer: An optional response serialization behavior.
+
+  Returns:
+    An RpcMethodHandler for a stream-unary RPC method constructed from the
+      given parameters.
+  """
+  from grpc import _utilities
+  return _utilities.RpcMethodHandler(
+      True, False, request_deserializer, response_serializer, None, None,
+      behavior, None)
+
+
+def stream_stream_rpc_method_handler(
+        behavior, request_deserializer=None, response_serializer=None):
+  """Creates an RpcMethodHandler for a stream-stream RPC method.
+
+  Args:
+    behavior: The implementation of an RPC method as a callable behavior taking
+      an iterator of request values and returning an iterator of response
+      values.
+    request_deserializer: An optional request deserialization behavior.
+    response_serializer: An optional response serialization behavior.
+
+  Returns:
+    An RpcMethodHandler for a stream-stream RPC method constructed from the
+      given parameters.
+  """
+  from grpc import _utilities
+  return _utilities.RpcMethodHandler(
+      True, True, request_deserializer, response_serializer, None, None, None,
+      behavior)
+
+
+def method_handlers_generic_handler(service, method_handlers):
+  """Creates a grpc.GenericRpcHandler from RpcMethodHandlers.
+
+  Args:
+    service: A service name to be used for the given method handlers.
+    method_handlers: A dictionary from method name to RpcMethodHandler
+      implementing the named method.
+
+  Returns:
+    A GenericRpcHandler constructed from the given parameters.
+  """
+  from grpc import _utilities
+  return _utilities.DictionaryGenericHandler(service, method_handlers)
+
+
+def ssl_channel_credentials(
+    root_certificates=None, private_key=None, certificate_chain=None):
+  """Creates a ChannelCredentials for use with an SSL-enabled Channel.
+
+  Args:
+    root_certificates: The PEM-encoded root certificates or unset to ask for
+      them to be retrieved from a default location.
+    private_key: The PEM-encoded private key to use or unset if no private key
+      should be used.
+    certificate_chain: The PEM-encoded certificate chain to use or unset if no
+      certificate chain should be used.
+
+  Returns:
+    A ChannelCredentials for use with an SSL-enabled Channel.
+  """
+  if private_key is not None or certificate_chain is not None:
+    pair = _cygrpc.SslPemKeyCertPair(private_key, certificate_chain)
+  else:
+    pair = None
+  return ChannelCredentials(
+      _cygrpc.channel_credentials_ssl(root_certificates, pair))
+
+
+def metadata_call_credentials(metadata_plugin, name=None):
+  """Construct CallCredentials from an AuthMetadataPlugin.
+
+  Args:
+    metadata_plugin: An AuthMetadataPlugin to use as the authentication behavior
+      in the created CallCredentials.
+    name: A name for the plugin.
+
+  Returns:
+    A CallCredentials.
+  """
+  from grpc import _plugin_wrapping
+  if name is None:
+    try:
+      effective_name = metadata_plugin.__name__
+    except AttributeError:
+      effective_name = metadata_plugin.__class__.__name__
+  else:
+    effective_name = name
+  return CallCredentials(
+      _plugin_wrapping.call_credentials_metadata_plugin(
+          metadata_plugin, effective_name))
+
+
+def access_token_call_credentials(access_token):
+  """Construct CallCredentials from an access token.
+
+  Args:
+    access_token: A string to place directly in the http request
+      authorization header, ie "Authorization: Bearer <access_token>".
+
+  Returns:
+    A CallCredentials.
+  """
+  from grpc import _auth
+  return metadata_call_credentials(
+      _auth.AccessTokenCallCredentials(access_token))
+
+
+def composite_call_credentials(call_credentials, additional_call_credentials):
+  """Compose two CallCredentials to make a new one.
+
+  Args:
+    call_credentials: A CallCredentials object.
+    additional_call_credentials: Another CallCredentials object to compose on
+      top of call_credentials.
+
+  Returns:
+    A new CallCredentials composed of the two given CallCredentials.
+  """
+  return CallCredentials(
+      _cygrpc.call_credentials_composite(
+          call_credentials._credentials,
+          additional_call_credentials._credentials))
+
+
+def composite_channel_credentials(channel_credentials, call_credentials):
+  """Compose a ChannelCredentials and a CallCredentials.
+
+  Args:
+    channel_credentials: A ChannelCredentials.
+    call_credentials: A CallCredentials.
+
+  Returns:
+    A ChannelCredentials composed of the given ChannelCredentials and
+      CallCredentials.
+  """
+  return ChannelCredentials(
+      _cygrpc.channel_credentials_composite(
+          channel_credentials._credentials, call_credentials._credentials))
+
+
+def ssl_server_credentials(
+    private_key_certificate_chain_pairs, root_certificates=None,
+    require_client_auth=False):
+  """Creates a ServerCredentials for use with an SSL-enabled Server.
+
+  Args:
+    private_key_certificate_chain_pairs: A nonempty sequence each element of
+      which is a pair the first element of which is a PEM-encoded private key
+      and the second element of which is the corresponding PEM-encoded
+      certificate chain.
+    root_certificates: PEM-encoded client root certificates to be used for
+      verifying authenticated clients. If omitted, require_client_auth must also
+      be omitted or be False.
+    require_client_auth: A boolean indicating whether or not to require clients
+      to be authenticated. May only be True if root_certificates is not None.
+
+  Returns:
+    A ServerCredentials for use with an SSL-enabled Server.
+  """
+  if len(private_key_certificate_chain_pairs) == 0:
+    raise ValueError(
+        'At least one private key-certificate chain pair is required!')
+  elif require_client_auth and root_certificates is None:
+    raise ValueError(
+        'Illegal to require client auth without providing root certificates!')
+  else:
+    return ServerCredentials(
+        _cygrpc.server_credentials_ssl(
+        root_certificates,
+        [_cygrpc.SslPemKeyCertPair(key, pem)
+         for key, pem in private_key_certificate_chain_pairs],
+        require_client_auth))
+
+
+def channel_ready_future(channel):
+  """Creates a Future tracking when a Channel is ready.
+
+  Cancelling the returned Future does not tell the given Channel to abandon
+  attempts it may have been making to connect; cancelling merely deactivates the
+  returned Future's subscription to the given Channel's connectivity.
+
+  Args:
+    channel: A Channel.
+
+  Returns:
+    A Future that matures when the given Channel has connectivity
+      ChannelConnectivity.READY.
+  """
+  from grpc import _utilities
+  return _utilities.channel_ready_future(channel)
+
+
+def insecure_channel(target, options=None):
+  """Creates an insecure Channel to a server.
+
+  Args:
+    target: The target to which to connect.
+    options: A sequence of string-value pairs according to which to configure
+      the created channel.
+
+  Returns:
+    A Channel to the target through which RPCs may be conducted.
+  """
+  from grpc import _channel
+  return _channel.Channel(target, options, None)
+
+
+def secure_channel(target, credentials, options=None):
+  """Creates an insecure Channel to a server.
+
+  Args:
+    target: The target to which to connect.
+    credentials: A ChannelCredentials instance.
+    options: A sequence of string-value pairs according to which to configure
+      the created channel.
+
+  Returns:
+    A Channel to the target through which RPCs may be conducted.
+  """
+  from grpc import _channel
+  return _channel.Channel(target, options, credentials._credentials)
+
+
+def server(generic_rpc_handlers, thread_pool, options=None):
+  """Creates a Server with which RPCs can be serviced.
+
+  The GenericRpcHandlers passed to this function needn't be the only
+  GenericRpcHandlers that will be used to serve RPCs; others may be added later
+  by calling add_generic_rpc_handlers any time before the returned server is
+  started.
+
+  Args:
+    generic_rpc_handlers: Some number of GenericRpcHandlers that will be used
+      to service RPCs after the returned Server is started.
+    thread_pool: A futures.ThreadPoolExecutor to be used by the returned Server
+      to service RPCs.
+
+  Returns:
+    A Server with which RPCs can be serviced.
+  """
+  from grpc import _server
+  return _server.Server(generic_rpc_handlers, thread_pool)
+
+
+###################################  __all__  #################################
+
+
+__all__ = (
+    'FutureTimeoutError',
+    'FutureCancelledError',
+    'Future',
+    'ChannelConnectivity',
+    'StatusCode',
+    'RpcError',
+    'RpcContext',
+    'Call',
+    'ChannelCredentials',
+    'CallCredentials',
+    'AuthMetadataContext',
+    'AuthMetadataPluginCallback',
+    'AuthMetadataPlugin',
+    'ServerCredentials',
+    'UnaryUnaryMultiCallable',
+    'UnaryStreamMultiCallable',
+    'StreamUnaryMultiCallable',
+    'StreamStreamMultiCallable',
+    'Channel',
+    'ServicerContext',
+    'RpcMethodHandler',
+    'HandlerCallDetails',
+    'GenericRpcHandler',
+    'Server',
+    'unary_unary_rpc_method_handler',
+    'unary_stream_rpc_method_handler',
+    'stream_unary_rpc_method_handler',
+    'stream_stream_rpc_method_handler',
+    'method_handlers_generic_handler',
+    'ssl_channel_credentials',
+    'metadata_call_credentials',
+    'access_token_call_credentials',
+    'composite_call_credentials',
+    'composite_channel_credentials',
+    'ssl_server_credentials',
+    'channel_ready_future',
+    'insecure_channel',
+    'secure_channel',
+    'server',
+)
diff --git a/src/python/grpcio/grpc/_adapter/_low.py b/src/python/grpcio/grpc/_adapter/_low.py
index b13d8dd9dd..48410167a0 100644
--- a/src/python/grpcio/grpc/_adapter/_low.py
+++ b/src/python/grpcio/grpc/_adapter/_low.py
@@ -30,8 +30,8 @@
 import threading
 
 from grpc import _grpcio_metadata
+from grpc import _plugin_wrapping
 from grpc._cython import cygrpc
-from grpc._adapter import _implementations
 from grpc._adapter import _types
 
 _USER_AGENT = 'Python-gRPC-{}'.format(_grpcio_metadata.__version__)
@@ -57,78 +57,8 @@ def channel_credentials_ssl(
   return cygrpc.channel_credentials_ssl(root_certificates, pair)
 
 
-class _WrappedCygrpcCallback(object):
-
-  def __init__(self, cygrpc_callback):
-    self.is_called = False
-    self.error = None
-    self.is_called_lock = threading.Lock()
-    self.cygrpc_callback = cygrpc_callback
-
-  def _invoke_failure(self, error):
-    # TODO(atash) translate different Exception superclasses into different
-    # status codes.
-    self.cygrpc_callback(
-        cygrpc.Metadata([]), cygrpc.StatusCode.internal, error.message)
-
-  def _invoke_success(self, metadata):
-    try:
-      cygrpc_metadata = cygrpc.Metadata(
-          cygrpc.Metadatum(key, value)
-          for key, value in metadata)
-    except Exception as error:
-      self._invoke_failure(error)
-      return
-    self.cygrpc_callback(cygrpc_metadata, cygrpc.StatusCode.ok, '')
-
-  def __call__(self, metadata, error):
-    with self.is_called_lock:
-      if self.is_called:
-        raise RuntimeError('callback should only ever be invoked once')
-      if self.error:
-        self._invoke_failure(self.error)
-        return
-      self.is_called = True
-    if error is None:
-      self._invoke_success(metadata)
-    else:
-      self._invoke_failure(error)
-
-  def notify_failure(self, error):
-    with self.is_called_lock:
-      if not self.is_called:
-        self.error = error
-
-
-class _WrappedPlugin(object):
-
-  def __init__(self, plugin):
-    self.plugin = plugin
-
-  def __call__(self, context, cygrpc_callback):
-    wrapped_cygrpc_callback = _WrappedCygrpcCallback(cygrpc_callback)
-    wrapped_context = _implementations.AuthMetadataContext(context.service_url,
-                                                           context.method_name)
-    try:
-      self.plugin(
-          wrapped_context,
-          _implementations.AuthMetadataPluginCallback(wrapped_cygrpc_callback))
-    except Exception as error:
-      wrapped_cygrpc_callback.notify_failure(error)
-      raise
-
-
-def call_credentials_metadata_plugin(plugin, name):
-  """
-  Args:
-    plugin: A callable accepting a _types.AuthMetadataContext
-      object and a callback (itself accepting a list of metadata key/value
-      2-tuples and a None-able exception value). The callback must be eventually
-      called, but need not be called in plugin's invocation.
-      plugin's invocation must be non-blocking.
-  """
-  return cygrpc.call_credentials_metadata_plugin(
-      cygrpc.CredentialsMetadataPlugin(_WrappedPlugin(plugin), name))
+call_credentials_metadata_plugin = (
+    _plugin_wrapping.call_credentials_metadata_plugin)
 
 
 class CompletionQueue(_types.CompletionQueue):
@@ -195,26 +125,30 @@ class Call(_types.Call):
         translated_op = cygrpc.operation_send_initial_metadata(
             cygrpc.Metadata(
                 cygrpc.Metadatum(key, value)
-                for key, value in op.initial_metadata))
+                for key, value in op.initial_metadata),
+            op.flags)
       elif op.type == _types.OpType.SEND_MESSAGE:
-        translated_op = cygrpc.operation_send_message(op.message)
+        translated_op = cygrpc.operation_send_message(op.message, op.flags)
       elif op.type == _types.OpType.SEND_CLOSE_FROM_CLIENT:
-        translated_op = cygrpc.operation_send_close_from_client()
+        translated_op = cygrpc.operation_send_close_from_client(op.flags)
       elif op.type == _types.OpType.SEND_STATUS_FROM_SERVER:
         translated_op = cygrpc.operation_send_status_from_server(
             cygrpc.Metadata(
                 cygrpc.Metadatum(key, value)
                 for key, value in op.trailing_metadata),
             op.status.code,
-            op.status.details)
+            op.status.details,
+            op.flags)
       elif op.type == _types.OpType.RECV_INITIAL_METADATA:
-        translated_op = cygrpc.operation_receive_initial_metadata()
+        translated_op = cygrpc.operation_receive_initial_metadata(
+            op.flags)
       elif op.type == _types.OpType.RECV_MESSAGE:
-        translated_op = cygrpc.operation_receive_message()
+        translated_op = cygrpc.operation_receive_message(op.flags)
       elif op.type == _types.OpType.RECV_STATUS_ON_CLIENT:
-        translated_op = cygrpc.operation_receive_status_on_client()
+        translated_op = cygrpc.operation_receive_status_on_client(
+            op.flags)
       elif op.type == _types.OpType.RECV_CLOSE_ON_SERVER:
-        translated_op = cygrpc.operation_receive_close_on_server()
+        translated_op = cygrpc.operation_receive_close_on_server(op.flags)
       else:
         raise ValueError('unexpected operation type {}'.format(op.type))
       translated_ops.append(translated_op)
diff --git a/src/python/grpcio/grpc/_adapter/_types.py b/src/python/grpcio/grpc/_adapter/_types.py
index 8ca7ff4b60..b7cc6fbbb5 100644
--- a/src/python/grpcio/grpc/_adapter/_types.py
+++ b/src/python/grpcio/grpc/_adapter/_types.py
@@ -114,7 +114,7 @@ class ConnectivityState(enum.IntEnum):
   CONNECTING        = cygrpc.ConnectivityState.connecting
   READY             = cygrpc.ConnectivityState.ready
   TRANSIENT_FAILURE = cygrpc.ConnectivityState.transient_failure
-  FATAL_FAILURE     = cygrpc.ConnectivityState.fatal_failure
+  FATAL_FAILURE     = cygrpc.ConnectivityState.shutdown
 
 
 class Status(collections.namedtuple(
@@ -152,7 +152,7 @@ class OpArgs(collections.namedtuple(
         'trailing_metadata',
         'message',
         'status',
-        'write_flags',
+        'flags',
     ])):
   """Arguments passed into a GRPC operation.
 
@@ -165,7 +165,7 @@ class OpArgs(collections.namedtuple(
     message (bytes): Only valid if type == OpType.SEND_MESSAGE, else is None.
     status (Status): Only valid if type == OpType.SEND_STATUS_FROM_SERVER, else
       is None.
-    write_flags (int): a bit OR'ing of 0 or more OpWriteFlags values.
+    flags (int): a bitwise OR'ing of 0 or more OpWriteFlags values.
   """
 
   @staticmethod
diff --git a/src/python/grpcio/grpc/_auth.py b/src/python/grpcio/grpc/_auth.py
new file mode 100644
index 0000000000..dea3221c9d
--- /dev/null
+++ b/src/python/grpcio/grpc/_auth.py
@@ -0,0 +1,86 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""GRPCAuthMetadataPlugins for standard authentication."""
+
+import inspect
+from concurrent import futures
+
+import grpc
+
+
+def _sign_request(callback, token, error):
+  metadata = (('authorization', 'Bearer {}'.format(token)),)
+  callback(metadata, error)
+
+
+class GoogleCallCredentials(grpc.AuthMetadataPlugin):
+  """Metadata wrapper for GoogleCredentials from the oauth2client library."""
+
+  def __init__(self, credentials):
+    self._credentials = credentials
+    self._pool = futures.ThreadPoolExecutor(max_workers=1)
+
+    # Hack to determine if these are JWT creds and we need to pass
+    # additional_claims when getting a token
+    if 'additional_claims' in inspect.getargspec(
+        credentials.get_access_token).args:
+      self._is_jwt = True
+    else:
+      self._is_jwt = False
+
+  def __call__(self, context, callback):
+    # MetadataPlugins cannot block (see grpc.beta.interfaces.py)
+    if self._is_jwt:
+      future = self._pool.submit(self._credentials.get_access_token,
+                                 additional_claims={'aud': context.service_url})
+    else:
+      future = self._pool.submit(self._credentials.get_access_token)
+    future.add_done_callback(lambda x: self._get_token_callback(callback, x))
+
+  def _get_token_callback(self, callback, future):
+    try:
+      access_token = future.result().access_token
+    except Exception as e:
+      _sign_request(callback, None, e)
+    else:
+      _sign_request(callback, access_token, None)
+
+  def __del__(self):
+    self._pool.shutdown(wait=False)
+
+
+class AccessTokenCallCredentials(grpc.AuthMetadataPlugin):
+  """Metadata wrapper for raw access token credentials."""
+
+  def __init__(self, access_token):
+    self._access_token = access_token
+
+  def __call__(self, context, callback):
+    _sign_request(callback, self._access_token, None)
diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py
new file mode 100644
index 0000000000..a89b501303
--- /dev/null
+++ b/src/python/grpcio/grpc/_channel.py
@@ -0,0 +1,904 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Invocation-side implementation of gRPC Python."""
+
+import sys
+import threading
+import time
+
+import grpc
+from grpc import _common
+from grpc import _grpcio_metadata
+from grpc.framework.foundation import callable_util
+from grpc._cython import cygrpc
+
+_USER_AGENT = 'Python-gRPC-{}'.format(_grpcio_metadata.__version__)
+
+_EMPTY_FLAGS = 0
+_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+_UNARY_UNARY_INITIAL_DUE = (
+    cygrpc.OperationType.send_initial_metadata,
+    cygrpc.OperationType.send_message,
+    cygrpc.OperationType.send_close_from_client,
+    cygrpc.OperationType.receive_initial_metadata,
+    cygrpc.OperationType.receive_message,
+    cygrpc.OperationType.receive_status_on_client,
+)
+_UNARY_STREAM_INITIAL_DUE = (
+    cygrpc.OperationType.send_initial_metadata,
+    cygrpc.OperationType.send_message,
+    cygrpc.OperationType.send_close_from_client,
+    cygrpc.OperationType.receive_initial_metadata,
+    cygrpc.OperationType.receive_status_on_client,
+)
+_STREAM_UNARY_INITIAL_DUE = (
+    cygrpc.OperationType.send_initial_metadata,
+    cygrpc.OperationType.receive_initial_metadata,
+    cygrpc.OperationType.receive_message,
+    cygrpc.OperationType.receive_status_on_client,
+)
+_STREAM_STREAM_INITIAL_DUE = (
+    cygrpc.OperationType.send_initial_metadata,
+    cygrpc.OperationType.receive_initial_metadata,
+    cygrpc.OperationType.receive_status_on_client,
+)
+
+_CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = (
+    'Exception calling channel subscription callback!')
+
+
+def _deadline(timeout):
+  if timeout is None:
+    return None, _INFINITE_FUTURE
+  else:
+    deadline = time.time() + timeout
+    return deadline, cygrpc.Timespec(deadline)
+
+
+def _unknown_code_details(unknown_cygrpc_code, details):
+  return 'Server sent unknown code {} and details "{}"'.format(
+      unknown_cygrpc_code, details)
+
+
+def _wait_once_until(condition, until):
+  if until is None:
+    condition.wait()
+  else:
+    remaining = until - time.time()
+    if remaining < 0:
+      raise grpc.FutureTimeoutError()
+    else:
+      condition.wait(timeout=remaining)
+
+
+class _RPCState(object):
+
+  def __init__(self, due, initial_metadata, trailing_metadata, code, details):
+    self.condition = threading.Condition()
+    # The cygrpc.OperationType objects representing events due from the RPC's
+    # completion queue.
+    self.due = set(due)
+    self.initial_metadata = initial_metadata
+    self.response = None
+    self.trailing_metadata = trailing_metadata
+    self.code = code
+    self.details = details
+    # The semantics of grpc.Future.cancel and grpc.Future.cancelled are
+    # slightly wonky, so they have to be tracked separately from the rest of the
+    # result of the RPC. This field tracks whether cancellation was requested
+    # prior to termination of the RPC.
+    self.cancelled = False
+    self.callbacks = []
+
+
+def _abort(state, code, details):
+  if state.code is None:
+    state.code = code
+    state.details = details
+    if state.initial_metadata is None:
+      state.initial_metadata = _EMPTY_METADATA
+    state.trailing_metadata = _EMPTY_METADATA
+
+
+def _handle_event(event, state, response_deserializer):
+  callbacks = []
+  for batch_operation in event.batch_operations:
+    operation_type = batch_operation.type
+    state.due.remove(operation_type)
+    if operation_type == cygrpc.OperationType.receive_initial_metadata:
+      state.initial_metadata = batch_operation.received_metadata
+    elif operation_type == cygrpc.OperationType.receive_message:
+      serialized_response = batch_operation.received_message.bytes()
+      if serialized_response is not None:
+        response = _common.deserialize(
+            serialized_response, response_deserializer)
+        if response is None:
+          details = 'Exception deserializing response!'
+          _abort(state, grpc.StatusCode.INTERNAL, details)
+        else:
+          state.response = response
+    elif operation_type == cygrpc.OperationType.receive_status_on_client:
+      state.trailing_metadata = batch_operation.received_metadata
+      if state.code is None:
+        code = _common.CYGRPC_STATUS_CODE_TO_STATUS_CODE.get(
+            batch_operation.received_status_code)
+        if code is None:
+          state.code = grpc.StatusCode.UNKNOWN
+          state.details = _unknown_code_details(
+              batch_operation.received_status_code,
+              batch_operation.received_status_details)
+        else:
+          state.code = code
+          state.details = batch_operation.received_status_details
+      callbacks.extend(state.callbacks)
+      state.callbacks = None
+  return callbacks
+
+
+def _event_handler(state, call, response_deserializer):
+  def handle_event(event):
+    with state.condition:
+      callbacks = _handle_event(event, state, response_deserializer)
+      state.condition.notify_all()
+      done = not state.due
+    for callback in callbacks:
+      callback()
+    return call if done else None
+  return handle_event
+
+
+def _consume_request_iterator(
+    request_iterator, state, call, request_serializer):
+  event_handler = _event_handler(state, call, None)
+
+  def consume_request_iterator():
+    for request in request_iterator:
+      serialized_request = _common.serialize(request, request_serializer)
+      with state.condition:
+        if state.code is None and not state.cancelled:
+          if serialized_request is None:
+            call.cancel()
+            details = 'Exception serializing request!'
+            _abort(state, grpc.StatusCode.INTERNAL, details)
+            return
+          else:
+            operations = (
+                cygrpc.operation_send_message(
+                    serialized_request, _EMPTY_FLAGS),
+            )
+            call.start_batch(cygrpc.Operations(operations), event_handler)
+            state.due.add(cygrpc.OperationType.send_message)
+            while True:
+              state.condition.wait()
+              if state.code is None:
+                if cygrpc.OperationType.send_message not in state.due:
+                  break
+              else:
+                return
+        else:
+          return
+    with state.condition:
+      if state.code is None:
+        operations = (
+            cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+        )
+        call.start_batch(cygrpc.Operations(operations), event_handler)
+        state.due.add(cygrpc.OperationType.send_close_from_client)
+
+  def stop_consumption_thread(timeout):
+    with state.condition:
+      if state.code is None:
+        call.cancel()
+        state.cancelled = True
+        _abort(state, grpc.StatusCode.CANCELLED, 'Cancelled!')
+        state.condition.notify_all()
+
+  consumption_thread = _common.CleanupThread(
+      stop_consumption_thread, target=consume_request_iterator)
+  consumption_thread.start()
+
+
+class _Rendezvous(grpc.RpcError, grpc.Future, grpc.Call):
+
+  def __init__(self, state, call, response_deserializer, deadline):
+    super(_Rendezvous, self).__init__()
+    self._state = state
+    self._call = call
+    self._response_deserializer = response_deserializer
+    self._deadline = deadline
+
+  def cancel(self):
+    with self._state.condition:
+      if self._state.code is None:
+        self._call.cancel()
+        self._state.cancelled = True
+        _abort(self._state, grpc.StatusCode.CANCELLED, 'Cancelled!')
+        self._state.condition.notify_all()
+      return False
+
+  def cancelled(self):
+    with self._state.condition:
+      return self._state.cancelled
+
+  def running(self):
+    with self._state.condition:
+      return self._state.code is None
+
+  def done(self):
+    with self._state.condition:
+      return self._state.code is not None
+
+  def result(self, timeout=None):
+    until = None if timeout is None else time.time() + timeout
+    with self._state.condition:
+      while True:
+        if self._state.code is None:
+          _wait_once_until(self._state.condition, until)
+        elif self._state.code is grpc.StatusCode.OK:
+          return self._state.response
+        elif self._state.cancelled:
+          raise grpc.FutureCancelledError()
+        else:
+          raise self
+
+  def exception(self, timeout=None):
+    until = None if timeout is None else time.time() + timeout
+    with self._state.condition:
+      while True:
+        if self._state.code is None:
+          _wait_once_until(self._state.condition, until)
+        elif self._state.code is grpc.StatusCode.OK:
+          return None
+        elif self._state.cancelled:
+          raise grpc.FutureCancelledError()
+        else:
+          return self
+
+  def traceback(self, timeout=None):
+    until = None if timeout is None else time.time() + timeout
+    with self._state.condition:
+      while True:
+        if self._state.code is None:
+          _wait_once_until(self._state.condition, until)
+        elif self._state.code is grpc.StatusCode.OK:
+          return None
+        elif self._state.cancelled:
+          raise grpc.FutureCancelledError()
+        else:
+          try:
+            raise self
+          except grpc.RpcError:
+            return sys.exc_info()[2]
+
+  def add_done_callback(self, fn):
+    with self._state.condition:
+      if self._state.code is None:
+        self._state.callbacks.append(lambda: fn(self))
+        return
+
+    fn(self)
+
+  def _next(self):
+    with self._state.condition:
+      if self._state.code is None:
+        event_handler = _event_handler(
+            self._state, self._call, self._response_deserializer)
+        self._call.start_batch(
+            cygrpc.Operations(
+                (cygrpc.operation_receive_message(_EMPTY_FLAGS),)),
+            event_handler)
+        self._state.due.add(cygrpc.OperationType.receive_message)
+      elif self._state.code is grpc.StatusCode.OK:
+        raise StopIteration()
+      else:
+        raise self
+      while True:
+        self._state.condition.wait()
+        if self._state.response is not None:
+          response = self._state.response
+          self._state.response = None
+          return response
+        elif cygrpc.OperationType.receive_message not in self._state.due:
+          if self._state.code is grpc.StatusCode.OK:
+            raise StopIteration()
+          elif self._state.code is not None:
+            raise self
+
+  def __iter__(self):
+    return self
+
+  def __next__(self):
+    return self._next()
+
+  def next(self):
+    return self._next()
+
+  def is_active(self):
+    with self._state.condition:
+      return self._state.code is None
+
+  def time_remaining(self):
+    if self._deadline is None:
+      return None
+    else:
+      return max(self._deadline - time.time(), 0)
+
+  def add_cancellation_callback(self, callback):
+    with self._state.condition:
+      if self._state.callbacks is None:
+        return False
+      else:
+        self._state.callbacks.append(lambda unused_future: callback())
+        return True
+
+  def initial_metadata(self):
+    with self._state.condition:
+      while self._state.initial_metadata is None:
+        self._state.condition.wait()
+      return _common.application_metadata(self._state.initial_metadata)
+
+  def trailing_metadata(self):
+    with self._state.condition:
+      while self._state.trailing_metadata is None:
+        self._state.condition.wait()
+      return _common.application_metadata(self._state.trailing_metadata)
+
+  def code(self):
+    with self._state.condition:
+      while self._state.code is None:
+        self._state.condition.wait()
+      return self._state.code
+
+  def details(self):
+    with self._state.condition:
+      while self._state.details is None:
+        self._state.condition.wait()
+      return _common.decode(self._state.details)
+
+  def _repr(self):
+    with self._state.condition:
+      if self._state.code is None:
+        return '<_Rendezvous object of in-flight RPC>'
+      else:
+        return '<_Rendezvous of RPC that terminated with ({}, {})>'.format(
+            self._state.code, _common.decode(self._state.details))
+
+  def __repr__(self):
+    return self._repr()
+
+  def __str__(self):
+    return self._repr()
+
+  def __del__(self):
+    with self._state.condition:
+      if self._state.code is None:
+        self._call.cancel()
+        self._state.cancelled = True
+        self._state.code = grpc.StatusCode.CANCELLED
+        self._state.condition.notify_all()
+
+
+def _start_unary_request(request, timeout, request_serializer):
+  deadline, deadline_timespec = _deadline(timeout)
+  serialized_request = _common.serialize(request, request_serializer)
+  if serialized_request is None:
+    state = _RPCState(
+        (), _EMPTY_METADATA, _EMPTY_METADATA, grpc.StatusCode.INTERNAL,
+        'Exception serializing request!')
+    rendezvous = _Rendezvous(state, None, None, deadline)
+    return deadline, deadline_timespec, None, rendezvous
+  else:
+    return deadline, deadline_timespec, serialized_request, None
+
+
+def _end_unary_response_blocking(state, with_call, deadline):
+  if state.code is grpc.StatusCode.OK:
+    if with_call:
+      rendezvous = _Rendezvous(state, None, None, deadline)
+      return state.response, rendezvous
+    else:
+      return state.response
+  else:
+    raise _Rendezvous(state, None, None, deadline)
+
+
+class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
+
+  def __init__(
+      self, channel, create_managed_call, method, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._create_managed_call = create_managed_call
+    self._method = method
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def _prepare(self, request, timeout, metadata):
+    deadline, deadline_timespec, serialized_request, rendezvous = (
+        _start_unary_request(request, timeout, self._request_serializer))
+    if serialized_request is None:
+      return None, None, None, None, rendezvous
+    else:
+      state = _RPCState(_UNARY_UNARY_INITIAL_DUE, None, None, None, None)
+      operations = (
+          cygrpc.operation_send_initial_metadata(
+              _common.cygrpc_metadata(metadata), _EMPTY_FLAGS),
+          cygrpc.operation_send_message(serialized_request, _EMPTY_FLAGS),
+          cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+          cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+          cygrpc.operation_receive_message(_EMPTY_FLAGS),
+          cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+      )
+      return state, operations, deadline, deadline_timespec, None
+
+  def _blocking(self, request, timeout, metadata, credentials):
+    state, operations, deadline, deadline_timespec, rendezvous = self._prepare(
+        request, timeout, metadata)
+    if rendezvous:
+      raise rendezvous
+    else:
+      completion_queue = cygrpc.CompletionQueue()
+      call = self._channel.create_call(
+          None, 0, completion_queue, self._method, None, deadline_timespec)
+      if credentials is not None:
+        call.set_credentials(credentials._credentials)
+      call.start_batch(cygrpc.Operations(operations), None)
+      _handle_event(completion_queue.poll(), state, self._response_deserializer)
+      return state, deadline
+
+  def __call__(self, request, timeout=None, metadata=None, credentials=None):
+    state, deadline, = self._blocking(request, timeout, metadata, credentials)
+    return _end_unary_response_blocking(state, False, deadline)
+
+  def with_call(self, request, timeout=None, metadata=None, credentials=None):
+    state, deadline, = self._blocking(request, timeout, metadata, credentials)
+    return _end_unary_response_blocking(state, True, deadline)
+
+  def future(self, request, timeout=None, metadata=None, credentials=None):
+    state, operations, deadline, deadline_timespec, rendezvous = self._prepare(
+        request, timeout, metadata)
+    if rendezvous:
+      return rendezvous
+    else:
+      call = self._create_managed_call(
+          None, 0, self._method, None, deadline_timespec)
+      if credentials is not None:
+        call.set_credentials(credentials._credentials)
+      event_handler = _event_handler(state, call, self._response_deserializer)
+      with state.condition:
+        call.start_batch(cygrpc.Operations(operations), event_handler)
+      return _Rendezvous(state, call, self._response_deserializer, deadline)
+
+
+class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable):
+
+  def __init__(
+      self, channel, create_managed_call, method, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._create_managed_call = create_managed_call
+    self._method = method
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def __call__(self, request, timeout=None, metadata=None, credentials=None):
+    deadline, deadline_timespec, serialized_request, rendezvous = (
+        _start_unary_request(request, timeout, self._request_serializer))
+    if serialized_request is None:
+      raise rendezvous
+    else:
+      state = _RPCState(_UNARY_STREAM_INITIAL_DUE, None, None, None, None)
+      call = self._create_managed_call(
+          None, 0, self._method, None, deadline_timespec)
+      if credentials is not None:
+        call.set_credentials(credentials._credentials)
+      event_handler = _event_handler(state, call, self._response_deserializer)
+      with state.condition:
+        call.start_batch(
+            cygrpc.Operations(
+                (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)),
+            event_handler)
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _common.cygrpc_metadata(metadata), _EMPTY_FLAGS),
+            cygrpc.operation_send_message(serialized_request, _EMPTY_FLAGS),
+            cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+            cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+        )
+        call.start_batch(cygrpc.Operations(operations), event_handler)
+      return _Rendezvous(state, call, self._response_deserializer, deadline)
+
+
+class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable):
+
+  def __init__(
+      self, channel, create_managed_call, method, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._create_managed_call = create_managed_call
+    self._method = method
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def _blocking(self, request_iterator, timeout, metadata, credentials):
+    deadline, deadline_timespec = _deadline(timeout)
+    state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None)
+    completion_queue = cygrpc.CompletionQueue()
+    call = self._channel.create_call(
+        None, 0, completion_queue, self._method, None, deadline_timespec)
+    if credentials is not None:
+      call.set_credentials(credentials._credentials)
+    with state.condition:
+      call.start_batch(
+          cygrpc.Operations(
+              (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)),
+          None)
+      operations = (
+          cygrpc.operation_send_initial_metadata(
+              _common.cygrpc_metadata(metadata), _EMPTY_FLAGS),
+          cygrpc.operation_receive_message(_EMPTY_FLAGS),
+          cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+      )
+      call.start_batch(cygrpc.Operations(operations), None)
+      _consume_request_iterator(
+          request_iterator, state, call, self._request_serializer)
+    while True:
+      event = completion_queue.poll()
+      with state.condition:
+        _handle_event(event, state, self._response_deserializer)
+        state.condition.notify_all()
+        if not state.due:
+          break
+    return state, deadline
+
+  def __call__(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    state, deadline, = self._blocking(
+        request_iterator, timeout, metadata, credentials)
+    return _end_unary_response_blocking(state, False, deadline)
+
+  def with_call(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    state, deadline, = self._blocking(
+        request_iterator, timeout, metadata, credentials)
+    return _end_unary_response_blocking(state, True, deadline)
+
+  def future(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    deadline, deadline_timespec = _deadline(timeout)
+    state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None)
+    call = self._create_managed_call(
+        None, 0, self._method, None, deadline_timespec)
+    if credentials is not None:
+      call.set_credentials(credentials._credentials)
+    event_handler = _event_handler(state, call, self._response_deserializer)
+    with state.condition:
+      call.start_batch(
+          cygrpc.Operations(
+              (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)),
+          event_handler)
+      operations = (
+          cygrpc.operation_send_initial_metadata(
+              _common.cygrpc_metadata(metadata), _EMPTY_FLAGS),
+          cygrpc.operation_receive_message(_EMPTY_FLAGS),
+          cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+      )
+      call.start_batch(cygrpc.Operations(operations), event_handler)
+      _consume_request_iterator(
+          request_iterator, state, call, self._request_serializer)
+    return _Rendezvous(state, call, self._response_deserializer, deadline)
+
+
+class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable):
+
+  def __init__(
+      self, channel, create_managed_call, method, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._create_managed_call = create_managed_call
+    self._method = method
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def __call__(
+      self, request_iterator, timeout=None, metadata=None, credentials=None):
+    deadline, deadline_timespec = _deadline(timeout)
+    state = _RPCState(_STREAM_STREAM_INITIAL_DUE, None, None, None, None)
+    call = self._create_managed_call(
+        None, 0, self._method, None, deadline_timespec)
+    if credentials is not None:
+      call.set_credentials(credentials._credentials)
+    event_handler = _event_handler(state, call, self._response_deserializer)
+    with state.condition:
+      call.start_batch(
+          cygrpc.Operations(
+              (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)),
+          event_handler)
+      operations = (
+          cygrpc.operation_send_initial_metadata(
+              _common.cygrpc_metadata(metadata), _EMPTY_FLAGS),
+          cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+      )
+      call.start_batch(cygrpc.Operations(operations), event_handler)
+      _consume_request_iterator(
+          request_iterator, state, call, self._request_serializer)
+    return _Rendezvous(state, call, self._response_deserializer, deadline)
+
+
+class _ChannelCallState(object):
+
+  def __init__(self, channel):
+    self.lock = threading.Lock()
+    self.channel = channel
+    self.completion_queue = cygrpc.CompletionQueue()
+    self.managed_calls = None
+
+
+def _run_channel_spin_thread(state):
+  def channel_spin():
+    while True:
+      event = state.completion_queue.poll()
+      completed_call = event.tag(event)
+      if completed_call is not None:
+        with state.lock:
+          state.managed_calls.remove(completed_call)
+          if not state.managed_calls:
+            state.managed_calls = None
+            return
+
+  def stop_channel_spin(timeout):
+    with state.lock:
+      if state.managed_calls is not None:
+        for call in state.managed_calls:
+          call.cancel()
+
+  channel_spin_thread = _common.CleanupThread(
+      stop_channel_spin, target=channel_spin)
+  channel_spin_thread.start()
+
+
+def _create_channel_managed_call(state):
+  def create_channel_managed_call(parent, flags, method, host, deadline):
+    """Creates a managed cygrpc.Call.
+
+    Callers of this function must conduct at least one operation on the returned
+    call. The tags associated with operations conducted on the returned call
+    must be no-argument callables that return None to indicate that this channel
+    should continue polling for events associated with the call and return the
+    call itself to indicate that no more events associated with the call will be
+    generated.
+
+    Args:
+      parent: A cygrpc.Call to be used as the parent of the created call.
+      flags: An integer bitfield of call flags.
+      method: The RPC method.
+      host: A host string for the created call.
+      deadline: A cygrpc.Timespec to be the deadline of the created call.
+
+    Returns:
+      A cygrpc.Call with which to conduct an RPC.
+    """
+    with state.lock:
+      call = state.channel.create_call(
+          parent, flags, state.completion_queue, method, host, deadline)
+      if state.managed_calls is None:
+        state.managed_calls = set((call,))
+        _run_channel_spin_thread(state)
+      else:
+        state.managed_calls.add(call)
+      return call
+  return create_channel_managed_call
+
+
+class _ChannelConnectivityState(object):
+
+  def __init__(self, channel):
+    self.lock = threading.Lock()
+    self.channel = channel
+    self.polling = False
+    self.connectivity = None
+    self.try_to_connect = False
+    self.callbacks_and_connectivities = []
+    self.delivering = False
+
+
+def _deliveries(state):
+  callbacks_needing_update = []
+  for callback_and_connectivity in state.callbacks_and_connectivities:
+    callback, callback_connectivity, = callback_and_connectivity
+    if callback_connectivity is not state.connectivity:
+      callbacks_needing_update.append(callback)
+      callback_and_connectivity[1] = state.connectivity
+  return callbacks_needing_update
+
+
+def _deliver(state, initial_connectivity, initial_callbacks):
+  connectivity = initial_connectivity
+  callbacks = initial_callbacks
+  while True:
+    for callback in callbacks:
+      callable_util.call_logging_exceptions(
+          callback, _CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE,
+          connectivity)
+    with state.lock:
+      callbacks = _deliveries(state)
+      if callbacks:
+        connectivity = state.connectivity
+      else:
+        state.delivering = False
+        return
+
+
+def _spawn_delivery(state, callbacks):
+  delivering_thread = threading.Thread(
+      target=_deliver, args=(state, state.connectivity, callbacks,))
+  delivering_thread.start()
+  state.delivering = True
+
+
+# NOTE(https://github.com/grpc/grpc/issues/3064): We'd rather not poll.
+def _poll_connectivity(state, channel, initial_try_to_connect):
+  try_to_connect = initial_try_to_connect
+  connectivity = channel.check_connectivity_state(try_to_connect)
+  with state.lock:
+    state.connectivity = (
+        _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[
+            connectivity])
+    callbacks = tuple(
+        callback for callback, unused_but_known_to_be_none_connectivity
+        in state.callbacks_and_connectivities)
+    for callback_and_connectivity in state.callbacks_and_connectivities:
+      callback_and_connectivity[1] = state.connectivity
+    if callbacks:
+      _spawn_delivery(state, callbacks)
+  completion_queue = cygrpc.CompletionQueue()
+  while True:
+    channel.watch_connectivity_state(
+        connectivity, cygrpc.Timespec(time.time() + 0.2),
+        completion_queue, None)
+    event = completion_queue.poll()
+    with state.lock:
+      if not state.callbacks_and_connectivities and not state.try_to_connect:
+        state.polling = False
+        state.connectivity = None
+        break
+      try_to_connect = state.try_to_connect
+      state.try_to_connect = False
+    if event.success or try_to_connect:
+      connectivity = channel.check_connectivity_state(try_to_connect)
+      with state.lock:
+        state.connectivity = (
+            _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[
+                connectivity])
+        if not state.delivering:
+          callbacks = _deliveries(state)
+          if callbacks:
+            _spawn_delivery(state, callbacks)
+
+
+def _moot(state):
+  with state.lock:
+    del state.callbacks_and_connectivities[:]
+
+
+def _subscribe(state, callback, try_to_connect):
+  with state.lock:
+    if not state.callbacks_and_connectivities and not state.polling:
+      def cancel_all_subscriptions(timeout):
+        _moot(state)
+      polling_thread = _common.CleanupThread(
+          cancel_all_subscriptions, target=_poll_connectivity,
+          args=(state, state.channel, bool(try_to_connect)))
+      polling_thread.start()
+      state.polling = True
+      state.callbacks_and_connectivities.append([callback, None])
+    elif not state.delivering and state.connectivity is not None:
+      _spawn_delivery(state, (callback,))
+      state.try_to_connect |= bool(try_to_connect)
+      state.callbacks_and_connectivities.append(
+          [callback, state.connectivity])
+    else:
+      state.try_to_connect |= bool(try_to_connect)
+      state.callbacks_and_connectivities.append([callback, None])
+
+
+def _unsubscribe(state, callback):
+  with state.lock:
+    for index, (subscribed_callback, unused_connectivity) in enumerate(
+        state.callbacks_and_connectivities):
+      if callback == subscribed_callback:
+        state.callbacks_and_connectivities.pop(index)
+        break
+
+
+def _options(options):
+  if options is None:
+    pairs = ((cygrpc.ChannelArgKey.primary_user_agent_string, _USER_AGENT),)
+  else:
+    pairs = list(options) + [
+        (cygrpc.ChannelArgKey.primary_user_agent_string, _USER_AGENT)]
+  encoded_pairs = [
+      (_common.encode(arg_name), arg_value) if isinstance(arg_value, int)
+      else (_common.encode(arg_name), _common.encode(arg_value))
+      for arg_name, arg_value in pairs]
+  return cygrpc.ChannelArgs([
+      cygrpc.ChannelArg(arg_name, arg_value)
+      for arg_name, arg_value in encoded_pairs])
+
+
+class Channel(grpc.Channel):
+
+  def __init__(self, target, options, credentials):
+    """Constructor.
+
+    Args:
+      target: The target to which to connect.
+      options: Configuration options for the channel.
+      credentials: A cygrpc.ChannelCredentials or None.
+    """
+    self._channel = cygrpc.Channel(
+        _common.encode(target), _options(options), credentials)
+    self._call_state = _ChannelCallState(self._channel)
+    self._connectivity_state = _ChannelConnectivityState(self._channel)
+
+  def subscribe(self, callback, try_to_connect=None):
+    _subscribe(self._connectivity_state, callback, try_to_connect)
+
+  def unsubscribe(self, callback):
+    _unsubscribe(self._connectivity_state, callback)
+
+  def unary_unary(
+      self, method, request_serializer=None, response_deserializer=None):
+    return _UnaryUnaryMultiCallable(
+        self._channel, _create_channel_managed_call(self._call_state),
+        _common.encode(method), request_serializer, response_deserializer)
+
+  def unary_stream(
+      self, method, request_serializer=None, response_deserializer=None):
+    return _UnaryStreamMultiCallable(
+        self._channel, _create_channel_managed_call(self._call_state),
+        _common.encode(method), request_serializer, response_deserializer)
+
+  def stream_unary(
+      self, method, request_serializer=None, response_deserializer=None):
+    return _StreamUnaryMultiCallable(
+        self._channel, _create_channel_managed_call(self._call_state),
+        _common.encode(method), request_serializer, response_deserializer)
+
+  def stream_stream(
+      self, method, request_serializer=None, response_deserializer=None):
+    return _StreamStreamMultiCallable(
+        self._channel, _create_channel_managed_call(self._call_state),
+        _common.encode(method), request_serializer, response_deserializer)
+
+  def __del__(self):
+    _moot(self._connectivity_state)
diff --git a/src/python/grpcio/grpc/_common.py b/src/python/grpcio/grpc/_common.py
new file mode 100644
index 0000000000..4d7d521419
--- /dev/null
+++ b/src/python/grpcio/grpc/_common.py
@@ -0,0 +1,173 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Shared implementation."""
+
+import logging
+import threading
+import time
+
+import six
+
+import grpc
+from grpc._cython import cygrpc
+
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY = {
+    cygrpc.ConnectivityState.idle: grpc.ChannelConnectivity.IDLE,
+    cygrpc.ConnectivityState.connecting: grpc.ChannelConnectivity.CONNECTING,
+    cygrpc.ConnectivityState.ready: grpc.ChannelConnectivity.READY,
+    cygrpc.ConnectivityState.transient_failure:
+        grpc.ChannelConnectivity.TRANSIENT_FAILURE,
+    cygrpc.ConnectivityState.shutdown:
+        grpc.ChannelConnectivity.SHUTDOWN,
+}
+
+CYGRPC_STATUS_CODE_TO_STATUS_CODE = {
+    cygrpc.StatusCode.ok: grpc.StatusCode.OK,
+    cygrpc.StatusCode.cancelled: grpc.StatusCode.CANCELLED,
+    cygrpc.StatusCode.unknown: grpc.StatusCode.UNKNOWN,
+    cygrpc.StatusCode.invalid_argument: grpc.StatusCode.INVALID_ARGUMENT,
+    cygrpc.StatusCode.deadline_exceeded: grpc.StatusCode.DEADLINE_EXCEEDED,
+    cygrpc.StatusCode.not_found: grpc.StatusCode.NOT_FOUND,
+    cygrpc.StatusCode.already_exists: grpc.StatusCode.ALREADY_EXISTS,
+    cygrpc.StatusCode.permission_denied: grpc.StatusCode.PERMISSION_DENIED,
+    cygrpc.StatusCode.unauthenticated: grpc.StatusCode.UNAUTHENTICATED,
+    cygrpc.StatusCode.resource_exhausted: grpc.StatusCode.RESOURCE_EXHAUSTED,
+    cygrpc.StatusCode.failed_precondition: grpc.StatusCode.FAILED_PRECONDITION,
+    cygrpc.StatusCode.aborted: grpc.StatusCode.ABORTED,
+    cygrpc.StatusCode.out_of_range: grpc.StatusCode.OUT_OF_RANGE,
+    cygrpc.StatusCode.unimplemented: grpc.StatusCode.UNIMPLEMENTED,
+    cygrpc.StatusCode.internal: grpc.StatusCode.INTERNAL,
+    cygrpc.StatusCode.unavailable: grpc.StatusCode.UNAVAILABLE,
+    cygrpc.StatusCode.data_loss: grpc.StatusCode.DATA_LOSS,
+}
+STATUS_CODE_TO_CYGRPC_STATUS_CODE = {
+    grpc_code: cygrpc_code
+    for cygrpc_code, grpc_code in six.iteritems(
+        CYGRPC_STATUS_CODE_TO_STATUS_CODE)
+}
+
+
+def encode(s):
+  if isinstance(s, bytes):
+    return s
+  else:
+    return s.encode('ascii')
+
+
+def decode(b):
+  if isinstance(b, str):
+    return b
+  else:
+    try:
+      return b.decode('utf8')
+    except UnicodeDecodeError:
+      logging.exception('Invalid encoding on {}'.format(b))
+      return b.decode('latin1')
+
+
+def cygrpc_metadata(application_metadata):
+  return _EMPTY_METADATA if application_metadata is None else cygrpc.Metadata(
+      cygrpc.Metadatum(encode(key), encode(value))
+      for key, value in application_metadata)
+
+
+def application_metadata(cygrpc_metadata):
+  if cygrpc_metadata is None:
+    return ()
+  else:
+    return tuple(
+        (decode(key), value if key[-4:] == b'-bin' else decode(value))
+        for key, value in cygrpc_metadata)
+
+
+def _transform(message, transformer, exception_message):
+  if transformer is None:
+    return message
+  else:
+    try:
+      return transformer(message)
+    except Exception:  # pylint: disable=broad-except
+      logging.exception(exception_message)
+      return None
+
+
+def serialize(message, serializer):
+  return _transform(message, serializer, 'Exception serializing message!')
+
+
+def deserialize(serialized_message, deserializer):
+  return _transform(serialized_message, deserializer,
+                    'Exception deserializing message!')
+
+
+def fully_qualified_method(group, method):
+  return '/{}/{}'.format(group, method)
+
+
+class CleanupThread(threading.Thread):
+  """A threading.Thread subclass supporting custom behavior on join().
+
+  On Python Interpreter exit, Python will attempt to join outstanding threads
+  prior to garbage collection.  We may need to do additional cleanup, and
+  we accomplish this by overriding the join() method.
+  """
+
+  def __init__(self, behavior, group=None, target=None, name=None,
+               args=(), kwargs={}):
+    """Constructor.
+
+    Args:
+      behavior (function): Function called on join() with a single
+          argument, timeout, indicating the maximum duration of
+          `behavior`, or None indicating `behavior` has no deadline.
+          `behavior` must be idempotent.
+      group (None): should be None.  Reseved for future extensions
+          when ThreadGroup is implemented.
+      target (function): The function to invoke when this thread is
+          run.  Defaults to None.
+      name (str): The name of this thread.  Defaults to None.
+        args (tuple[object]): A tuple of arguments to pass to `target`.
+      kwargs (dict[str,object]): A dictionary of keyword arguments to
+           pass to `target`.
+    """
+    super(CleanupThread, self).__init__(group=group, target=target,
+                                        name=name, args=args, kwargs=kwargs)
+    self._behavior = behavior
+
+  def join(self, timeout=None):
+    start_time = time.time()
+    self._behavior(timeout)
+    end_time = time.time()
+    if timeout is not None:
+      timeout -= end_time - start_time
+      timeout = max(timeout, 0)
+    super(CleanupThread, self).join(timeout)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
index 1bfe6344e0..a09bbc75fe 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
@@ -55,6 +55,7 @@ cdef class Call:
   def cancel(
       self, grpc_status_code error_code=GRPC_STATUS__DO_NOT_USE,
       details=None):
+    details = str_to_bytes(details)
     if not self.is_valid:
       raise ValueError("invalid call object cannot be used from Python")
     if (details is None) != (error_code == GRPC_STATUS__DO_NOT_USE):
@@ -63,12 +64,6 @@ cdef class Call:
     cdef grpc_call_error result
     cdef char *c_details = NULL
     if error_code != GRPC_STATUS__DO_NOT_USE:
-      if isinstance(details, bytes):
-        pass
-      elif isinstance(details, basestring):
-        details = details.encode()
-      else:
-        raise TypeError("expected details to be str or bytes")
       self.references.append(details)
       c_details = details
       with nogil:
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
index c26bc083cf..1406696510 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
@@ -32,7 +32,7 @@ cimport cpython
 
 cdef class Channel:
 
-  def __cinit__(self, target, ChannelArgs arguments=None,
+  def __cinit__(self, bytes target, ChannelArgs arguments=None,
                 ChannelCredentials channel_credentials=None):
     cdef grpc_channel_args *c_arguments = NULL
     cdef char *c_target = NULL
@@ -40,12 +40,6 @@ cdef class Channel:
     self.references = []
     if arguments is not None:
       c_arguments = &arguments.c_args
-    if isinstance(target, bytes):
-      pass
-    elif isinstance(target, basestring):
-      target = target.encode()
-    else:
-      raise TypeError("expected target to be str or bytes")
     c_target = target
     if channel_credentials is None:
       with nogil:
@@ -64,23 +58,10 @@ cdef class Channel:
                   method, host, Timespec deadline not None):
     if queue.is_shutting_down:
       raise ValueError("queue must not be shutting down or shutdown")
-    if isinstance(method, bytes):
-      pass
-    elif isinstance(method, basestring):
-      method = method.encode()
-    else:
-      raise TypeError("expected method to be str or bytes")
     cdef char *method_c_string = method
     cdef char *host_c_string = NULL
-    if host is None:
-      pass
-    elif isinstance(host, bytes):
+    if host is not None:
       host_c_string = host
-    elif isinstance(host, basestring):
-      host = host.encode()
-      host_c_string = host
-    else:
-      raise TypeError("expected host to be str, bytes, or None")
     cdef Call operation_call = Call()
     operation_call.references = [self, method, host, queue]
     cdef grpc_call *parent_call = NULL
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi
index a67c963684..01089c3dc0 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi
@@ -31,9 +31,6 @@
 cdef class CompletionQueue:
 
   cdef grpc_completion_queue *c_completion_queue
-  cdef object pluck_condition
-  cdef int num_plucking
-  cdef int num_polling
   cdef bint is_shutting_down
   cdef bint is_shutdown
 
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
index cdae39d519..90266516fe 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
@@ -32,6 +32,8 @@ cimport cpython
 import threading
 import time
 
+cdef int _INTERRUPT_CHECK_PERIOD_MS = 200
+
 
 cdef class CompletionQueue:
 
@@ -40,9 +42,6 @@ cdef class CompletionQueue:
       self.c_completion_queue = grpc_completion_queue_create(NULL)
     self.is_shutting_down = False
     self.is_shutdown = False
-    self.pluck_condition = threading.Condition()
-    self.num_plucking = 0
-    self.num_polling = 0
 
   cdef _interpret_event(self, grpc_event event):
     cdef OperationTag tag = None
@@ -83,45 +82,27 @@ cdef class CompletionQueue:
   def poll(self, Timespec deadline=None):
     # We name this 'poll' to avoid problems with CPython's expectations for
     # 'special' methods (like next and __next__).
+    cdef gpr_timespec c_increment
+    cdef gpr_timespec c_timeout
     cdef gpr_timespec c_deadline
     with nogil:
+      c_increment = gpr_time_from_millis(_INTERRUPT_CHECK_PERIOD_MS, GPR_TIMESPAN)
       c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME)
-    if deadline is not None:
-      c_deadline = deadline.c_time
-    cdef grpc_event event
-
-    # Poll within a critical section to detect contention
-    with self.pluck_condition:
-      assert self.num_plucking == 0, 'cannot simultaneously pluck and poll'
-      self.num_polling += 1
-    with nogil:
-      event = grpc_completion_queue_next(
-          self.c_completion_queue, c_deadline, NULL)
-    with self.pluck_condition:
-      self.num_polling -= 1
-    return self._interpret_event(event)
-
-  def pluck(self, OperationTag tag, Timespec deadline=None):
-    # Plucking a 'None' tag is equivalent to passing control to GRPC core until
-    # the deadline.
-    cdef gpr_timespec c_deadline = gpr_inf_future(
-        GPR_CLOCK_REALTIME)
-    if deadline is not None:
-      c_deadline = deadline.c_time
-    cdef grpc_event event
-
-    # Pluck within a critical section to detect contention
-    with self.pluck_condition:
-      assert self.num_polling == 0, 'cannot simultaneously pluck and poll'
-      assert self.num_plucking < GRPC_MAX_COMPLETION_QUEUE_PLUCKERS, (
-          'cannot pluck more than {} times simultaneously'.format(
-              GRPC_MAX_COMPLETION_QUEUE_PLUCKERS))
-      self.num_plucking += 1
-    with nogil:
-      event = grpc_completion_queue_pluck(
-          self.c_completion_queue, <cpython.PyObject *>tag, c_deadline, NULL)
-    with self.pluck_condition:
-      self.num_plucking -= 1
+      if deadline is not None:
+        c_deadline = deadline.c_time
+      
+      while True:
+        c_timeout = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c_increment)
+        if gpr_time_cmp(c_timeout, c_deadline) > 0:
+          c_timeout = c_deadline
+        event = grpc_completion_queue_next(
+          self.c_completion_queue, c_timeout, NULL)
+        if event.type != GRPC_QUEUE_TIMEOUT or gpr_time_cmp(c_timeout, c_deadline) == 0:
+          break;
+        
+        # Handle any signals
+        with gil:
+          cpython.PyErr_CheckSignals()
     return self._interpret_event(event)
 
   def shutdown(self):
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi
index c793c8f5e5..d377a67520 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi
@@ -54,7 +54,7 @@ cdef class ServerCredentials:
 cdef class CredentialsMetadataPlugin:
 
   cdef object plugin_callback
-  cdef str plugin_name
+  cdef bytes plugin_name
 
   cdef grpc_metadata_credentials_plugin make_c_plugin(self)
 
@@ -68,4 +68,4 @@ cdef void plugin_get_metadata(
     void *state, grpc_auth_metadata_context context,
     grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil
 
-cdef void plugin_destroy_c_plugin_state(void *state)
+cdef void plugin_destroy_c_plugin_state(void *state) with gil
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
index 94d13b5999..b24e69243e 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
@@ -82,7 +82,7 @@ cdef class ServerCredentials:
 
 cdef class CredentialsMetadataPlugin:
 
-  def __cinit__(self, object plugin_callback, str name):
+  def __cinit__(self, object plugin_callback, bytes name):
     """
     Args:
       plugin_callback (callable): Callback accepting a service URL (str/bytes)
@@ -91,7 +91,7 @@ cdef class CredentialsMetadataPlugin:
         when called should be non-blocking and eventually call the callback
         object with the appropriate status code/details and metadata (if
         successful).
-      name (str): Plugin name.
+      name (bytes): Plugin name.
     """
     if not callable(plugin_callback):
       raise ValueError('expected callable plugin_callback')
@@ -129,7 +129,7 @@ cdef void plugin_get_metadata(
     grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil:
   def python_callback(
       Metadata metadata, grpc_status_code status,
-      const char *error_details):
+      bytes error_details):
     cb(user_data, metadata.c_metadata_array.metadata,
        metadata.c_metadata_array.count, status, error_details)
   cdef CredentialsMetadataPlugin self = <CredentialsMetadataPlugin>state
@@ -137,7 +137,7 @@ cdef void plugin_get_metadata(
   cy_context.context = context
   self.plugin_callback(cy_context, python_callback)
 
-cdef void plugin_destroy_c_plugin_state(void *state):
+cdef void plugin_destroy_c_plugin_state(void *state) with gil:
   cpython.Py_DECREF(<CredentialsMetadataPlugin>state)
 
 def channel_credentials_google_default():
@@ -148,14 +148,7 @@ def channel_credentials_google_default():
 
 def channel_credentials_ssl(pem_root_certificates,
                             SslPemKeyCertPair ssl_pem_key_cert_pair):
-  if pem_root_certificates is None:
-    pass
-  elif isinstance(pem_root_certificates, bytes):
-    pass
-  elif isinstance(pem_root_certificates, basestring):
-    pem_root_certificates = pem_root_certificates.encode()
-  else:
-    raise TypeError("expected str or bytes for pem_root_certificates")
+  pem_root_certificates = str_to_bytes(pem_root_certificates)
   cdef ChannelCredentials credentials = ChannelCredentials()
   cdef const char *c_pem_root_certificates = NULL
   if pem_root_certificates is not None:
@@ -207,12 +200,7 @@ def call_credentials_google_compute_engine():
 
 def call_credentials_service_account_jwt_access(
     json_key, Timespec token_lifetime not None):
-  if isinstance(json_key, bytes):
-    pass
-  elif isinstance(json_key, basestring):
-    json_key = json_key.encode()
-  else:
-    raise TypeError("expected json_key to be str or bytes")
+  json_key = str_to_bytes(json_key)
   cdef CallCredentials credentials = CallCredentials()
   cdef char *json_key_c_string = json_key
   with nogil:
@@ -223,12 +211,7 @@ def call_credentials_service_account_jwt_access(
   return credentials
 
 def call_credentials_google_refresh_token(json_refresh_token):
-  if isinstance(json_refresh_token, bytes):
-    pass
-  elif isinstance(json_refresh_token, basestring):
-    json_refresh_token = json_refresh_token.encode()
-  else:
-    raise TypeError("expected json_refresh_token to be str or bytes")
+  json_refresh_token = str_to_bytes(json_refresh_token)
   cdef CallCredentials credentials = CallCredentials()
   cdef char *json_refresh_token_c_string = json_refresh_token
   with nogil:
@@ -238,18 +221,8 @@ def call_credentials_google_refresh_token(json_refresh_token):
   return credentials
 
 def call_credentials_google_iam(authorization_token, authority_selector):
-  if isinstance(authorization_token, bytes):
-    pass
-  elif isinstance(authorization_token, basestring):
-    authorization_token = authorization_token.encode()
-  else:
-    raise TypeError("expected authorization_token to be str or bytes")
-  if isinstance(authority_selector, bytes):
-    pass
-  elif isinstance(authority_selector, basestring):
-    authority_selector = authority_selector.encode()
-  else:
-    raise TypeError("expected authority_selector to be str or bytes")
+  authorization_token = str_to_bytes(authorization_token)
+  authority_selector = str_to_bytes(authority_selector)
   cdef CallCredentials credentials = CallCredentials()
   cdef char *authorization_token_c_string = authorization_token
   cdef char *authority_selector_c_string = authority_selector
@@ -272,16 +245,10 @@ def call_credentials_metadata_plugin(CredentialsMetadataPlugin plugin):
 
 def server_credentials_ssl(pem_root_certs, pem_key_cert_pairs,
                            bint force_client_auth):
+  pem_root_certs = str_to_bytes(pem_root_certs)
   cdef char *c_pem_root_certs = NULL
-  if pem_root_certs is None:
-    pass
-  elif isinstance(pem_root_certs, bytes):
-    c_pem_root_certs = pem_root_certs
-  elif isinstance(pem_root_certs, basestring):
-    pem_root_certs = pem_root_certs.encode()
+  if pem_root_certs is not None: 
     c_pem_root_certs = pem_root_certs
-  else:
-    raise TypeError("expected pem_root_certs to be str or bytes")
   pem_key_cert_pairs = list(pem_key_cert_pairs)
   for pair in pem_key_cert_pairs:
     if not isinstance(pair, SslPemKeyCertPair):
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index 3d158a7707..f3b3d61273 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -37,6 +37,7 @@ cdef extern from "grpc/_cython/loader.h":
   ctypedef long int64_t
 
   int pygrpc_load_core(char*)
+  int pygrpc_initialize_core()
 
   void *gpr_malloc(size_t size) nogil
   void gpr_free(void *ptr) nogil
@@ -80,6 +81,12 @@ cdef extern from "grpc/_cython/loader.h":
   gpr_timespec gpr_convert_clock_type(gpr_timespec t,
                                       gpr_clock_type target_clock) nogil
 
+  gpr_timespec gpr_time_from_millis(int64_t ms, gpr_clock_type type) nogil
+
+  gpr_timespec gpr_time_add(gpr_timespec a, gpr_timespec b) nogil
+
+  int gpr_time_cmp(gpr_timespec a, gpr_timespec b) nogil
+  
   ctypedef enum grpc_status_code:
     GRPC_STATUS_OK
     GRPC_STATUS_CANCELLED
@@ -140,6 +147,9 @@ cdef extern from "grpc/_cython/loader.h":
   const char *GRPC_ARG_PRIMARY_USER_AGENT_STRING
   const char *GRPC_ARG_SECONDARY_USER_AGENT_STRING
   const char *GRPC_SSL_TARGET_NAME_OVERRIDE_ARG
+  const char *GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM
+  const char *GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL
+  const char *GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET
 
   const int GRPC_WRITE_BUFFER_HINT
   const int GRPC_WRITE_NO_COMPRESS
@@ -205,7 +215,7 @@ cdef extern from "grpc/_cython/loader.h":
     GRPC_CHANNEL_CONNECTING
     GRPC_CHANNEL_READY
     GRPC_CHANNEL_TRANSIENT_FAILURE
-    GRPC_CHANNEL_FATAL_FAILURE
+    GRPC_CHANNEL_SHUTDOWN
 
   ctypedef struct grpc_metadata:
     const char *key
@@ -333,6 +343,8 @@ cdef extern from "grpc/_cython/loader.h":
   void grpc_server_register_completion_queue(grpc_server *server,
                                              grpc_completion_queue *cq,
                                              void *reserved) nogil
+  void grpc_server_register_non_listening_completion_queue(
+      grpc_server *server, grpc_completion_queue *cq, void *reserved) nogil
   int grpc_server_add_insecure_http2_port(
       grpc_server *server, const char *addr) nogil
   void grpc_server_start(grpc_server *server) nogil
@@ -425,3 +437,38 @@ cdef extern from "grpc/_cython/loader.h":
 
   grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
       grpc_metadata_credentials_plugin plugin, void *reserved) nogil
+
+  ctypedef enum grpc_compression_algorithm:
+    GRPC_COMPRESS_NONE
+    GRPC_COMPRESS_DEFLATE
+    GRPC_COMPRESS_GZIP
+    GRPC_COMPRESS_ALGORITHMS_COUNT
+
+  ctypedef enum grpc_compression_level:
+    GRPC_COMPRESS_LEVEL_NONE
+    GRPC_COMPRESS_LEVEL_LOW
+    GRPC_COMPRESS_LEVEL_MED
+    GRPC_COMPRESS_LEVEL_HIGH
+    GRPC_COMPRESS_LEVEL_COUNT
+
+  ctypedef struct grpc_compression_options:
+    uint32_t enabled_algorithms_bitset
+    grpc_compression_algorithm default_compression_algorithm
+
+  int grpc_compression_algorithm_parse(
+      const char *name, size_t name_length,
+      grpc_compression_algorithm *algorithm) nogil
+  int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm,
+                                      char **name) nogil
+  grpc_compression_algorithm grpc_compression_algorithm_for_level(
+      grpc_compression_level level, uint32_t accepted_encodings) nogil
+  void grpc_compression_options_init(grpc_compression_options *opts) nogil
+  void grpc_compression_options_enable_algorithm(
+      grpc_compression_options *opts,
+      grpc_compression_algorithm algorithm) nogil
+  void grpc_compression_options_disable_algorithm(
+      grpc_compression_options *opts,
+      grpc_compression_algorithm algorithm) nogil
+  int grpc_compression_options_is_algorithm_enabled(
+      const grpc_compression_options *opts,
+      grpc_compression_algorithm algorithm) nogil
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi
new file mode 100644
index 0000000000..274f038e0a
--- /dev/null
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi
@@ -0,0 +1,39 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# This function will ascii encode unicode string inputs if neccesary.
+# In Python3, unicode strings are the default str type.
+cdef bytes str_to_bytes(object s):
+  if s is None or isinstance(s, bytes):
+    return s
+  elif isinstance(s, unicode):
+    return s.encode('ascii')
+  else:
+    raise TypeError('Expected bytes, str, or unicode, not {}'.format(type(s)))
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi
index 30397818a1..0474697af8 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi
@@ -124,3 +124,7 @@ cdef class Operations:
   cdef size_t c_nops
   cdef list operations
 
+
+cdef class CompressionOptions:
+
+  cdef grpc_compression_options c_options
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
index c2202bdab2..8e651e880f 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
@@ -33,7 +33,7 @@ class ConnectivityState:
   connecting = GRPC_CHANNEL_CONNECTING
   ready = GRPC_CHANNEL_READY
   transient_failure = GRPC_CHANNEL_TRANSIENT_FAILURE
-  fatal_failure = GRPC_CHANNEL_FATAL_FAILURE
+  shutdown = GRPC_CHANNEL_SHUTDOWN
 
 
 class ChannelArgKey:
@@ -103,6 +103,19 @@ class OperationType:
   receive_close_on_server = GRPC_OP_RECV_CLOSE_ON_SERVER
 
 
+class CompressionAlgorithm:
+  none = GRPC_COMPRESS_NONE
+  deflate = GRPC_COMPRESS_DEFLATE
+  gzip = GRPC_COMPRESS_GZIP
+
+
+class CompressionLevel:
+  none = GRPC_COMPRESS_LEVEL_NONE
+  low = GRPC_COMPRESS_LEVEL_LOW
+  medium = GRPC_COMPRESS_LEVEL_MED
+  high = GRPC_COMPRESS_LEVEL_HIGH
+
+
 cdef class Timespec:
 
   def __cinit__(self, time):
@@ -218,22 +231,10 @@ cdef class Event:
 
 cdef class ByteBuffer:
 
-  def __cinit__(self, data):
+  def __cinit__(self, bytes data):
     if data is None:
       self.c_byte_buffer = NULL
       return
-    if isinstance(data, bytes):
-      pass
-    elif isinstance(data, basestring):
-      data = data.encode()
-    elif isinstance(data, ByteBuffer):
-      data = (<ByteBuffer>data).bytes()
-      if data is None:
-        self.c_byte_buffer = NULL
-        return
-    else:
-      raise TypeError("expected value to be of type str, bytes, or "
-                      "ByteBuffer, not {}".format(type(data)))
 
     cdef char *c_data = data
     cdef gpr_slice data_slice
@@ -261,6 +262,7 @@ cdef class ByteBuffer:
           data_slice_length = gpr_slice_length(data_slice)
           with gil:
             result += (<char *>data_slice_pointer)[:data_slice_length]
+          gpr_slice_unref(data_slice)
       with nogil:
         grpc_byte_buffer_reader_destroy(&reader)
       return bytes(result)
@@ -287,48 +289,28 @@ cdef class ByteBuffer:
 
 cdef class SslPemKeyCertPair:
 
-  def __cinit__(self, private_key, certificate_chain):
-    if isinstance(private_key, bytes):
-      self.private_key = private_key
-    elif isinstance(private_key, basestring):
-      self.private_key = private_key.encode()
-    else:
-      raise TypeError("expected private_key to be of type str or bytes")
-    if isinstance(certificate_chain, bytes):
-      self.certificate_chain = certificate_chain
-    elif isinstance(certificate_chain, basestring):
-      self.certificate_chain = certificate_chain.encode()
-    else:
-      raise TypeError("expected certificate_chain to be of type str or bytes "
-                      "or int")
+  def __cinit__(self, bytes private_key, bytes certificate_chain):
+    self.private_key = private_key
+    self.certificate_chain = certificate_chain
     self.c_pair.private_key = self.private_key
     self.c_pair.certificate_chain = self.certificate_chain
 
 
 cdef class ChannelArg:
 
-  def __cinit__(self, key, value):
-    if isinstance(key, bytes):
-      self.key = key
-    elif isinstance(key, basestring):
-      self.key = key.encode()
-    else:
-      raise TypeError("expected key to be of type str or bytes")
-    if isinstance(value, bytes):
+  def __cinit__(self, bytes key, value):
+    self.key = key
+    self.c_arg.key = self.key
+    if isinstance(value, int):
       self.value = value
-      self.c_arg.type = GRPC_ARG_STRING
-      self.c_arg.value.string = self.value
-    elif isinstance(value, basestring):
-      self.value = value.encode()
-      self.c_arg.type = GRPC_ARG_STRING
-      self.c_arg.value.string = self.value
-    elif isinstance(value, int):
-      self.value = int(value)
       self.c_arg.type = GRPC_ARG_INTEGER
       self.c_arg.value.integer = self.value
+    elif isinstance(value, bytes):
+      self.value = value
+      self.c_arg.type = GRPC_ARG_STRING
+      self.c_arg.value.string = self.value
     else:
-      raise TypeError("expected value to be of type str or bytes or int")
-    self.c_arg.key = self.key
+      raise TypeError('Expected int or bytes, got {}'.format(type(value)))
 
 
 cdef class ChannelArgs:
@@ -360,19 +342,9 @@ cdef class ChannelArgs:
 
 cdef class Metadatum:
 
-  def __cinit__(self, key, value):
-    if isinstance(key, bytes):
-      self._key = key
-    elif isinstance(key, basestring):
-      self._key = key.encode()
-    else:
-      raise TypeError("expected key to be of type str or bytes")
-    if isinstance(value, bytes):
-      self._value = value
-    elif isinstance(value, basestring):
-      self._value = value.encode()
-    else:
-      raise TypeError("expected value to be of type str or bytes")
+  def __cinit__(self, bytes key, bytes value):
+    self._key = key
+    self._value = value
     self.c_metadata.key = self._key
     self.c_metadata.value = self._value
     self.c_metadata.value_length = len(self._value)
@@ -473,6 +445,10 @@ cdef class Operation:
     return self.c_op.type
 
   @property
+  def flags(self):
+    return self.c_op.flags
+
+  @property
   def has_status(self):
     return self.c_op.type == GRPC_OP_RECV_STATUS_ON_CLIENT
 
@@ -553,9 +529,10 @@ cdef class Operation:
       with nogil:
         gpr_free(self._received_status_details)
 
-def operation_send_initial_metadata(Metadata metadata):
+def operation_send_initial_metadata(Metadata metadata, int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_SEND_INITIAL_METADATA
+  op.c_op.flags = flags
   op.c_op.data.send_initial_metadata.count = metadata.c_metadata_array.count
   op.c_op.data.send_initial_metadata.metadata = (
       metadata.c_metadata_array.metadata)
@@ -563,31 +540,28 @@ def operation_send_initial_metadata(Metadata metadata):
   op.is_valid = True
   return op
 
-def operation_send_message(data):
+def operation_send_message(data, int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_SEND_MESSAGE
+  op.c_op.flags = flags
   byte_buffer = ByteBuffer(data)
   op.c_op.data.send_message = byte_buffer.c_byte_buffer
   op.references.append(byte_buffer)
   op.is_valid = True
   return op
 
-def operation_send_close_from_client():
+def operation_send_close_from_client(int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_SEND_CLOSE_FROM_CLIENT
+  op.c_op.flags = flags
   op.is_valid = True
   return op
 
 def operation_send_status_from_server(
-    Metadata metadata, grpc_status_code code, details):
-  if isinstance(details, bytes):
-    pass
-  elif isinstance(details, basestring):
-    details = details.encode()
-  else:
-    raise TypeError("expected a str or bytes object for details")
+    Metadata metadata, grpc_status_code code, bytes details, int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_SEND_STATUS_FROM_SERVER
+  op.c_op.flags = flags
   op.c_op.data.send_status_from_server.trailing_metadata_count = (
       metadata.c_metadata_array.count)
   op.c_op.data.send_status_from_server.trailing_metadata = (
@@ -599,18 +573,20 @@ def operation_send_status_from_server(
   op.is_valid = True
   return op
 
-def operation_receive_initial_metadata():
+def operation_receive_initial_metadata(int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_RECV_INITIAL_METADATA
+  op.c_op.flags = flags
   op._received_metadata = Metadata([])
   op.c_op.data.receive_initial_metadata = (
       &op._received_metadata.c_metadata_array)
   op.is_valid = True
   return op
 
-def operation_receive_message():
+def operation_receive_message(int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_RECV_MESSAGE
+  op.c_op.flags = flags
   op._received_message = ByteBuffer(None)
   # n.b. the c_op.data.receive_message field needs to be deleted by us,
   # anyway, so we just let that be handled by the ByteBuffer() we allocated
@@ -619,9 +595,10 @@ def operation_receive_message():
   op.is_valid = True
   return op
 
-def operation_receive_status_on_client():
+def operation_receive_status_on_client(int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_RECV_STATUS_ON_CLIENT
+  op.c_op.flags = flags
   op._received_metadata = Metadata([])
   op.c_op.data.receive_status_on_client.trailing_metadata = (
       &op._received_metadata.c_metadata_array)
@@ -634,9 +611,10 @@ def operation_receive_status_on_client():
   op.is_valid = True
   return op
 
-def operation_receive_close_on_server():
+def operation_receive_close_on_server(int flags):
   cdef Operation op = Operation()
   op.c_op.type = GRPC_OP_RECV_CLOSE_ON_SERVER
+  op.c_op.flags = flags
   op.c_op.data.receive_close_on_server.cancelled = &op._received_cancelled
   op.is_valid = True
   return op
@@ -692,3 +670,36 @@ cdef class Operations:
   def __iter__(self):
     return _OperationsIterator(self)
 
+
+cdef class CompressionOptions:
+
+  def __cinit__(self):
+    with nogil:
+      grpc_compression_options_init(&self.c_options)
+
+  def enable_algorithm(self, grpc_compression_algorithm algorithm):
+    with nogil:
+      grpc_compression_options_enable_algorithm(&self.c_options, algorithm)
+
+  def disable_algorithm(self, grpc_compression_algorithm algorithm):
+    with nogil:
+      grpc_compression_options_disable_algorithm(&self.c_options, algorithm)
+
+  def is_algorithm_enabled(self, grpc_compression_algorithm algorithm):
+    cdef int result
+    with nogil:
+      result = grpc_compression_options_is_algorithm_enabled(
+          &self.c_options, algorithm)
+    return result
+
+  def to_channel_arg(self):
+    return ChannelArg(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET,
+                      self.c_options.enabled_algorithms_bitset)
+
+
+def compression_algorithm_name(grpc_compression_algorithm algorithm):
+  cdef char* name
+  with nogil:
+    grpc_compression_algorithm_name(algorithm, &name)
+  # Let Cython do the right thing with string casting
+  return name
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
index 8419a59068..3e03b6efe1 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
@@ -81,25 +81,29 @@ cdef class Server:
           self.c_server, queue.c_completion_queue, NULL)
     self.registered_completion_queues.append(queue)
 
+  def register_non_listening_completion_queue(
+      self, CompletionQueue queue not None):
+    if self.is_started:
+      raise ValueError("cannot register completion queues after start")
+    with nogil:
+      grpc_server_register_non_listening_completion_queue(
+          self.c_server, queue.c_completion_queue, NULL)
+    self.registered_completion_queues.append(queue)
+
   def start(self):
     if self.is_started:
       raise ValueError("the server has already started")
     self.backup_shutdown_queue = CompletionQueue()
-    self.register_completion_queue(self.backup_shutdown_queue)
+    self.register_non_listening_completion_queue(self.backup_shutdown_queue)
     self.is_started = True
     with nogil:
       grpc_server_start(self.c_server)
     # Ensure the core has gotten a chance to do the start-up work
-    self.backup_shutdown_queue.pluck(None, Timespec(None))
+    self.backup_shutdown_queue.poll(Timespec(None))
 
-  def add_http2_port(self, address,
+  def add_http2_port(self, bytes address,
                      ServerCredentials server_credentials=None):
-    if isinstance(address, bytes):
-      pass
-    elif isinstance(address, basestring):
-      address = address.encode()
-    else:
-      raise TypeError("expected address to be a str or bytes")
+    address = str_to_bytes(address)
     self.references.append(address)
     cdef int result
     cdef char *address_c_string = address
@@ -169,4 +173,3 @@ cdef class Server:
           time.sleep(0)
       with nogil:
         grpc_server_destroy(self.c_server)
-
diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx
index 8823ea5ef5..7a8d0dd8a1 100644
--- a/src/python/grpcio/grpc/_cython/cygrpc.pyx
+++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx
@@ -35,6 +35,7 @@ import sys
 
 # TODO(atash): figure out why the coverage tool gets confused about the Cython
 # coverage plugin when the following files don't have a '.pxi' suffix.
+include "grpc/_cython/_cygrpc/grpc_string.pyx.pxi"
 include "grpc/_cython/_cygrpc/call.pyx.pxi"
 include "grpc/_cython/_cygrpc/channel.pyx.pxi"
 include "grpc/_cython/_cygrpc/credentials.pyx.pxi"
@@ -44,30 +45,22 @@ include "grpc/_cython/_cygrpc/security.pyx.pxi"
 include "grpc/_cython/_cygrpc/server.pyx.pxi"
 
 #
-# Global state
+# initialize gRPC
 #
 
-cdef class _ModuleState:
 
-  cdef bint is_loaded
+def _initialize():
+  if 'win32' in sys.platform:
+    filename = pkg_resources.resource_filename(
+        'grpc._cython', '_windows/grpc_c.64.python')
+    if not isinstance(filename, bytes):
+      filename = filename.encode()
+    if not pygrpc_load_core(filename):
+      raise ImportError('failed to load core gRPC library')
+  if not pygrpc_initialize_core():
+    raise ImportError('failed to initialize core gRPC library')
 
-  def __cinit__(self):
-    if 'win32' in sys.platform:
-      filename = pkg_resources.resource_filename(
-          'grpc._cython', '_windows/grpc_c.64.python')
-      if not pygrpc_load_core(filename):
-        raise ImportError('failed to load core gRPC library')
-    with nogil:
-      grpc_init()
-    self.is_loaded = True
-    with nogil:
-      grpc_set_ssl_roots_override_callback(
+  grpc_set_ssl_roots_override_callback(
           <grpc_ssl_roots_override_callback>ssl_roots_override_callback)
 
-  def __dealloc__(self):
-    if self.is_loaded:
-      with nogil:
-        grpc_shutdown()
-
-_module_state = _ModuleState()
-
+_initialize()
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c
index f0a40dbb35..d78ec2f66e 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.c
+++ b/src/python/grpcio/grpc/_cython/imports.generated.c
@@ -35,7 +35,7 @@
 
 #include "imports.generated.h"
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 census_initialize_type census_initialize_import;
 census_shutdown_type census_shutdown_import;
@@ -115,6 +115,7 @@ grpc_server_register_method_type grpc_server_register_method_import;
 grpc_server_request_registered_call_type grpc_server_request_registered_call_import;
 grpc_server_create_type grpc_server_create_import;
 grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
+grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
 grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 grpc_server_start_type grpc_server_start_import;
 grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import;
@@ -125,6 +126,9 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import;
 grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import;
 grpc_is_binary_header_type grpc_is_binary_header_import;
 grpc_call_error_to_string_type grpc_call_error_to_string_import;
+grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import;
+grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import;
+grpc_use_signal_type grpc_use_signal_import;
 grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import;
 grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import;
 grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import;
@@ -257,6 +261,8 @@ gpr_avl_unref_type gpr_avl_unref_import;
 gpr_avl_add_type gpr_avl_add_import;
 gpr_avl_remove_type gpr_avl_remove_import;
 gpr_avl_get_type gpr_avl_get_import;
+gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
+gpr_avl_is_empty_type gpr_avl_is_empty_import;
 gpr_cmdline_create_type gpr_cmdline_create_import;
 gpr_cmdline_add_int_type gpr_cmdline_add_int_import;
 gpr_cmdline_add_flag_type gpr_cmdline_add_flag_import;
@@ -385,6 +391,7 @@ void pygrpc_load_imports(HMODULE library) {
   grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call");
   grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create");
   grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue");
+  grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue");
   grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port");
   grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start");
   grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify");
@@ -395,6 +402,9 @@ void pygrpc_load_imports(HMODULE library) {
   grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal");
   grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header");
   grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string");
+  grpc_insecure_channel_create_from_fd_import = (grpc_insecure_channel_create_from_fd_type) GetProcAddress(library, "grpc_insecure_channel_create_from_fd");
+  grpc_server_add_insecure_channel_from_fd_import = (grpc_server_add_insecure_channel_from_fd_type) GetProcAddress(library, "grpc_server_add_insecure_channel_from_fd");
+  grpc_use_signal_import = (grpc_use_signal_type) GetProcAddress(library, "grpc_use_signal");
   grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next");
   grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator");
   grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity");
@@ -527,6 +537,8 @@ void pygrpc_load_imports(HMODULE library) {
   gpr_avl_add_import = (gpr_avl_add_type) GetProcAddress(library, "gpr_avl_add");
   gpr_avl_remove_import = (gpr_avl_remove_type) GetProcAddress(library, "gpr_avl_remove");
   gpr_avl_get_import = (gpr_avl_get_type) GetProcAddress(library, "gpr_avl_get");
+  gpr_avl_maybe_get_import = (gpr_avl_maybe_get_type) GetProcAddress(library, "gpr_avl_maybe_get");
+  gpr_avl_is_empty_import = (gpr_avl_is_empty_type) GetProcAddress(library, "gpr_avl_is_empty");
   gpr_cmdline_create_import = (gpr_cmdline_create_type) GetProcAddress(library, "gpr_cmdline_create");
   gpr_cmdline_add_int_import = (gpr_cmdline_add_int_type) GetProcAddress(library, "gpr_cmdline_add_int");
   gpr_cmdline_add_flag_import = (gpr_cmdline_add_flag_type) GetProcAddress(library, "gpr_cmdline_add_flag");
@@ -577,4 +589,4 @@ void pygrpc_load_imports(HMODULE library) {
 }
 #endif  /* __cpluslus */
 
-#endif /* !GPR_WIN32 */
+#endif /* !GPR_WINDOWS */
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
index d5e810b7cf..b3e341fe25 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.h
+++ b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -36,13 +36,14 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include <windows.h>
 
 #include <grpc/census.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
 #include <grpc/grpc_security.h>
 #include <grpc/impl/codegen/alloc.h>
 #include <grpc/impl/codegen/byte_buffer.h>
@@ -56,7 +57,7 @@
 #include <grpc/support/cpu.h>
 #include <grpc/support/histogram.h>
 #include <grpc/support/host_port.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
 #include <grpc/support/thd.h>
@@ -295,6 +296,9 @@ extern grpc_server_create_type grpc_server_create_import;
 typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved);
 extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
 #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import
+typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved);
+extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
+#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import
 typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr);
 extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import
@@ -325,6 +329,15 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import;
 typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error);
 extern grpc_call_error_to_string_type grpc_call_error_to_string_import;
 #define grpc_call_error_to_string grpc_call_error_to_string_import
+typedef grpc_channel *(*grpc_insecure_channel_create_from_fd_type)(const char *target, int fd, const grpc_channel_args *args);
+extern grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import;
+#define grpc_insecure_channel_create_from_fd grpc_insecure_channel_create_from_fd_import
+typedef void(*grpc_server_add_insecure_channel_from_fd_type)(grpc_server *server, grpc_completion_queue *cq, int fd);
+extern grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import;
+#define grpc_server_add_insecure_channel_from_fd grpc_server_add_insecure_channel_from_fd_import
+typedef void(*grpc_use_signal_type)(int signum);
+extern grpc_use_signal_type grpc_use_signal_import;
+#define grpc_use_signal grpc_use_signal_import
 typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it);
 extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import;
 #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import
@@ -472,7 +485,7 @@ extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_impo
 typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader);
 extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
 #define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import
-typedef void(*gpr_log_type)(const char *file, int line, gpr_log_severity severity, const char *format, ...);
+typedef void(*gpr_log_type)(const char *file, int line, gpr_log_severity severity, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(4, 5);
 extern gpr_log_type gpr_log_import;
 #define gpr_log gpr_log_import
 typedef void(*gpr_log_message_type)(const char *file, int line, gpr_log_severity severity, const char *message);
@@ -721,6 +734,12 @@ extern gpr_avl_remove_type gpr_avl_remove_import;
 typedef void *(*gpr_avl_get_type)(gpr_avl avl, void *key);
 extern gpr_avl_get_type gpr_avl_get_import;
 #define gpr_avl_get gpr_avl_get_import
+typedef int(*gpr_avl_maybe_get_type)(gpr_avl avl, void *key, void **value);
+extern gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
+#define gpr_avl_maybe_get gpr_avl_maybe_get_import
+typedef int(*gpr_avl_is_empty_type)(gpr_avl avl);
+extern gpr_avl_is_empty_type gpr_avl_is_empty_import;
+#define gpr_avl_is_empty gpr_avl_is_empty_import
 typedef gpr_cmdline *(*gpr_cmdline_create_type)(const char *description);
 extern gpr_cmdline_create_type gpr_cmdline_create_import;
 #define gpr_cmdline_create gpr_cmdline_create_import
@@ -811,7 +830,7 @@ extern gpr_format_message_type gpr_format_message_import;
 typedef char *(*gpr_strdup_type)(const char *src);
 extern gpr_strdup_type gpr_strdup_import;
 #define gpr_strdup gpr_strdup_import
-typedef int(*gpr_asprintf_type)(char **strp, const char *format, ...);
+typedef int(*gpr_asprintf_type)(char **strp, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(2, 3);
 extern gpr_asprintf_type gpr_asprintf_import;
 #define gpr_asprintf gpr_asprintf_import
 typedef const char *(*gpr_subprocess_binary_extension_type)();
@@ -864,17 +883,18 @@ void pygrpc_load_imports(HMODULE library);
 }
 #endif  /* __cpluslus */
 
-#else /* !GPR_WIN32 */
+#else /* !GPR_WINDOWS */
 
-#include <grpc/support/alloc.h>
-#include <grpc/support/slice.h>
-#include <grpc/support/time.h>
-#include <grpc/status.h>
 #include <grpc/byte_buffer.h>
 #include <grpc/byte_buffer_reader.h>
+#include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/slice.h>
+#include <grpc/support/time.h>
+#include <grpc/status.h>
 
-#endif /* !GPR_WIN32 */
+#endif /* !GPR_WINDOWS */
 
 #endif
diff --git a/src/python/grpcio/grpc/_cython/loader.c b/src/python/grpcio/grpc/_cython/loader.c
index 3b72806ea1..86b70dbb02 100644
--- a/src/python/grpcio/grpc/_cython/loader.c
+++ b/src/python/grpcio/grpc/_cython/loader.c
@@ -31,13 +31,14 @@
  *
  */
 
+#include <Python.h>
 #include "loader.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif  /* __cpluslus  */
 
-#if GPR_WIN32
+#if GPR_WINDOWS
 
 int pygrpc_load_core(char *path) {
   HMODULE grpc_c;
@@ -60,7 +61,13 @@ int pygrpc_load_core(char *path) {
 
 int pygrpc_load_core(char *path) { return 1; }
 
-#endif  /* !GPR_WIN32 */
+#endif  /* !GPR_WINDOWS */
+
+// Cython doesn't have Py_AtExit bindings, so we call the C_API directly
+int pygrpc_initialize_core(void) {
+  grpc_init();
+  return Py_AtExit(grpc_shutdown) < 0 ? 0 : 1;
+}
 
 #ifdef __cplusplus
 }
diff --git a/src/python/grpcio/grpc/_cython/loader.h b/src/python/grpcio/grpc/_cython/loader.h
index 3b8796d39f..eb4b1a1b01 100644
--- a/src/python/grpcio/grpc/_cython/loader.h
+++ b/src/python/grpcio/grpc/_cython/loader.h
@@ -46,6 +46,11 @@ extern "C" {
 /* Attempts to load the core if necessary, and return non-zero upon succes. */
 int pygrpc_load_core(char *path);
 
+/* Initializes grpc and registers grpc_shutdown() to be called right before
+ * interpreter exit.  Returns non-zero upon success.
+ */
+int pygrpc_initialize_core(void);
+
 #ifdef __cplusplus
 }
 #endif  /* __cpluslus */
diff --git a/src/python/grpcio/grpc/_links/service.py b/src/python/grpcio/grpc/_links/service.py
index 11310e2240..5fc4994ca0 100644
--- a/src/python/grpcio/grpc/_links/service.py
+++ b/src/python/grpcio/grpc/_links/service.py
@@ -33,6 +33,7 @@ import abc
 import enum
 import logging
 import threading
+import six
 import time
 
 from grpc._adapter import _intermediary_low
@@ -177,7 +178,10 @@ class _Kernel(object):
     call = service_acceptance.call
     call.accept(self._completion_queue, call)
     try:
-      group, method = service_acceptance.method.split(b'/')[1:3]
+      service_method = service_acceptance.method
+      if six.PY3:
+          service_method = service_method.decode('latin1')
+      group, method = service_method.split('/')[1:3]
     except ValueError:
       logging.info('Illegal path "%s"!', service_acceptance.method)
       return
diff --git a/src/python/grpcio/grpc/_plugin_wrapping.py b/src/python/grpcio/grpc/_plugin_wrapping.py
new file mode 100644
index 0000000000..7cb5218c22
--- /dev/null
+++ b/src/python/grpcio/grpc/_plugin_wrapping.py
@@ -0,0 +1,123 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import collections
+import threading
+
+import grpc
+from grpc import _common
+from grpc._cython import cygrpc
+
+
+class AuthMetadataContext(
+    collections.namedtuple(
+        'AuthMetadataContext', ('service_url', 'method_name',)),
+    grpc.AuthMetadataContext):
+  pass
+
+
+class AuthMetadataPluginCallback(grpc.AuthMetadataContext):
+
+  def __init__(self, callback):
+    self._callback = callback
+
+  def __call__(self, metadata, error):
+    self._callback(metadata, error)
+
+
+class _WrappedCygrpcCallback(object):
+
+  def __init__(self, cygrpc_callback):
+    self.is_called = False
+    self.error = None
+    self.is_called_lock = threading.Lock()
+    self.cygrpc_callback = cygrpc_callback
+
+  def _invoke_failure(self, error):
+    # TODO(atash) translate different Exception superclasses into different
+    # status codes.
+    self.cygrpc_callback(
+        _common.EMPTY_METADATA, cygrpc.StatusCode.internal,
+        _common.encode(str(error)))
+
+  def _invoke_success(self, metadata):
+    try:
+      cygrpc_metadata = _common.cygrpc_metadata(metadata)
+    except Exception as error:
+      self._invoke_failure(error)
+      return
+    self.cygrpc_callback(cygrpc_metadata, cygrpc.StatusCode.ok, b'')
+
+  def __call__(self, metadata, error):
+    with self.is_called_lock:
+      if self.is_called:
+        raise RuntimeError('callback should only ever be invoked once')
+      if self.error:
+        self._invoke_failure(self.error)
+        return
+      self.is_called = True
+    if error is None:
+      self._invoke_success(metadata)
+    else:
+      self._invoke_failure(error)
+
+  def notify_failure(self, error):
+    with self.is_called_lock:
+      if not self.is_called:
+        self.error = error
+
+
+class _WrappedPlugin(object):
+
+  def __init__(self, plugin):
+    self.plugin = plugin
+
+  def __call__(self, context, cygrpc_callback):
+    wrapped_cygrpc_callback = _WrappedCygrpcCallback(cygrpc_callback)
+    wrapped_context = AuthMetadataContext(
+        _common.decode(context.service_url), _common.decode(context.method_name))
+    try:
+      self.plugin(
+          wrapped_context, AuthMetadataPluginCallback(wrapped_cygrpc_callback))
+    except Exception as error:
+      wrapped_cygrpc_callback.notify_failure(error)
+      raise
+
+
+def call_credentials_metadata_plugin(plugin, name):
+  """
+  Args:
+    plugin: A callable accepting a grpc.AuthMetadataContext
+      object and a callback (itself accepting a list of metadata key/value
+      2-tuples and a None-able exception value). The callback must be eventually
+      called, but need not be called in plugin's invocation.
+      plugin's invocation must be non-blocking.
+  """
+  return cygrpc.call_credentials_metadata_plugin(
+      cygrpc.CredentialsMetadataPlugin(_WrappedPlugin(plugin), _common.encode(name)))
diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py
new file mode 100644
index 0000000000..f4c114056f
--- /dev/null
+++ b/src/python/grpcio/grpc/_server.py
@@ -0,0 +1,757 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Service-side implementation of gRPC Python."""
+
+import collections
+import enum
+import logging
+import threading
+import time
+
+import grpc
+from grpc import _common
+from grpc._cython import cygrpc
+from grpc.framework.foundation import callable_util
+
+_SHUTDOWN_TAG = 'shutdown'
+_REQUEST_CALL_TAG = 'request_call'
+
+_RECEIVE_CLOSE_ON_SERVER_TOKEN = 'receive_close_on_server'
+_SEND_INITIAL_METADATA_TOKEN = 'send_initial_metadata'
+_RECEIVE_MESSAGE_TOKEN = 'receive_message'
+_SEND_MESSAGE_TOKEN = 'send_message'
+_SEND_INITIAL_METADATA_AND_SEND_MESSAGE_TOKEN = (
+    'send_initial_metadata * send_message')
+_SEND_STATUS_FROM_SERVER_TOKEN = 'send_status_from_server'
+_SEND_INITIAL_METADATA_AND_SEND_STATUS_FROM_SERVER_TOKEN = (
+    'send_initial_metadata * send_status_from_server')
+
+_OPEN = 'open'
+_CLOSED = 'closed'
+_CANCELLED = 'cancelled'
+
+_EMPTY_FLAGS = 0
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+_UNEXPECTED_EXIT_SERVER_GRACE = 1.0
+
+
+def _serialized_request(request_event):
+  return request_event.batch_operations[0].received_message.bytes()
+
+
+def _application_code(code):
+  cygrpc_code = _common.STATUS_CODE_TO_CYGRPC_STATUS_CODE.get(code)
+  return cygrpc.StatusCode.unknown if cygrpc_code is None else cygrpc_code
+
+
+def _completion_code(state):
+  if state.code is None:
+    return cygrpc.StatusCode.ok
+  else:
+    return _application_code(state.code)
+
+
+def _abortion_code(state, code):
+  if state.code is None:
+    return code
+  else:
+    return _application_code(state.code)
+
+
+def _details(state):
+  return b'' if state.details is None else state.details
+
+
+class _HandlerCallDetails(
+    collections.namedtuple(
+        '_HandlerCallDetails', ('method', 'invocation_metadata',)),
+    grpc.HandlerCallDetails):
+  pass
+
+
+class _RPCState(object):
+
+  def __init__(self):
+    self.condition = threading.Condition()
+    self.due = set()
+    self.request = None
+    self.client = _OPEN
+    self.initial_metadata_allowed = True
+    self.disable_next_compression = False
+    self.trailing_metadata = None
+    self.code = None
+    self.details = None
+    self.statused = False
+    self.rpc_errors = []
+    self.callbacks = []
+
+
+def _raise_rpc_error(state):
+  rpc_error = grpc.RpcError()
+  state.rpc_errors.append(rpc_error)
+  raise rpc_error
+
+
+def _possibly_finish_call(state, token):
+  state.due.remove(token)
+  if (state.client is _CANCELLED or state.statused) and not state.due:
+    callbacks = state.callbacks
+    state.callbacks = None
+    return state, callbacks
+  else:
+    return None, ()
+
+
+def _send_status_from_server(state, token):
+  def send_status_from_server(unused_send_status_from_server_event):
+    with state.condition:
+      return _possibly_finish_call(state, token)
+  return send_status_from_server
+
+
+def _abort(state, call, code, details):
+  if state.client is not _CANCELLED:
+    effective_code = _abortion_code(state, code)
+    effective_details = details if state.details is None else state.details
+    if state.initial_metadata_allowed:
+      operations = (
+          cygrpc.operation_send_initial_metadata(
+              _EMPTY_METADATA, _EMPTY_FLAGS),
+          cygrpc.operation_send_status_from_server(
+              _common.cygrpc_metadata(state.trailing_metadata), effective_code,
+              effective_details, _EMPTY_FLAGS),
+      )
+      token = _SEND_INITIAL_METADATA_AND_SEND_STATUS_FROM_SERVER_TOKEN
+    else:
+      operations = (
+          cygrpc.operation_send_status_from_server(
+              _common.cygrpc_metadata(state.trailing_metadata), effective_code,
+              effective_details, _EMPTY_FLAGS),
+      )
+      token = _SEND_STATUS_FROM_SERVER_TOKEN
+    call.start_batch(
+        cygrpc.Operations(operations),
+        _send_status_from_server(state, token))
+    state.statused = True
+    state.due.add(token)
+
+
+def _receive_close_on_server(state):
+  def receive_close_on_server(receive_close_on_server_event):
+    with state.condition:
+      if receive_close_on_server_event.batch_operations[0].received_cancelled:
+        state.client = _CANCELLED
+      elif state.client is _OPEN:
+        state.client = _CLOSED
+      state.condition.notify_all()
+      return _possibly_finish_call(state, _RECEIVE_CLOSE_ON_SERVER_TOKEN)
+  return receive_close_on_server
+
+
+def _receive_message(state, call, request_deserializer):
+  def receive_message(receive_message_event):
+    serialized_request = _serialized_request(receive_message_event)
+    if serialized_request is None:
+      with state.condition:
+        if state.client is _OPEN:
+          state.client = _CLOSED
+        state.condition.notify_all()
+        return _possibly_finish_call(state, _RECEIVE_MESSAGE_TOKEN)
+    else:
+      request = _common.deserialize(serialized_request, request_deserializer)
+      with state.condition:
+        if request is None:
+          _abort(
+              state, call, cygrpc.StatusCode.internal,
+              b'Exception deserializing request!')
+        else:
+          state.request = request
+        state.condition.notify_all()
+        return _possibly_finish_call(state, _RECEIVE_MESSAGE_TOKEN)
+  return receive_message
+
+
+def _send_initial_metadata(state):
+  def send_initial_metadata(unused_send_initial_metadata_event):
+    with state.condition:
+      return _possibly_finish_call(state, _SEND_INITIAL_METADATA_TOKEN)
+  return send_initial_metadata
+
+
+def _send_message(state, token):
+  def send_message(unused_send_message_event):
+    with state.condition:
+      state.condition.notify_all()
+      return _possibly_finish_call(state, token)
+  return send_message
+
+
+class _Context(grpc.ServicerContext):
+
+  def __init__(self, rpc_event, state, request_deserializer):
+    self._rpc_event = rpc_event
+    self._state = state
+    self._request_deserializer = request_deserializer
+
+  def is_active(self):
+    with self._state.condition:
+      return self._state.client is not _CANCELLED and not self._state.statused
+
+  def time_remaining(self):
+    return max(self._rpc_event.request_call_details.deadline - time.time(), 0)
+
+  def cancel(self):
+    self._rpc_event.operation_call.cancel()
+
+  def add_callback(self, callback):
+    with self._state.condition:
+      if self._state.callbacks is None:
+        return False
+      else:
+        self._state.callbacks.append(callback)
+        return True
+
+  def disable_next_message_compression(self):
+    with self._state.condition:
+      self._state.disable_next_compression = True
+
+  def invocation_metadata(self):
+    return _common.application_metadata(self._rpc_event.request_metadata)
+
+  def peer(self):
+    return _common.decode(self._rpc_event.operation_call.peer())
+
+  def send_initial_metadata(self, initial_metadata):
+    with self._state.condition:
+      if self._state.client is _CANCELLED:
+        _raise_rpc_error(self._state)
+      else:
+        if self._state.initial_metadata_allowed:
+          operation = cygrpc.operation_send_initial_metadata(
+              _common.cygrpc_metadata(initial_metadata), _EMPTY_FLAGS)
+          self._rpc_event.operation_call.start_batch(
+              cygrpc.Operations((operation,)),
+              _send_initial_metadata(self._state))
+          self._state.initial_metadata_allowed = False
+          self._state.due.add(_SEND_INITIAL_METADATA_TOKEN)
+        else:
+          raise ValueError('Initial metadata no longer allowed!')
+
+  def set_trailing_metadata(self, trailing_metadata):
+    with self._state.condition:
+      self._state.trailing_metadata = _common.cygrpc_metadata(
+          trailing_metadata)
+
+  def set_code(self, code):
+    with self._state.condition:
+      self._state.code = code
+
+  def set_details(self, details):
+    with self._state.condition:
+      self._state.details = _common.encode(details)
+
+
+class _RequestIterator(object):
+
+  def __init__(self, state, call, request_deserializer):
+    self._state = state
+    self._call = call
+    self._request_deserializer = request_deserializer
+
+  def _raise_or_start_receive_message(self):
+    if self._state.client is _CANCELLED:
+      _raise_rpc_error(self._state)
+    elif self._state.client is _CLOSED or self._state.statused:
+      raise StopIteration()
+    else:
+      self._call.start_batch(
+          cygrpc.Operations((cygrpc.operation_receive_message(_EMPTY_FLAGS),)),
+          _receive_message(self._state, self._call, self._request_deserializer))
+      self._state.due.add(_RECEIVE_MESSAGE_TOKEN)
+
+  def _look_for_request(self):
+    if self._state.client is _CANCELLED:
+      _raise_rpc_error(self._state)
+    elif (self._state.request is None and
+          _RECEIVE_MESSAGE_TOKEN not in self._state.due):
+      raise StopIteration()
+    else:
+      request = self._state.request
+      self._state.request = None
+      return request
+
+  def _next(self):
+    with self._state.condition:
+      self._raise_or_start_receive_message()
+      while True:
+        self._state.condition.wait()
+        request = self._look_for_request()
+        if request is not None:
+          return request
+
+  def __iter__(self):
+    return self
+
+  def __next__(self):
+    return self._next()
+
+  def next(self):
+    return self._next()
+
+
+def _unary_request(rpc_event, state, request_deserializer):
+  def unary_request():
+    with state.condition:
+      if state.client is _CANCELLED or state.statused:
+        return None
+      else:
+        start_batch_result = rpc_event.operation_call.start_batch(
+            cygrpc.Operations(
+                (cygrpc.operation_receive_message(_EMPTY_FLAGS),)),
+            _receive_message(
+                state, rpc_event.operation_call, request_deserializer))
+        state.due.add(_RECEIVE_MESSAGE_TOKEN)
+        while True:
+          state.condition.wait()
+          if state.request is None:
+            if state.client is _CLOSED:
+              details = '"{}" requires exactly one request message.'.format(
+                  rpc_event.request_call_details.method)
+              _abort(
+                  state, rpc_event.operation_call,
+                  cygrpc.StatusCode.unimplemented, _common.encode(details))
+              return None
+            elif state.client is _CANCELLED:
+              return None
+          else:
+            request = state.request
+            state.request = None
+            return request
+  return unary_request
+
+
+def _call_behavior(rpc_event, state, behavior, argument, request_deserializer):
+  context = _Context(rpc_event, state, request_deserializer)
+  try:
+    return behavior(argument, context), True
+  except Exception as e:  # pylint: disable=broad-except
+    with state.condition:
+      if e not in state.rpc_errors:
+        details = 'Exception calling application: {}'.format(e)
+        logging.exception(details)
+        _abort(state, rpc_event.operation_call,
+               cygrpc.StatusCode.unknown, _common.encode(details))
+    return None, False
+
+
+def _take_response_from_response_iterator(rpc_event, state, response_iterator):
+  try:
+    return next(response_iterator), True
+  except StopIteration:
+    return None, True
+  except Exception as e:  # pylint: disable=broad-except
+    with state.condition:
+      if e not in state.rpc_errors:
+        details = 'Exception iterating responses: {}'.format(e)
+        logging.exception(details)
+        _abort(state, rpc_event.operation_call,
+               cygrpc.StatusCode.unknown, _common.encode(details))
+    return None, False
+
+
+def _serialize_response(rpc_event, state, response, response_serializer):
+  serialized_response = _common.serialize(response, response_serializer)
+  if serialized_response is None:
+    with state.condition:
+      _abort(
+          state, rpc_event.operation_call, cygrpc.StatusCode.internal,
+          b'Failed to serialize response!')
+    return None
+  else:
+    return serialized_response
+
+
+def _send_response(rpc_event, state, serialized_response):
+  with state.condition:
+    if state.client is _CANCELLED or state.statused:
+      return False
+    else:
+      if state.initial_metadata_allowed:
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(serialized_response, _EMPTY_FLAGS),
+        )
+        state.initial_metadata_allowed = False
+        token = _SEND_INITIAL_METADATA_AND_SEND_MESSAGE_TOKEN
+      else:
+        operations = (
+            cygrpc.operation_send_message(serialized_response, _EMPTY_FLAGS),
+        )
+        token = _SEND_MESSAGE_TOKEN
+      rpc_event.operation_call.start_batch(
+          cygrpc.Operations(operations), _send_message(state, token))
+      state.due.add(token)
+      while True:
+        state.condition.wait()
+        if token not in state.due:
+          return state.client is not _CANCELLED and not state.statused
+
+
+def _status(rpc_event, state, serialized_response):
+  with state.condition:
+    if state.client is not _CANCELLED:
+      trailing_metadata = _common.cygrpc_metadata(state.trailing_metadata)
+      code = _completion_code(state)
+      details = _details(state)
+      operations = [
+          cygrpc.operation_send_status_from_server(
+              trailing_metadata, code, details, _EMPTY_FLAGS),
+      ]
+      if state.initial_metadata_allowed:
+        operations.append(
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS))
+      if serialized_response is not None:
+        operations.append(cygrpc.operation_send_message(
+            serialized_response, _EMPTY_FLAGS))
+      rpc_event.operation_call.start_batch(
+          cygrpc.Operations(operations),
+          _send_status_from_server(state, _SEND_STATUS_FROM_SERVER_TOKEN))
+      state.statused = True
+      state.due.add(_SEND_STATUS_FROM_SERVER_TOKEN)
+
+
+def _unary_response_in_pool(
+    rpc_event, state, behavior, argument_thunk, request_deserializer,
+    response_serializer):
+  argument = argument_thunk()
+  if argument is not None:
+    response, proceed = _call_behavior(
+        rpc_event, state, behavior, argument, request_deserializer)
+    if proceed:
+      serialized_response = _serialize_response(
+          rpc_event, state, response, response_serializer)
+      if serialized_response is not None:
+        _status(rpc_event, state, serialized_response)
+  return
+
+
+def _stream_response_in_pool(
+    rpc_event, state, behavior, argument_thunk, request_deserializer,
+    response_serializer):
+  argument = argument_thunk()
+  if argument is not None:
+    response_iterator, proceed = _call_behavior(
+        rpc_event, state, behavior, argument, request_deserializer)
+    if proceed:
+      while True:
+        response, proceed = _take_response_from_response_iterator(
+            rpc_event, state, response_iterator)
+        if proceed:
+          if response is None:
+            _status(rpc_event, state, None)
+            break
+          else:
+            serialized_response = _serialize_response(
+                rpc_event, state, response, response_serializer)
+            if serialized_response is not None:
+              proceed = _send_response(rpc_event, state, serialized_response)
+              if not proceed:
+                break
+            else:
+              break
+        else:
+          break
+
+
+def _handle_unary_unary(rpc_event, state, method_handler, thread_pool):
+  unary_request = _unary_request(
+      rpc_event, state, method_handler.request_deserializer)
+  thread_pool.submit(
+      _unary_response_in_pool, rpc_event, state, method_handler.unary_unary,
+      unary_request, method_handler.request_deserializer,
+      method_handler.response_serializer)
+
+
+def _handle_unary_stream(rpc_event, state, method_handler, thread_pool):
+  unary_request = _unary_request(
+      rpc_event, state, method_handler.request_deserializer)
+  thread_pool.submit(
+      _stream_response_in_pool, rpc_event, state, method_handler.unary_stream,
+      unary_request, method_handler.request_deserializer,
+      method_handler.response_serializer)
+
+
+def _handle_stream_unary(rpc_event, state, method_handler, thread_pool):
+  request_iterator = _RequestIterator(
+      state, rpc_event.operation_call, method_handler.request_deserializer)
+  thread_pool.submit(
+      _unary_response_in_pool, rpc_event, state, method_handler.stream_unary,
+      lambda: request_iterator, method_handler.request_deserializer,
+      method_handler.response_serializer)
+
+
+def _handle_stream_stream(rpc_event, state, method_handler, thread_pool):
+  request_iterator = _RequestIterator(
+      state, rpc_event.operation_call, method_handler.request_deserializer)
+  thread_pool.submit(
+      _stream_response_in_pool, rpc_event, state, method_handler.stream_stream,
+      lambda: request_iterator, method_handler.request_deserializer,
+      method_handler.response_serializer)
+
+
+def _find_method_handler(rpc_event, generic_handlers):
+  for generic_handler in generic_handlers:
+    method_handler = generic_handler.service(
+        _HandlerCallDetails(
+            _common.decode(rpc_event.request_call_details.method),
+            rpc_event.request_metadata))
+    if method_handler is not None:
+      return method_handler
+  else:
+    return None
+
+
+def _handle_unrecognized_method(rpc_event):
+  operations = (
+      cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, _EMPTY_FLAGS),
+      cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),
+      cygrpc.operation_send_status_from_server(
+          _EMPTY_METADATA, cygrpc.StatusCode.unimplemented,
+          b'Method not found!', _EMPTY_FLAGS),
+  )
+  rpc_state = _RPCState()
+  rpc_event.operation_call.start_batch(
+      operations, lambda ignored_event: (rpc_state, (),))
+  return rpc_state
+
+
+def _handle_with_method_handler(rpc_event, method_handler, thread_pool):
+  state = _RPCState()
+  with state.condition:
+    rpc_event.operation_call.start_batch(
+        cygrpc.Operations(
+            (cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),)),
+        _receive_close_on_server(state))
+    state.due.add(_RECEIVE_CLOSE_ON_SERVER_TOKEN)
+    if method_handler.request_streaming:
+      if method_handler.response_streaming:
+        _handle_stream_stream(rpc_event, state, method_handler, thread_pool)
+      else:
+        _handle_stream_unary(rpc_event, state, method_handler, thread_pool)
+    else:
+      if method_handler.response_streaming:
+        _handle_unary_stream(rpc_event, state, method_handler, thread_pool)
+      else:
+        _handle_unary_unary(rpc_event, state, method_handler, thread_pool)
+    return state
+
+
+def _handle_call(rpc_event, generic_handlers, thread_pool):
+  if rpc_event.request_call_details.method is not None:
+    method_handler = _find_method_handler(rpc_event, generic_handlers)
+    if method_handler is None:
+      return _handle_unrecognized_method(rpc_event)
+    else:
+      return _handle_with_method_handler(rpc_event, method_handler, thread_pool)
+  else:
+    return None
+
+
+@enum.unique
+class _ServerStage(enum.Enum):
+  STOPPED = 'stopped'
+  STARTED = 'started'
+  GRACE = 'grace'
+
+
+class _ServerState(object):
+
+  def __init__(self, completion_queue, server, generic_handlers, thread_pool):
+    self.lock = threading.Lock()
+    self.completion_queue = completion_queue
+    self.server = server
+    self.generic_handlers = list(generic_handlers)
+    self.thread_pool = thread_pool
+    self.stage = _ServerStage.STOPPED
+    self.shutdown_events = None
+
+    # TODO(https://github.com/grpc/grpc/issues/6597): eliminate these fields.
+    self.rpc_states = set()
+    self.due = set()
+
+
+def _add_generic_handlers(state, generic_handlers):
+  with state.lock:
+    state.generic_handlers.extend(generic_handlers)
+
+
+def _add_insecure_port(state, address):
+  with state.lock:
+    return state.server.add_http2_port(address)
+
+
+def _add_secure_port(state, address, server_credentials):
+  with state.lock:
+    return state.server.add_http2_port(address, server_credentials._credentials)
+
+
+def _request_call(state):
+  state.server.request_call(
+      state.completion_queue, state.completion_queue, _REQUEST_CALL_TAG)
+  state.due.add(_REQUEST_CALL_TAG)
+
+
+# TODO(https://github.com/grpc/grpc/issues/6597): delete this function.
+def _stop_serving(state):
+  if not state.rpc_states and not state.due:
+    for shutdown_event in state.shutdown_events:
+      shutdown_event.set()
+    state.stage = _ServerStage.STOPPED
+    return True
+  else:
+    return False
+
+
+def _serve(state):
+  while True:
+    event = state.completion_queue.poll()
+    if event.tag is _SHUTDOWN_TAG:
+      with state.lock:
+        state.due.remove(_SHUTDOWN_TAG)
+        if _stop_serving(state):
+          return
+    elif event.tag is _REQUEST_CALL_TAG:
+      with state.lock:
+        state.due.remove(_REQUEST_CALL_TAG)
+        rpc_state = _handle_call(
+            event, state.generic_handlers, state.thread_pool)
+        if rpc_state is not None:
+          state.rpc_states.add(rpc_state)
+        if state.stage is _ServerStage.STARTED:
+          _request_call(state)
+        elif _stop_serving(state):
+          return
+    else:
+      rpc_state, callbacks = event.tag(event)
+      for callback in callbacks:
+        callable_util.call_logging_exceptions(
+            callback, 'Exception calling callback!')
+      if rpc_state is not None:
+        with state.lock:
+          state.rpc_states.remove(rpc_state)
+          if _stop_serving(state):
+            return
+
+
+def _stop(state, grace):
+  with state.lock:
+    if state.stage is _ServerStage.STOPPED:
+      shutdown_event = threading.Event()
+      shutdown_event.set()
+      return shutdown_event
+    else:
+      if state.stage is _ServerStage.STARTED:
+        state.server.shutdown(state.completion_queue, _SHUTDOWN_TAG)
+        state.stage = _ServerStage.GRACE
+        state.shutdown_events = []
+        state.due.add(_SHUTDOWN_TAG)
+      shutdown_event = threading.Event()
+      state.shutdown_events.append(shutdown_event)
+      if grace is None:
+        state.server.cancel_all_calls()
+        # TODO(https://github.com/grpc/grpc/issues/6597): delete this loop.
+        for rpc_state in state.rpc_states:
+          with rpc_state.condition:
+            rpc_state.client = _CANCELLED
+            rpc_state.condition.notify_all()
+      else:
+        def cancel_all_calls_after_grace():
+          shutdown_event.wait(timeout=grace)
+          with state.lock:
+            state.server.cancel_all_calls()
+            # TODO(https://github.com/grpc/grpc/issues/6597): delete this loop.
+            for rpc_state in state.rpc_states:
+              with rpc_state.condition:
+                rpc_state.client = _CANCELLED
+                rpc_state.condition.notify_all()
+        thread = threading.Thread(target=cancel_all_calls_after_grace)
+        thread.start()
+        return shutdown_event
+  shutdown_event.wait()
+  return shutdown_event
+
+
+def _start(state):
+  with state.lock:
+    if state.stage is not _ServerStage.STOPPED:
+      raise ValueError('Cannot start already-started server!')
+    state.server.start()
+    state.stage = _ServerStage.STARTED
+    _request_call(state)    
+    def cleanup_server(timeout):
+      if timeout is None:
+        _stop(state, _UNEXPECTED_EXIT_SERVER_GRACE).wait()
+      else:
+        _stop(state, timeout).wait()
+
+    thread = _common.CleanupThread(
+        cleanup_server, target=_serve, args=(state,))
+    thread.start()
+
+
+class Server(grpc.Server):
+
+  def __init__(self, generic_handlers, thread_pool):
+    completion_queue = cygrpc.CompletionQueue()
+    server = cygrpc.Server()
+    server.register_completion_queue(completion_queue)
+    self._state = _ServerState(
+        completion_queue, server, generic_handlers, thread_pool)
+
+  def add_generic_rpc_handlers(self, generic_rpc_handlers):
+    _add_generic_handlers(self._state, generic_rpc_handlers)
+
+  def add_insecure_port(self, address):
+    return _add_insecure_port(self._state, _common.encode(address))
+
+  def add_secure_port(self, address, server_credentials):
+    return _add_secure_port(self._state, _common.encode(address), server_credentials)
+
+  def start(self):
+    _start(self._state)
+
+  def stop(self, grace):
+    return _stop(self._state, grace)
+
+  def __del__(self):
+    _stop(self._state, None)
diff --git a/src/python/grpcio/grpc/_utilities.py b/src/python/grpcio/grpc/_utilities.py
new file mode 100644
index 0000000000..4850967fbc
--- /dev/null
+++ b/src/python/grpcio/grpc/_utilities.py
@@ -0,0 +1,171 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Internal utilities for gRPC Python."""
+
+import collections
+import threading
+import time
+
+import six
+
+import grpc
+from grpc import _common
+from grpc.framework.foundation import callable_util
+
+_DONE_CALLBACK_EXCEPTION_LOG_MESSAGE = (
+    'Exception calling connectivity future "done" callback!')
+
+
+class RpcMethodHandler(
+    collections.namedtuple(
+        '_RpcMethodHandler',
+        ('request_streaming', 'response_streaming', 'request_deserializer',
+         'response_serializer', 'unary_unary', 'unary_stream', 'stream_unary',
+         'stream_stream',)),
+    grpc.RpcMethodHandler):
+  pass
+
+
+class DictionaryGenericHandler(grpc.GenericRpcHandler):
+
+  def __init__(self, service, method_handlers):
+    self._method_handlers = {
+        _common.fully_qualified_method(service, method): method_handler
+        for method, method_handler in six.iteritems(method_handlers)}
+
+  def service(self, handler_call_details):
+    return self._method_handlers.get(handler_call_details.method)
+
+
+class _ChannelReadyFuture(grpc.Future):
+
+  def __init__(self, channel):
+    self._condition = threading.Condition()
+    self._channel = channel
+
+    self._matured = False
+    self._cancelled = False
+    self._done_callbacks = []
+
+  def _block(self, timeout):
+    until = None if timeout is None else time.time() + timeout
+    with self._condition:
+      while True:
+        if self._cancelled:
+          raise grpc.FutureCancelledError()
+        elif self._matured:
+          return
+        else:
+          if until is None:
+            self._condition.wait()
+          else:
+            remaining = until - time.time()
+            if remaining < 0:
+              raise grpc.FutureTimeoutError()
+            else:
+              self._condition.wait(timeout=remaining)
+
+  def _update(self, connectivity):
+    with self._condition:
+      if (not self._cancelled and
+          connectivity is grpc.ChannelConnectivity.READY):
+        self._matured = True
+        self._channel.unsubscribe(self._update)
+        self._condition.notify_all()
+        done_callbacks = tuple(self._done_callbacks)
+        self._done_callbacks = None
+      else:
+        return
+
+    for done_callback in done_callbacks:
+      callable_util.call_logging_exceptions(
+          done_callback, _DONE_CALLBACK_EXCEPTION_LOG_MESSAGE, self)
+
+  def cancel(self):
+    with self._condition:
+      if not self._matured:
+        self._cancelled = True
+        self._channel.unsubscribe(self._update)
+        self._condition.notify_all()
+        done_callbacks = tuple(self._done_callbacks)
+        self._done_callbacks = None
+      else:
+        return False
+
+    for done_callback in done_callbacks:
+      callable_util.call_logging_exceptions(
+          done_callback, _DONE_CALLBACK_EXCEPTION_LOG_MESSAGE, self)
+
+  def cancelled(self):
+    with self._condition:
+      return self._cancelled
+
+  def running(self):
+    with self._condition:
+      return not self._cancelled and not self._matured
+
+  def done(self):
+    with self._condition:
+      return self._cancelled or self._matured
+
+  def result(self, timeout=None):
+    self._block(timeout)
+    return None
+
+  def exception(self, timeout=None):
+    self._block(timeout)
+    return None
+
+  def traceback(self, timeout=None):
+    self._block(timeout)
+    return None
+
+  def add_done_callback(self, fn):
+    with self._condition:
+      if not self._cancelled and not self._matured:
+        self._done_callbacks.append(fn)
+        return
+
+    fn(self)
+
+  def start(self):
+    with self._condition:
+      self._channel.subscribe(self._update, try_to_connect=True)
+
+  def __del__(self):
+    with self._condition:
+      if not self._cancelled and not self._matured:
+        self._channel.unsubscribe(self._update)
+
+
+def channel_ready_future(channel):
+  ready_future = _ChannelReadyFuture(channel)
+  ready_future.start()
+  return ready_future
diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py
new file mode 100644
index 0000000000..56456cc117
--- /dev/null
+++ b/src/python/grpcio/grpc/beta/_client_adaptations.py
@@ -0,0 +1,563 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Translates gRPC's client-side API into gRPC's client-side Beta API."""
+
+import grpc
+from grpc import _common
+from grpc._cython import cygrpc
+from grpc.beta import interfaces
+from grpc.framework.common import cardinality
+from grpc.framework.foundation import future
+from grpc.framework.interfaces.face import face
+
+_STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS = {
+    grpc.StatusCode.CANCELLED: (
+        face.Abortion.Kind.CANCELLED, face.CancellationError),
+    grpc.StatusCode.UNKNOWN: (
+        face.Abortion.Kind.REMOTE_FAILURE, face.RemoteError),
+    grpc.StatusCode.DEADLINE_EXCEEDED: (
+        face.Abortion.Kind.EXPIRED, face.ExpirationError),
+    grpc.StatusCode.UNIMPLEMENTED: (
+        face.Abortion.Kind.LOCAL_FAILURE, face.LocalError),
+}
+
+
+def _effective_metadata(metadata, metadata_transformer):
+  non_none_metadata = () if metadata is None else metadata
+  if metadata_transformer is None:
+    return non_none_metadata
+  else:
+    return metadata_transformer(non_none_metadata)
+
+
+def _credentials(grpc_call_options):
+  return None if grpc_call_options is None else grpc_call_options.credentials
+
+
+def _abortion(rpc_error_call):
+  code = rpc_error_call.code()
+  pair = _STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS.get(code)
+  error_kind = face.Abortion.Kind.LOCAL_FAILURE if pair is None else pair[0]
+  return face.Abortion(
+      error_kind, rpc_error_call.initial_metadata(),
+      rpc_error_call.trailing_metadata(), code, rpc_error_code.details())
+
+
+def _abortion_error(rpc_error_call):
+  code = rpc_error_call.code()
+  pair = _STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS.get(code)
+  exception_class = face.AbortionError if pair is None else pair[1]
+  return exception_class(
+      rpc_error_call.initial_metadata(), rpc_error_call.trailing_metadata(),
+      code, rpc_error_call.details())
+
+
+class _InvocationProtocolContext(interfaces.GRPCInvocationContext):
+
+  def disable_next_request_compression(self):
+    pass  # TODO(https://github.com/grpc/grpc/issues/4078): design, implement.
+
+
+class _Rendezvous(future.Future, face.Call):
+
+  def __init__(self, response_future, response_iterator, call):
+    self._future = response_future
+    self._iterator = response_iterator
+    self._call = call
+
+  def cancel(self):
+    return self._call.cancel()
+
+  def cancelled(self):
+    return self._future.cancelled()
+
+  def running(self):
+    return self._future.running()
+
+  def done(self):
+    return self._future.done()
+
+  def result(self, timeout=None):
+    try:
+      return self._future.result(timeout=timeout)
+    except grpc.RpcError as rpc_error_call:
+      raise _abortion_error(rpc_error_call)
+    except grpc.FutureTimeoutError:
+      raise future.TimeoutError()
+    except grpc.FutureCancelledError:
+      raise future.CancelledError()
+
+  def exception(self, timeout=None):
+    try:
+      rpc_error_call = self._future.exception(timeout=timeout)
+      return _abortion_error(rpc_error_call)
+    except grpc.FutureTimeoutError:
+      raise future.TimeoutError()
+    except grpc.FutureCancelledError:
+      raise future.CancelledError()
+
+  def traceback(self, timeout=None):
+    try:
+      return self._future.traceback(timeout=timeout)
+    except grpc.FutureTimeoutError:
+      raise future.TimeoutError()
+    except grpc.FutureCancelledError:
+      raise future.CancelledError()
+
+  def add_done_callback(self, fn):
+    self._future.add_done_callback(lambda ignored_callback: fn(self))
+
+  def __iter__(self):
+    return self
+
+  def _next(self):
+    try:
+      return next(self._iterator)
+    except grpc.RpcError as rpc_error_call:
+      raise _abortion_error(rpc_error_call)
+
+  def __next__(self):
+    return self._next()
+
+  def next(self):
+    return self._next()
+
+  def is_active(self):
+    return self._call.is_active()
+
+  def time_remaining(self):
+    return self._call.time_remaining()
+
+  def add_abortion_callback(self, abortion_callback):
+    registered = self._call.add_callback(
+        lambda: abortion_callback(_abortion(self._call)))
+    return None if registered else _abortion(self._call)
+
+  def protocol_context(self):
+    return _InvocationProtocolContext()
+
+  def initial_metadata(self):
+    return self._call.initial_metadata()
+
+  def terminal_metadata(self):
+    return self._call.terminal_metadata()
+
+  def code(self):
+    return self._call.code()
+
+  def details(self):
+    return self._call.details()
+
+
+def _blocking_unary_unary(
+    channel, group, method, timeout, with_call, protocol_options, metadata,
+    metadata_transformer, request, request_serializer, response_deserializer):
+  try:
+    multi_callable = channel.unary_unary(
+        _common.fully_qualified_method(group, method),
+        request_serializer=request_serializer,
+        response_deserializer=response_deserializer)
+    effective_metadata = _effective_metadata(metadata, metadata_transformer)
+    if with_call:
+      response, call = multi_callable.with_call(
+          request, timeout=timeout, metadata=effective_metadata,
+          credentials=_credentials(protocol_options))
+      return response, _Rendezvous(None, None, call)
+    else:
+      return multi_callable(
+          request, timeout=timeout, metadata=effective_metadata,
+          credentials=_credentials(protocol_options))
+  except grpc.RpcError as rpc_error_call:
+    raise _abortion_error(rpc_error_call)
+
+
+def _future_unary_unary(
+    channel, group, method, timeout, protocol_options, metadata,
+    metadata_transformer, request, request_serializer, response_deserializer):
+  multi_callable = channel.unary_unary(
+      _common.fully_qualified_method(group, method),
+      request_serializer=request_serializer,
+      response_deserializer=response_deserializer)
+  effective_metadata = _effective_metadata(metadata, metadata_transformer)
+  response_future = multi_callable.future(
+      request, timeout=timeout, metadata=effective_metadata,
+      credentials=_credentials(protocol_options))
+  return _Rendezvous(response_future, None, response_future)
+
+
+def _unary_stream(
+    channel, group, method, timeout, protocol_options, metadata,
+    metadata_transformer, request, request_serializer, response_deserializer):
+  multi_callable = channel.unary_stream(
+      _common.fully_qualified_method(group, method),
+      request_serializer=request_serializer,
+      response_deserializer=response_deserializer)
+  effective_metadata = _effective_metadata(metadata, metadata_transformer)
+  response_iterator = multi_callable(
+      request, timeout=timeout, metadata=effective_metadata,
+      credentials=_credentials(protocol_options))
+  return _Rendezvous(None, response_iterator, response_iterator)
+
+
+def _blocking_stream_unary(
+    channel, group, method, timeout, with_call, protocol_options, metadata,
+    metadata_transformer, request_iterator, request_serializer,
+    response_deserializer):
+  try:
+    multi_callable = channel.stream_unary(
+        _common.fully_qualified_method(group, method),
+        request_serializer=request_serializer,
+        response_deserializer=response_deserializer)
+    effective_metadata = _effective_metadata(metadata, metadata_transformer)
+    if with_call:
+      response, call = multi_callable.with_call(
+          request_iterator, timeout=timeout, metadata=effective_metadata,
+          credentials=_credentials(protocol_options))
+      return response, _Rendezvous(None, None, call)
+    else:
+      return multi_callable(
+          request_iterator, timeout=timeout, metadata=effective_metadata,
+          credentials=_credentials(protocol_options))
+  except grpc.RpcError as rpc_error_call:
+    raise _abortion_error(rpc_error_call)
+
+
+def _future_stream_unary(
+    channel, group, method, timeout, protocol_options, metadata,
+    metadata_transformer, request_iterator, request_serializer,
+    response_deserializer):
+  multi_callable = channel.stream_unary(
+      _common.fully_qualified_method(group, method),
+      request_serializer=request_serializer,
+      response_deserializer=response_deserializer)
+  effective_metadata = _effective_metadata(metadata, metadata_transformer)
+  response_future = multi_callable.future(
+      request_iterator, timeout=timeout, metadata=effective_metadata,
+      credentials=_credentials(protocol_options))
+  return _Rendezvous(response_future, None, response_future)
+
+
+def _stream_stream(
+    channel, group, method, timeout, protocol_options, metadata,
+    metadata_transformer, request_iterator, request_serializer,
+    response_deserializer):
+  multi_callable = channel.stream_stream(
+      _common.fully_qualified_method(group, method),
+      request_serializer=request_serializer,
+      response_deserializer=response_deserializer)
+  effective_metadata = _effective_metadata(metadata, metadata_transformer)
+  response_iterator = multi_callable(
+      request_iterator, timeout=timeout, metadata=effective_metadata,
+      credentials=_credentials(protocol_options))
+  return _Rendezvous(None, response_iterator, response_iterator)
+
+
+class _UnaryUnaryMultiCallable(face.UnaryUnaryMultiCallable):
+
+  def __init__(
+      self, channel, group, method, metadata_transformer, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._group = group
+    self._method = method
+    self._metadata_transformer = metadata_transformer
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def __call__(
+      self, request, timeout, metadata=None, with_call=False,
+      protocol_options=None):
+    return _blocking_unary_unary(
+        self._channel, self._group, self._method, timeout, with_call,
+        protocol_options, metadata, self._metadata_transformer, request,
+        self._request_serializer, self._response_deserializer)
+
+  def future(self, request, timeout, metadata=None, protocol_options=None):
+    return _future_unary_unary(
+        self._channel, self._group, self._method, timeout, protocol_options,
+        metadata, self._metadata_transformer, request, self._request_serializer,
+        self._response_deserializer)
+
+  def event(
+      self, request, receiver, abortion_callback, timeout,
+      metadata=None, protocol_options=None):
+    raise NotImplementedError()
+
+
+class _UnaryStreamMultiCallable(face.UnaryStreamMultiCallable):
+
+  def __init__(
+      self, channel, group, method, metadata_transformer, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._group = group
+    self._method = method
+    self._metadata_transformer = metadata_transformer
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def __call__(self, request, timeout, metadata=None, protocol_options=None):
+    return _unary_stream(
+        self._channel, self._group, self._method, timeout, protocol_options,
+        metadata, self._metadata_transformer, request, self._request_serializer,
+        self._response_deserializer)
+
+  def event(
+      self, request, receiver, abortion_callback, timeout,
+      metadata=None, protocol_options=None):
+    raise NotImplementedError()
+
+
+class _StreamUnaryMultiCallable(face.StreamUnaryMultiCallable):
+
+  def __init__(
+      self, channel, group, method, metadata_transformer, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._group = group
+    self._method = method
+    self._metadata_transformer = metadata_transformer
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def __call__(
+      self, request_iterator, timeout, metadata=None, with_call=False,
+      protocol_options=None):
+    return _blocking_stream_unary(
+        self._channel, self._group, self._method, timeout, with_call,
+        protocol_options, metadata, self._metadata_transformer,
+        request_iterator, self._request_serializer, self._response_deserializer)
+
+  def future(
+      self, request_iterator, timeout, metadata=None, protocol_options=None):
+    return _future_stream_unary(
+        self._channel, self._group, self._method, timeout, protocol_options,
+        metadata, self._metadata_transformer, request_iterator,
+        self._request_serializer, self._response_deserializer)
+
+  def event(
+      self, receiver, abortion_callback, timeout, metadata=None,
+      protocol_options=None):
+    raise NotImplementedError()
+
+
+class _StreamStreamMultiCallable(face.StreamStreamMultiCallable):
+
+  def __init__(
+      self, channel, group, method, metadata_transformer, request_serializer,
+      response_deserializer):
+    self._channel = channel
+    self._group = group
+    self._method = method
+    self._metadata_transformer = metadata_transformer
+    self._request_serializer = request_serializer
+    self._response_deserializer = response_deserializer
+
+  def __call__(
+      self, request_iterator, timeout, metadata=None, protocol_options=None):
+    return _stream_stream(
+        self._channel, self._group, self._method, timeout, protocol_options,
+        metadata, self._metadata_transformer, request_iterator,
+        self._request_serializer, self._response_deserializer)
+
+  def event(
+      self, receiver, abortion_callback, timeout, metadata=None,
+      protocol_options=None):
+    raise NotImplementedError()
+
+
+class _GenericStub(face.GenericStub):
+
+  def __init__(
+      self, channel, metadata_transformer, request_serializers,
+      response_deserializers):
+    self._channel = channel
+    self._metadata_transformer = metadata_transformer
+    self._request_serializers = request_serializers or {}
+    self._response_deserializers = response_deserializers or {}
+
+  def blocking_unary_unary(
+      self, group, method, request, timeout, metadata=None,
+      with_call=None, protocol_options=None):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _blocking_unary_unary(
+        self._channel, group, method, timeout, with_call, protocol_options,
+        metadata, self._metadata_transformer, request, request_serializer,
+        response_deserializer)
+
+  def future_unary_unary(
+      self, group, method, request, timeout, metadata=None,
+      protocol_options=None):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _future_unary_unary(
+        self._channel, group, method, timeout, protocol_options, metadata,
+        self._metadata_transformer, request, request_serializer,
+        response_deserializer)
+
+  def inline_unary_stream(
+      self, group, method, request, timeout, metadata=None,
+      protocol_options=None):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _unary_stream(
+        self._channel, group, method, timeout, protocol_options, metadata,
+        self._metadata_transformer, request, request_serializer,
+        response_deserializer)
+
+  def blocking_stream_unary(
+      self, group, method, request_iterator, timeout, metadata=None,
+      with_call=None, protocol_options=None):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _blocking_stream_unary(
+        self._channel, group, method, timeout, with_call, protocol_options,
+        metadata, self._metadata_transformer, request_iterator,
+        request_serializer, response_deserializer)
+
+  def future_stream_unary(
+      self, group, method, request_iterator, timeout, metadata=None,
+      protocol_options=None):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _future_stream_unary(
+        self._channel, group, method, timeout, protocol_options, metadata,
+        self._metadata_transformer, request_iterator, request_serializer,
+        response_deserializer)
+
+  def inline_stream_stream(
+      self, group, method, request_iterator, timeout, metadata=None,
+      protocol_options=None):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _stream_stream(
+        self._channel, group, method, timeout, protocol_options, metadata,
+        self._metadata_transformer, request_iterator, request_serializer,
+        response_deserializer)
+
+  def event_unary_unary(
+      self, group, method, request, receiver, abortion_callback, timeout,
+      metadata=None, protocol_options=None):
+    raise NotImplementedError()
+
+  def event_unary_stream(
+      self, group, method, request, receiver, abortion_callback, timeout,
+      metadata=None, protocol_options=None):
+    raise NotImplementedError()
+
+  def event_stream_unary(
+      self, group, method, receiver, abortion_callback, timeout,
+      metadata=None, protocol_options=None):
+    raise NotImplementedError()
+
+  def event_stream_stream(
+      self, group, method, receiver, abortion_callback, timeout,
+      metadata=None, protocol_options=None):
+    raise NotImplementedError()
+
+  def unary_unary(self, group, method):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _UnaryUnaryMultiCallable(
+        self._channel, group, method, self._metadata_transformer,
+        request_serializer, response_deserializer)
+
+  def unary_stream(self, group, method):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _UnaryStreamMultiCallable(
+        self._channel, group, method, self._metadata_transformer,
+        request_serializer, response_deserializer)
+
+  def stream_unary(self, group, method):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _StreamUnaryMultiCallable(
+        self._channel, group, method, self._metadata_transformer,
+        request_serializer, response_deserializer)
+
+  def stream_stream(self, group, method):
+    request_serializer = self._request_serializers.get((group, method,))
+    response_deserializer = self._response_deserializers.get((group, method,))
+    return _StreamStreamMultiCallable(
+        self._channel, group, method, self._metadata_transformer,
+        request_serializer, response_deserializer)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    return False
+
+
+class _DynamicStub(face.DynamicStub):
+
+  def __init__(self, generic_stub, group, cardinalities):
+    self._generic_stub = generic_stub
+    self._group = group
+    self._cardinalities = cardinalities
+
+  def __getattr__(self, attr):
+    method_cardinality = self._cardinalities.get(attr)
+    if method_cardinality is cardinality.Cardinality.UNARY_UNARY:
+      return self._generic_stub.unary_unary(self._group, attr)
+    elif method_cardinality is cardinality.Cardinality.UNARY_STREAM:
+      return self._generic_stub.unary_stream(self._group, attr)
+    elif method_cardinality is cardinality.Cardinality.STREAM_UNARY:
+      return self._generic_stub.stream_unary(self._group, attr)
+    elif method_cardinality is cardinality.Cardinality.STREAM_STREAM:
+      return self._generic_stub.stream_stream(self._group, attr)
+    else:
+      raise AttributeError('_DynamicStub object has no attribute "%s"!' % attr)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    return False
+
+
+def generic_stub(
+    channel, host, metadata_transformer, request_serializers,
+    response_deserializers):
+  return _GenericStub(
+      channel, metadata_transformer, request_serializers,
+      response_deserializers)
+
+
+def dynamic_stub(
+    channel, service, cardinalities, host, metadata_transformer,
+    request_serializers, response_deserializers):
+  return _DynamicStub(
+      _GenericStub(
+          channel, metadata_transformer, request_serializers,
+          response_deserializers),
+      service, cardinalities)
diff --git a/src/python/grpcio/grpc/beta/_server_adaptations.py b/src/python/grpcio/grpc/beta/_server_adaptations.py
new file mode 100644
index 0000000000..1e1f80156a
--- /dev/null
+++ b/src/python/grpcio/grpc/beta/_server_adaptations.py
@@ -0,0 +1,374 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Translates gRPC's server-side API into gRPC's server-side Beta API."""
+
+import collections
+import threading
+
+import grpc
+from grpc import _common
+from grpc.beta import interfaces
+from grpc.framework.common import cardinality
+from grpc.framework.common import style
+from grpc.framework.foundation import abandonment
+from grpc.framework.foundation import logging_pool
+from grpc.framework.foundation import stream
+from grpc.framework.interfaces.face import face
+
+_DEFAULT_POOL_SIZE = 8
+
+
+class _ServerProtocolContext(interfaces.GRPCServicerContext):
+
+  def __init__(self, servicer_context):
+    self._servicer_context = servicer_context
+
+  def peer(self):
+    return self._servicer_context.peer()
+
+  def disable_next_response_compression(self):
+    pass  # TODO(https://github.com/grpc/grpc/issues/4078): design, implement.
+
+
+class _FaceServicerContext(face.ServicerContext):
+
+  def __init__(self, servicer_context):
+    self._servicer_context = servicer_context
+
+  def is_active(self):
+    return self._servicer_context.is_active()
+
+  def time_remaining(self):
+    return self._servicer_context.time_remaining()
+
+  def add_abortion_callback(self, abortion_callback):
+    raise NotImplementedError(
+        'add_abortion_callback no longer supported server-side!')
+
+  def cancel(self):
+    self._servicer_context.cancel()
+
+  def protocol_context(self):
+    return _ServerProtocolContext(self._servicer_context)
+
+  def invocation_metadata(self):
+    return _common.cygrpc_metadata(
+        self._servicer_context.invocation_metadata())
+
+  def initial_metadata(self, initial_metadata):
+    self._servicer_context.send_initial_metadata(initial_metadata)
+
+  def terminal_metadata(self, terminal_metadata):
+    self._servicer_context.set_terminal_metadata(terminal_metadata)
+
+  def code(self, code):
+    self._servicer_context.set_code(code)
+
+  def details(self, details):
+    self._servicer_context.set_details(details)
+
+
+def _adapt_unary_request_inline(unary_request_inline):
+  def adaptation(request, servicer_context):
+    return unary_request_inline(request, _FaceServicerContext(servicer_context))
+  return adaptation
+
+
+def _adapt_stream_request_inline(stream_request_inline):
+  def adaptation(request_iterator, servicer_context):
+    return stream_request_inline(
+        request_iterator, _FaceServicerContext(servicer_context))
+  return adaptation
+
+
+class _Callback(stream.Consumer):
+
+  def __init__(self):
+    self._condition = threading.Condition()
+    self._values = []
+    self._terminated = False
+    self._cancelled = False
+
+  def consume(self, value):
+    with self._condition:
+      self._values.append(value)
+      self._condition.notify_all()
+
+  def terminate(self):
+    with self._condition:
+      self._terminated = True
+      self._condition.notify_all()
+
+  def consume_and_terminate(self, value):
+    with self._condition:
+      self._values.append(value)
+      self._terminated = True
+      self._condition.notify_all()
+
+  def cancel(self):
+    with self._condition:
+      self._cancelled = True
+      self._condition.notify_all()
+
+  def draw_one_value(self):
+    with self._condition:
+      while True:
+        if self._cancelled:
+          raise abandonment.Abandoned()
+        elif self._values:
+          return self._values.pop(0)
+        elif self._terminated:
+          return None
+        else:
+          self._condition.wait()
+
+  def draw_all_values(self):
+    with self._condition:
+      while True:
+        if self._cancelled:
+          raise abandonment.Abandoned()
+        elif self._terminated:
+          all_values = tuple(self._values)
+          self._values = None
+          return all_values
+        else:
+          self._condition.wait()
+
+
+def _run_request_pipe_thread(request_iterator, request_consumer,
+                             servicer_context):
+  thread_joined = threading.Event()
+  def pipe_requests():
+    for request in request_iterator:
+      if not servicer_context.is_active() or thread_joined.is_set():
+        return
+      request_consumer.consume(request)
+      if not servicer_context.is_active() or thread_joined.is_set():
+        return
+    request_consumer.terminate()
+
+  def stop_request_pipe(timeout):
+    thread_joined.set()
+
+  request_pipe_thread = _common.CleanupThread(
+      stop_request_pipe, target=pipe_requests)
+  request_pipe_thread.start()
+
+
+def _adapt_unary_unary_event(unary_unary_event):
+  def adaptation(request, servicer_context):
+    callback = _Callback()
+    if not servicer_context.add_callback(callback.cancel):
+      raise abandonment.Abandoned()
+    unary_unary_event(
+        request, callback.consume_and_terminate,
+        _FaceServicerContext(servicer_context))
+    return callback.draw_all_values()[0]
+  return adaptation
+
+
+def _adapt_unary_stream_event(unary_stream_event):
+  def adaptation(request, servicer_context):
+    callback = _Callback()
+    if not servicer_context.add_callback(callback.cancel):
+      raise abandonment.Abandoned()
+    unary_stream_event(
+        request, callback, _FaceServicerContext(servicer_context))
+    while True:
+      response = callback.draw_one_value()
+      if response is None:
+        return
+      else:
+        yield response
+  return adaptation
+
+
+def _adapt_stream_unary_event(stream_unary_event):
+  def adaptation(request_iterator, servicer_context):
+    callback = _Callback()
+    if not servicer_context.add_callback(callback.cancel):
+      raise abandonment.Abandoned()
+    request_consumer = stream_unary_event(
+        callback.consume_and_terminate, _FaceServicerContext(servicer_context))
+    _run_request_pipe_thread(
+        request_iterator, request_consumer, servicer_context)
+    return callback.draw_all_values()[0]
+  return adaptation
+
+
+def _adapt_stream_stream_event(stream_stream_event):
+  def adaptation(request_iterator, servicer_context):
+    callback = _Callback()
+    if not servicer_context.add_callback(callback.cancel):
+      raise abandonment.Abandoned()
+    request_consumer = stream_stream_event(
+        callback, _FaceServicerContext(servicer_context))
+    _run_request_pipe_thread(
+        request_iterator, request_consumer, servicer_context)
+    while True:
+      response = callback.draw_one_value()
+      if response is None:
+        return
+      else:
+        yield response
+  return adaptation
+
+
+class _SimpleMethodHandler(
+    collections.namedtuple(
+        '_MethodHandler',
+        ('request_streaming', 'response_streaming', 'request_deserializer',
+         'response_serializer', 'unary_unary', 'unary_stream', 'stream_unary',
+         'stream_stream',)),
+    grpc.RpcMethodHandler):
+  pass
+
+
+def _simple_method_handler(
+    implementation, request_deserializer, response_serializer):
+  if implementation.style is style.Service.INLINE:
+    if implementation.cardinality is cardinality.Cardinality.UNARY_UNARY:
+      return _SimpleMethodHandler(
+          False, False, request_deserializer, response_serializer,
+          _adapt_unary_request_inline(implementation.unary_unary_inline), None,
+          None, None)
+    elif implementation.cardinality is cardinality.Cardinality.UNARY_STREAM:
+      return _SimpleMethodHandler(
+          False, True, request_deserializer, response_serializer, None,
+          _adapt_unary_request_inline(implementation.unary_stream_inline), None,
+          None)
+    elif implementation.cardinality is cardinality.Cardinality.STREAM_UNARY:
+      return _SimpleMethodHandler(
+          True, False, request_deserializer, response_serializer, None, None,
+          _adapt_stream_request_inline(implementation.stream_unary_inline),
+          None)
+    elif implementation.cardinality is cardinality.Cardinality.STREAM_STREAM:
+      return _SimpleMethodHandler(
+          True, True, request_deserializer, response_serializer, None, None,
+          None,
+          _adapt_stream_request_inline(implementation.stream_stream_inline))
+  elif implementation.style is style.Service.EVENT:
+    if implementation.cardinality is cardinality.Cardinality.UNARY_UNARY:
+      return _SimpleMethodHandler(
+          False, False, request_deserializer, response_serializer,
+          _adapt_unary_unary_event(implementation.unary_unary_event), None,
+          None, None)
+    elif implementation.cardinality is cardinality.Cardinality.UNARY_STREAM:
+      return _SimpleMethodHandler(
+          False, True, request_deserializer, response_serializer, None,
+          _adapt_unary_stream_event(implementation.unary_stream_event), None,
+          None)
+    elif implementation.cardinality is cardinality.Cardinality.STREAM_UNARY:
+      return _SimpleMethodHandler(
+          True, False, request_deserializer, response_serializer, None, None,
+          _adapt_stream_unary_event(implementation.stream_unary_event), None)
+    elif implementation.cardinality is cardinality.Cardinality.STREAM_STREAM:
+      return _SimpleMethodHandler(
+          True, True, request_deserializer, response_serializer, None, None,
+          None, _adapt_stream_stream_event(implementation.stream_stream_event))
+
+
+def _flatten_method_pair_map(method_pair_map):
+  method_pair_map = method_pair_map or {}
+  flat_map = {}
+  for method_pair in method_pair_map:
+    method = _common.fully_qualified_method(method_pair[0], method_pair[1])
+    flat_map[method] = method_pair_map[method_pair]
+  return flat_map
+
+
+class _GenericRpcHandler(grpc.GenericRpcHandler):
+
+  def __init__(
+      self, method_implementations, multi_method_implementation,
+      request_deserializers, response_serializers):
+    self._method_implementations = _flatten_method_pair_map(
+        method_implementations)
+    self._request_deserializers = _flatten_method_pair_map(
+        request_deserializers)
+    self._response_serializers = _flatten_method_pair_map(
+        response_serializers)
+    self._multi_method_implementation = multi_method_implementation
+
+  def service(self, handler_call_details):
+    method_implementation = self._method_implementations.get(
+        handler_call_details.method)
+    if method_implementation is not None:
+      return _simple_method_handler(
+          method_implementation,
+          self._request_deserializers.get(handler_call_details.method),
+          self._response_serializers.get(handler_call_details.method))
+    elif self._multi_method_implementation is None:
+      return None
+    else:
+      try:
+        return None  #TODO(nathaniel): call the multimethod.
+      except face.NoSuchMethodError:
+        return None
+
+
+class _Server(interfaces.Server):
+
+  def __init__(self, server):
+    self._server = server
+
+  def add_insecure_port(self, address):
+    return self._server.add_insecure_port(address)
+
+  def add_secure_port(self, address, server_credentials):
+    return self._server.add_secure_port(address, server_credentials)
+
+  def start(self):
+    self._server.start()
+
+  def stop(self, grace):
+    return self._server.stop(grace)
+
+  def __enter__(self):
+    self._server.start()
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    self._server.stop(None)
+    return False
+
+
+def server(
+    service_implementations, multi_method_implementation, request_deserializers,
+    response_serializers, thread_pool, thread_pool_size):
+  generic_rpc_handler = _GenericRpcHandler(
+      service_implementations, multi_method_implementation,
+      request_deserializers, response_serializers)
+  if thread_pool is None:
+    effective_thread_pool = logging_pool.pool(
+        _DEFAULT_POOL_SIZE if thread_pool_size is None else thread_pool_size)
+  else:
+    effective_thread_pool = thread_pool
+  return _Server(grpc.server((generic_rpc_handler,), effective_thread_pool))
diff --git a/src/python/grpcio/grpc/beta/implementations.py b/src/python/grpcio/grpc/beta/implementations.py
index 822f593323..4ae6e7d675 100644
--- a/src/python/grpcio/grpc/beta/implementations.py
+++ b/src/python/grpcio/grpc/beta/implementations.py
@@ -35,112 +35,36 @@ import enum
 import threading  # pylint: disable=unused-import
 
 # cardinality and face are referenced from specification in this module.
-from grpc._adapter import _intermediary_low
-from grpc._adapter import _low
+import grpc
+from grpc import _auth
 from grpc._adapter import _types
-from grpc.beta import _connectivity_channel
-from grpc.beta import _server
-from grpc.beta import _stub
+from grpc.beta import _client_adaptations
+from grpc.beta import _server_adaptations
 from grpc.beta import interfaces
 from grpc.framework.common import cardinality  # pylint: disable=unused-import
 from grpc.framework.interfaces.face import face  # pylint: disable=unused-import
 
-_CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = (
-    'Exception calling channel subscription callback!')
 
+ChannelCredentials = grpc.ChannelCredentials
+ssl_channel_credentials = grpc.ssl_channel_credentials
+CallCredentials = grpc.CallCredentials
+metadata_call_credentials = grpc.metadata_call_credentials
 
-class ChannelCredentials(object):
-  """A value encapsulating the data required to create a secure Channel.
 
-  This class and its instances have no supported interface - it exists to define
-  the type of its instances and its instances exist to be passed to other
-  functions.
-  """
-
-  def __init__(self, low_credentials):
-    self._low_credentials = low_credentials
-
-
-def ssl_channel_credentials(root_certificates=None, private_key=None,
-                            certificate_chain=None):
-  """Creates a ChannelCredentials for use with an SSL-enabled Channel.
-
-  Args:
-    root_certificates: The PEM-encoded root certificates or unset to ask for
-      them to be retrieved from a default location.
-    private_key: The PEM-encoded private key to use or unset if no private key
-      should be used.
-    certificate_chain: The PEM-encoded certificate chain to use or unset if no
-      certificate chain should be used.
-
-  Returns:
-    A ChannelCredentials for use with an SSL-enabled Channel.
-  """
-  return ChannelCredentials(_low.channel_credentials_ssl(
-      root_certificates, private_key, certificate_chain))
-
-
-class CallCredentials(object):
-  """A value encapsulating data asserting an identity over an *established*
-  channel. May be composed with ChannelCredentials to always assert identity for
-  every call over that channel.
-
-  This class and its instances have no supported interface - it exists to define
-  the type of its instances and its instances exist to be passed to other
-  functions.
-  """
-
-  def __init__(self, low_credentials):
-    self._low_credentials = low_credentials
-
-
-def metadata_call_credentials(metadata_plugin, name=None):
-  """Construct CallCredentials from an interfaces.GRPCAuthMetadataPlugin.
-
-  Args:
-    metadata_plugin: An interfaces.GRPCAuthMetadataPlugin to use in constructing
-      the CallCredentials object.
-
-  Returns:
-    A CallCredentials object for use in a GRPCCallOptions object.
-  """
-  if name is None:
-    name = metadata_plugin.__name__
-  return CallCredentials(
-      _low.call_credentials_metadata_plugin(metadata_plugin, name))
-
-def composite_call_credentials(call_credentials, additional_call_credentials):
-  """Compose two CallCredentials to make a new one.
+def google_call_credentials(credentials):
+  """Construct CallCredentials from GoogleCredentials.
 
   Args:
-    call_credentials: A CallCredentials object.
-    additional_call_credentials: Another CallCredentials object to compose on
-      top of call_credentials.
+    credentials: A GoogleCredentials object from the oauth2client library.
 
   Returns:
     A CallCredentials object for use in a GRPCCallOptions object.
   """
-  return CallCredentials(
-      _low.call_credentials_composite(
-          call_credentials._low_credentials,
-          additional_call_credentials._low_credentials))
+  return metadata_call_credentials(_auth.GoogleCallCredentials(credentials))
 
-def composite_channel_credentials(channel_credentials,
-                                 additional_call_credentials):
-  """Compose ChannelCredentials on top of client credentials to make a new one.
-
-  Args:
-    channel_credentials: A ChannelCredentials object.
-    additional_call_credentials: A CallCredentials object to compose on
-      top of channel_credentials.
-
-  Returns:
-    A ChannelCredentials object for use in a GRPCCallOptions object.
-  """
-  return ChannelCredentials(
-      _low.channel_credentials_composite(
-          channel_credentials._low_credentials,
-          additional_call_credentials._low_credentials))
+access_token_call_credentials = grpc.access_token_call_credentials
+composite_call_credentials = grpc.composite_call_credentials
+composite_channel_credentials = grpc.composite_channel_credentials
 
 
 class Channel(object):
@@ -151,11 +75,8 @@ class Channel(object):
   unsupported.
   """
 
-  def __init__(self, low_channel, intermediary_low_channel):
-    self._low_channel = low_channel
-    self._intermediary_low_channel = intermediary_low_channel
-    self._connectivity_channel = _connectivity_channel.ConnectivityChannel(
-        low_channel)
+  def __init__(self, channel):
+    self._channel = channel
 
   def subscribe(self, callback, try_to_connect=None):
     """Subscribes to this Channel's connectivity.
@@ -170,7 +91,7 @@ class Channel(object):
         attempt to connect if it is not already connected and ready to conduct
         RPCs.
     """
-    self._connectivity_channel.subscribe(callback, try_to_connect)
+    self._channel.subscribe(callback, try_to_connect=try_to_connect)
 
   def unsubscribe(self, callback):
     """Unsubscribes a callback from this Channel's connectivity.
@@ -179,7 +100,7 @@ class Channel(object):
       callback: A callable previously registered with this Channel from having
         been passed to its "subscribe" method.
     """
-    self._connectivity_channel.unsubscribe(callback)
+    self._channel.unsubscribe(callback)
 
 
 def insecure_channel(host, port):
@@ -193,9 +114,9 @@ def insecure_channel(host, port):
   Returns:
     A Channel to the remote host through which RPCs may be conducted.
   """
-  intermediary_low_channel = _intermediary_low.Channel(
-      '%s:%d' % (host, port) if port else host, None)
-  return Channel(intermediary_low_channel._internal, intermediary_low_channel)  # pylint: disable=protected-access
+  channel = grpc.insecure_channel(
+      host if port is None else '%s:%d' % (host, port))
+  return Channel(channel)
 
 
 def secure_channel(host, port, channel_credentials):
@@ -210,10 +131,9 @@ def secure_channel(host, port, channel_credentials):
   Returns:
     A secure Channel to the remote host through which RPCs may be conducted.
   """
-  intermediary_low_channel = _intermediary_low.Channel(
-      '%s:%d' % (host, port) if port else host,
-      channel_credentials._low_credentials)
-  return Channel(intermediary_low_channel._internal, intermediary_low_channel)  # pylint: disable=protected-access
+  channel = grpc.secure_channel(
+      host if port is None else '%s:%d' % (host, port), channel_credentials)
+  return Channel(channel)
 
 
 class StubOptions(object):
@@ -277,12 +197,11 @@ def generic_stub(channel, options=None):
     A face.GenericStub on which RPCs can be made.
   """
   effective_options = _EMPTY_STUB_OPTIONS if options is None else options
-  return _stub.generic_stub(
-      channel._intermediary_low_channel, effective_options.host,  # pylint: disable=protected-access
-      effective_options.metadata_transformer,
+  return _client_adaptations.generic_stub(
+      channel._channel,  # pylint: disable=protected-access
+      effective_options.host, effective_options.metadata_transformer,
       effective_options.request_serializers,
-      effective_options.response_deserializers, effective_options.thread_pool,
-      effective_options.thread_pool_size)
+      effective_options.response_deserializers)
 
 
 def dynamic_stub(channel, service, cardinalities, options=None):
@@ -300,55 +219,16 @@ def dynamic_stub(channel, service, cardinalities, options=None):
     A face.DynamicStub with which RPCs can be invoked.
   """
   effective_options = StubOptions() if options is None else options
-  return _stub.dynamic_stub(
-      channel._intermediary_low_channel, effective_options.host, service,  # pylint: disable=protected-access
-      cardinalities, effective_options.metadata_transformer,
+  return _client_adaptations.dynamic_stub(
+      channel._channel,  # pylint: disable=protected-access
+      service, cardinalities, effective_options.host,
+      effective_options.metadata_transformer,
       effective_options.request_serializers,
-      effective_options.response_deserializers, effective_options.thread_pool,
-      effective_options.thread_pool_size)
-
+      effective_options.response_deserializers)
 
-class ServerCredentials(object):
-  """A value encapsulating the data required to open a secure port on a Server.
 
-  This class and its instances have no supported interface - it exists to define
-  the type of its instances and its instances exist to be passed to other
-  functions.
-  """
-
-  def __init__(self, low_credentials):
-    self._low_credentials = low_credentials
-
-
-def ssl_server_credentials(
-    private_key_certificate_chain_pairs, root_certificates=None,
-    require_client_auth=False):
-  """Creates a ServerCredentials for use with an SSL-enabled Server.
-
-  Args:
-    private_key_certificate_chain_pairs: A nonempty sequence each element of
-      which is a pair the first element of which is a PEM-encoded private key
-      and the second element of which is the corresponding PEM-encoded
-      certificate chain.
-    root_certificates: PEM-encoded client root certificates to be used for
-      verifying authenticated clients. If omitted, require_client_auth must also
-      be omitted or be False.
-    require_client_auth: A boolean indicating whether or not to require clients
-      to be authenticated. May only be True if root_certificates is not None.
-
-  Returns:
-    A ServerCredentials for use with an SSL-enabled Server.
-  """
-  if len(private_key_certificate_chain_pairs) == 0:
-    raise ValueError(
-        'At least one private key-certificate chain pairis required!')
-  elif require_client_auth and root_certificates is None:
-    raise ValueError(
-        'Illegal to require client auth without providing root certificates!')
-  else:
-    return ServerCredentials(_low.server_credentials_ssl(
-        root_certificates, private_key_certificate_chain_pairs,
-        require_client_auth))
+ServerCredentials = grpc.ServerCredentials
+ssl_server_credentials = grpc.ssl_server_credentials
 
 
 class ServerOptions(object):
@@ -421,9 +301,8 @@ def server(service_implementations, options=None):
     An interfaces.Server with which RPCs can be serviced.
   """
   effective_options = _EMPTY_SERVER_OPTIONS if options is None else options
-  return _server.server(
+  return _server_adaptations.server(
       service_implementations, effective_options.multi_method_implementation,
       effective_options.request_deserializers,
       effective_options.response_serializers, effective_options.thread_pool,
-      effective_options.thread_pool_size, effective_options.default_timeout,
-      effective_options.maximum_timeout)
+      effective_options.thread_pool_size)
diff --git a/src/python/grpcio/grpc/beta/interfaces.py b/src/python/grpcio/grpc/beta/interfaces.py
index 33ca45ac5b..90f6bbbfcc 100644
--- a/src/python/grpcio/grpc/beta/interfaces.py
+++ b/src/python/grpcio/grpc/beta/interfaces.py
@@ -30,53 +30,16 @@
 """Constants and interfaces of the Beta API of gRPC Python."""
 
 import abc
-import enum
 
 import six
 
-from grpc._adapter import _types
+import grpc
 
+ChannelConnectivity = grpc.ChannelConnectivity
+# FATAL_FAILURE was a Beta-API name for SHUTDOWN
+ChannelConnectivity.FATAL_FAILURE = ChannelConnectivity.SHUTDOWN
 
-@enum.unique
-class ChannelConnectivity(enum.Enum):
-  """Mirrors grpc_connectivity_state in the gRPC Core.
-
-  Attributes:
-    IDLE: The channel is idle.
-    CONNECTING: The channel is connecting.
-    READY: The channel is ready to conduct RPCs.
-    TRANSIENT_FAILURE: The channel has seen a failure from which it expects to
-      recover.
-    FATAL_FAILURE: The channel has seen a failure from which it cannot recover.
-  """
-  IDLE = (_types.ConnectivityState.IDLE, 'idle',)
-  CONNECTING = (_types.ConnectivityState.CONNECTING, 'connecting',)
-  READY = (_types.ConnectivityState.READY, 'ready',)
-  TRANSIENT_FAILURE = (
-      _types.ConnectivityState.TRANSIENT_FAILURE, 'transient failure',)
-  FATAL_FAILURE = (_types.ConnectivityState.FATAL_FAILURE, 'fatal failure',)
-
-
-@enum.unique
-class StatusCode(enum.Enum):
-  """Mirrors grpc_status_code in the C core."""
-  OK                  = 0
-  CANCELLED           = 1
-  UNKNOWN             = 2
-  INVALID_ARGUMENT    = 3
-  DEADLINE_EXCEEDED   = 4
-  NOT_FOUND           = 5
-  ALREADY_EXISTS      = 6
-  PERMISSION_DENIED   = 7
-  RESOURCE_EXHAUSTED  = 8
-  FAILED_PRECONDITION = 9
-  ABORTED             = 10
-  OUT_OF_RANGE        = 11
-  UNIMPLEMENTED       = 12
-  INTERNAL            = 13
-  UNAVAILABLE         = 14
-  DATA_LOSS           = 15
-  UNAUTHENTICATED     = 16
+StatusCode = grpc.StatusCode
 
 
 class GRPCCallOptions(object):
@@ -106,46 +69,9 @@ def grpc_call_options(disable_compression=False, credentials=None):
   """
   return GRPCCallOptions(disable_compression, None, credentials)
 
-
-class GRPCAuthMetadataContext(six.with_metaclass(abc.ABCMeta)):
-  """Provides information to call credentials metadata plugins.
-
-  Attributes:
-    service_url: A string URL of the service being called into.
-    method_name: A string of the fully qualified method name being called.
-  """
-
-
-class GRPCAuthMetadataPluginCallback(six.with_metaclass(abc.ABCMeta)):
-  """Callback object received by a metadata plugin."""
-
-  def __call__(self, metadata, error):
-    """Inform the gRPC runtime of the metadata to construct a CallCredentials.
-
-    Args:
-      metadata: An iterable of 2-sequences (e.g. tuples) of metadata key/value
-        pairs.
-      error: An Exception to indicate error or None to indicate success.
-    """
-    raise NotImplementedError()
-
-
-class GRPCAuthMetadataPlugin(six.with_metaclass(abc.ABCMeta)):
-  """
-  """
-
-  def __call__(self, context, callback):
-    """Invoke the plugin.
-
-    Must not block. Need only be called by the gRPC runtime.
-
-    Args:
-      context: A GRPCAuthMetadataContext providing information on what the
-        plugin is being used for.
-      callback: A GRPCAuthMetadataPluginCallback to be invoked either
-        synchronously or asynchronously.
-    """
-    raise NotImplementedError()
+GRPCAuthMetadataContext = grpc.AuthMetadataContext
+GRPCAuthMetadataPluginCallback = grpc.AuthMetadataPluginCallback
+GRPCAuthMetadataPlugin = grpc.AuthMetadataPlugin
 
 
 class GRPCServicerContext(six.with_metaclass(abc.ABCMeta)):
@@ -235,7 +161,7 @@ class Server(six.with_metaclass(abc.ABCMeta)):
     This method may be called at any time and is idempotent. Passing a smaller
     grace value than has been passed in a previous call will have the effect of
     stopping the Server sooner. Passing a larger grace value than has been
-    passed in a previous call will not have the effect of stopping the sooner
+    passed in a previous call will not have the effect of stopping the server
     later.
 
     Args:
diff --git a/src/python/grpcio/grpc/framework/foundation/future.py b/src/python/grpcio/grpc/framework/foundation/future.py
index 9210616150..6fb58eadb6 100644
--- a/src/python/grpcio/grpc/framework/foundation/future.py
+++ b/src/python/grpcio/grpc/framework/foundation/future.py
@@ -232,6 +232,6 @@ class Future(six.with_metaclass(abc.ABCMeta)):
     immediately.
 
     Args:
-      fn: A callable taking a this Future object as its single parameter.
+      fn: A callable taking this Future object as its single parameter.
     """
     raise NotImplementedError()
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index dab62530aa..b37e27c27e 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -42,39 +42,38 @@ CORE_SOURCE_FILES = [
   'src/core/lib/support/cpu_windows.c',
   'src/core/lib/support/env_linux.c',
   'src/core/lib/support/env_posix.c',
-  'src/core/lib/support/env_win32.c',
+  'src/core/lib/support/env_windows.c',
   'src/core/lib/support/histogram.c',
   'src/core/lib/support/host_port.c',
-  'src/core/lib/support/load_file.c',
   'src/core/lib/support/log.c',
   'src/core/lib/support/log_android.c',
   'src/core/lib/support/log_linux.c',
   'src/core/lib/support/log_posix.c',
-  'src/core/lib/support/log_win32.c',
+  'src/core/lib/support/log_windows.c',
   'src/core/lib/support/murmur_hash.c',
   'src/core/lib/support/slice.c',
   'src/core/lib/support/slice_buffer.c',
   'src/core/lib/support/stack_lockfree.c',
   'src/core/lib/support/string.c',
   'src/core/lib/support/string_posix.c',
-  'src/core/lib/support/string_util_win32.c',
-  'src/core/lib/support/string_win32.c',
+  'src/core/lib/support/string_util_windows.c',
+  'src/core/lib/support/string_windows.c',
   'src/core/lib/support/subprocess_posix.c',
   'src/core/lib/support/subprocess_windows.c',
   'src/core/lib/support/sync.c',
   'src/core/lib/support/sync_posix.c',
-  'src/core/lib/support/sync_win32.c',
+  'src/core/lib/support/sync_windows.c',
   'src/core/lib/support/thd.c',
   'src/core/lib/support/thd_posix.c',
-  'src/core/lib/support/thd_win32.c',
+  'src/core/lib/support/thd_windows.c',
   'src/core/lib/support/time.c',
   'src/core/lib/support/time_posix.c',
   'src/core/lib/support/time_precise.c',
-  'src/core/lib/support/time_win32.c',
+  'src/core/lib/support/time_windows.c',
   'src/core/lib/support/tls_pthread.c',
   'src/core/lib/support/tmpfile_msys.c',
   'src/core/lib/support/tmpfile_posix.c',
-  'src/core/lib/support/tmpfile_win32.c',
+  'src/core/lib/support/tmpfile_windows.c',
   'src/core/lib/support/wrap_memcpy.c',
   'src/core/lib/surface/init.c',
   'src/core/lib/channel/channel_args.c',
@@ -84,7 +83,7 @@ CORE_SOURCE_FILES = [
   'src/core/lib/channel/connected_channel.c',
   'src/core/lib/channel/http_client_filter.c',
   'src/core/lib/channel/http_server_filter.c',
-  'src/core/lib/compression/compression_algorithm.c',
+  'src/core/lib/compression/compression.c',
   'src/core/lib/compression/message_compress.c',
   'src/core/lib/debug/trace.c',
   'src/core/lib/http/format_request.c',
@@ -94,7 +93,10 @@ CORE_SOURCE_FILES = [
   'src/core/lib/iomgr/endpoint.c',
   'src/core/lib/iomgr/endpoint_pair_posix.c',
   'src/core/lib/iomgr/endpoint_pair_windows.c',
+  'src/core/lib/iomgr/error.c',
+  'src/core/lib/iomgr/ev_epoll_linux.c',
   'src/core/lib/iomgr/ev_poll_and_epoll_posix.c',
+  'src/core/lib/iomgr/ev_poll_posix.c',
   'src/core/lib/iomgr/ev_posix.c',
   'src/core/lib/iomgr/exec_ctx.c',
   'src/core/lib/iomgr/executor.c',
@@ -102,6 +104,9 @@ CORE_SOURCE_FILES = [
   'src/core/lib/iomgr/iomgr.c',
   'src/core/lib/iomgr/iomgr_posix.c',
   'src/core/lib/iomgr/iomgr_windows.c',
+  'src/core/lib/iomgr/load_file.c',
+  'src/core/lib/iomgr/network_status_tracker.c',
+  'src/core/lib/iomgr/polling_entity.c',
   'src/core/lib/iomgr/pollset_set_windows.c',
   'src/core/lib/iomgr/pollset_windows.c',
   'src/core/lib/iomgr/resolve_address_posix.c',
@@ -159,6 +164,7 @@ CORE_SOURCE_FILES = [
   'src/core/lib/transport/transport.c',
   'src/core/lib/transport/transport_op_string.c',
   'src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c',
+  'src/core/ext/transport/chttp2/transport/bin_decoder.c',
   'src/core/ext/transport/chttp2/transport/bin_encoder.c',
   'src/core/ext/transport/chttp2/transport/chttp2_plugin.c',
   'src/core/ext/transport/chttp2/transport/chttp2_transport.c',
@@ -182,20 +188,29 @@ CORE_SOURCE_FILES = [
   'src/core/ext/transport/chttp2/transport/writing.c',
   'src/core/ext/transport/chttp2/alpn/alpn.c',
   'src/core/lib/http/httpcli_security_connector.c',
-  'src/core/lib/security/b64.c',
-  'src/core/lib/security/client_auth_filter.c',
-  'src/core/lib/security/credentials.c',
-  'src/core/lib/security/credentials_metadata.c',
-  'src/core/lib/security/credentials_posix.c',
-  'src/core/lib/security/credentials_win32.c',
-  'src/core/lib/security/google_default_credentials.c',
-  'src/core/lib/security/handshake.c',
-  'src/core/lib/security/json_token.c',
-  'src/core/lib/security/jwt_verifier.c',
-  'src/core/lib/security/secure_endpoint.c',
-  'src/core/lib/security/security_connector.c',
-  'src/core/lib/security/security_context.c',
-  'src/core/lib/security/server_auth_filter.c',
+  'src/core/lib/security/context/security_context.c',
+  'src/core/lib/security/credentials/composite/composite_credentials.c',
+  'src/core/lib/security/credentials/credentials.c',
+  'src/core/lib/security/credentials/credentials_metadata.c',
+  'src/core/lib/security/credentials/fake/fake_credentials.c',
+  'src/core/lib/security/credentials/google_default/credentials_posix.c',
+  'src/core/lib/security/credentials/google_default/credentials_windows.c',
+  'src/core/lib/security/credentials/google_default/google_default_credentials.c',
+  'src/core/lib/security/credentials/iam/iam_credentials.c',
+  'src/core/lib/security/credentials/jwt/json_token.c',
+  'src/core/lib/security/credentials/jwt/jwt_credentials.c',
+  'src/core/lib/security/credentials/jwt/jwt_verifier.c',
+  'src/core/lib/security/credentials/oauth2/oauth2_credentials.c',
+  'src/core/lib/security/credentials/plugin/plugin_credentials.c',
+  'src/core/lib/security/credentials/ssl/ssl_credentials.c',
+  'src/core/lib/security/transport/client_auth_filter.c',
+  'src/core/lib/security/transport/handshake.c',
+  'src/core/lib/security/transport/secure_endpoint.c',
+  'src/core/lib/security/transport/security_connector.c',
+  'src/core/lib/security/transport/server_auth_filter.c',
+  'src/core/lib/security/transport/tsi_error.c',
+  'src/core/lib/security/util/b64.c',
+  'src/core/lib/security/util/json_util.c',
   'src/core/lib/surface/init_secure.c',
   'src/core/lib/tsi/fake_transport_security.c',
   'src/core/lib/tsi/ssl_transport_security.c',
@@ -221,9 +236,11 @@ CORE_SOURCE_FILES = [
   'src/core/ext/client_config/subchannel_index.c',
   'src/core/ext/client_config/uri_parser.c',
   'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
+  'src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c',
   'src/core/ext/transport/chttp2/client/insecure/channel_create.c',
+  'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c',
   'src/core/ext/lb_policy/grpclb/load_balancer_api.c',
-  'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c',
+  'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c',
   'third_party/nanopb/pb_common.c',
   'third_party/nanopb/pb_decode.c',
   'third_party/nanopb/pb_encode.c',
@@ -231,7 +248,10 @@ CORE_SOURCE_FILES = [
   'src/core/ext/lb_policy/round_robin/round_robin.c',
   'src/core/ext/resolver/dns/native/dns_resolver.c',
   'src/core/ext/resolver/sockaddr/sockaddr_resolver.c',
+  'src/core/ext/load_reporting/load_reporting.c',
+  'src/core/ext/load_reporting/load_reporting_filter.c',
   'src/core/ext/census/context.c',
+  'src/core/ext/census/gen/census.pb.c',
   'src/core/ext/census/grpc_context.c',
   'src/core/ext/census/grpc_filter.c',
   'src/core/ext/census/grpc_plugin.c',
diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py
index eedb381185..0f4db9d972 100644
--- a/src/python/grpcio/grpc_version.py
+++ b/src/python/grpcio/grpc_version.py
@@ -29,4 +29,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!!
 
-VERSION='0.14.2rc1'
+VERSION='0.16.0.dev0'
diff --git a/src/python/grpcio/tests/health_check/__init__.py b/src/python/grpcio/tests/health_check/__init__.py
new file mode 100644
index 0000000000..100a624dc9
--- /dev/null
+++ b/src/python/grpcio/tests/health_check/__init__.py
@@ -0,0 +1,28 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/python/grpcio/tests/health_check/_health_servicer_test.py b/src/python/grpcio/tests/health_check/_health_servicer_test.py
new file mode 100644
index 0000000000..1b63388663
--- /dev/null
+++ b/src/python/grpcio/tests/health_check/_health_servicer_test.py
@@ -0,0 +1,75 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests of grpc_health.health.v1.health."""
+
+import unittest
+
+from grpc_health.health.v1 import health
+from grpc_health.health.v1 import health_pb2
+
+
+class HealthServicerTest(unittest.TestCase):
+
+  def setUp(self):
+    self.servicer = health.HealthServicer()
+    self.servicer.set('', health_pb2.HealthCheckResponse.SERVING)
+    self.servicer.set('grpc.test.TestServiceServing',
+                      health_pb2.HealthCheckResponse.SERVING)
+    self.servicer.set('grpc.test.TestServiceUnknown',
+                      health_pb2.HealthCheckResponse.UNKNOWN)
+    self.servicer.set('grpc.test.TestServiceNotServing',
+                      health_pb2.HealthCheckResponse.NOT_SERVING)
+
+  def test_empty_service(self):
+    request = health_pb2.HealthCheckRequest()
+    resp = self.servicer.Check(request, None)
+    self.assertEqual(resp.status, health_pb2.HealthCheckResponse.SERVING)
+
+  def test_serving_service(self):
+    request = health_pb2.HealthCheckRequest(
+        service='grpc.test.TestServiceServing')
+    resp = self.servicer.Check(request, None)
+    self.assertEqual(resp.status, health_pb2.HealthCheckResponse.SERVING)
+
+  def test_unknown_serivce(self):
+    request = health_pb2.HealthCheckRequest(
+        service='grpc.test.TestServiceUnknown')
+    resp = self.servicer.Check(request, None)
+    self.assertEqual(resp.status, health_pb2.HealthCheckResponse.UNKNOWN)
+
+  def test_not_serving_service(self):
+    request = health_pb2.HealthCheckRequest(
+        service='grpc.test.TestServiceNotServing')
+    resp = self.servicer.Check(request, None)
+    self.assertEqual(resp.status, health_pb2.HealthCheckResponse.NOT_SERVING)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/interop/client.py b/src/python/grpcio/tests/interop/client.py
index db29eb4aa7..8aa1ce30c1 100644
--- a/src/python/grpcio/tests/interop/client.py
+++ b/src/python/grpcio/tests/interop/client.py
@@ -65,39 +65,37 @@ def _args():
       help='email address of the default service account', type=str)
   return parser.parse_args()
 
-def _oauth_access_token(args):
-  credentials = oauth2client_client.GoogleCredentials.get_application_default()
-  scoped_credentials = credentials.create_scoped([args.oauth_scope])
-  return scoped_credentials.get_access_token().access_token
 
 def _stub(args):
-  if args.oauth_scope:
-    if args.test_case == 'oauth2_auth_token':
-      # TODO(jtattermusch): This testcase sets the auth metadata key-value
-      # manually, which also means that the user would need to do the same
-      # thing every time he/she would like to use and out of band oauth token.
-      # The transformer function that produces the metadata key-value from
-      # the access token should be provided by gRPC auth library.
-      access_token = _oauth_access_token(args)
-      metadata_transformer = lambda x: [
-          ('authorization', 'Bearer %s' % access_token)]
-    else:
-      metadata_transformer = lambda x: [
-          ('authorization', 'Bearer %s' % _oauth_access_token(args))]
+  if args.test_case == 'oauth2_auth_token':
+    creds = oauth2client_client.GoogleCredentials.get_application_default()
+    scoped_creds = creds.create_scoped([args.oauth_scope])
+    access_token = scoped_creds.get_access_token().access_token
+    call_creds = implementations.access_token_call_credentials(access_token)
+  elif args.test_case == 'compute_engine_creds':
+    creds = oauth2client_client.GoogleCredentials.get_application_default()
+    scoped_creds = creds.create_scoped([args.oauth_scope])
+    call_creds = implementations.google_call_credentials(scoped_creds)
+  elif args.test_case == 'jwt_token_creds':
+    creds = oauth2client_client.GoogleCredentials.get_application_default()
+    call_creds = implementations.google_call_credentials(creds)
   else:
-    metadata_transformer = lambda x: []
+    call_creds = None
   if args.use_tls:
     if args.use_test_ca:
       root_certificates = resources.test_root_certificates()
     else:
       root_certificates = None  # will load default roots.
 
+    channel_creds = implementations.ssl_channel_credentials(root_certificates)
+    if call_creds is not None:
+      channel_creds = implementations.composite_channel_credentials(
+          channel_creds, call_creds)
+
     channel = test_utilities.not_really_secure_channel(
-        args.server_host, args.server_port,
-        implementations.ssl_channel_credentials(root_certificates),
+        args.server_host, args.server_port, channel_creds,
         args.server_host_override)
-    stub = test_pb2.beta_create_TestService_stub(
-        channel, metadata_transformer=metadata_transformer)
+    stub = test_pb2.beta_create_TestService_stub(channel)
   else:
     channel = implementations.insecure_channel(
         args.server_host, args.server_port)
diff --git a/src/python/grpcio/tests/interop/methods.py b/src/python/grpcio/tests/interop/methods.py
index 67862ed7d3..86aa0495a2 100644
--- a/src/python/grpcio/tests/interop/methods.py
+++ b/src/python/grpcio/tests/interop/methods.py
@@ -39,6 +39,8 @@ import time
 
 from oauth2client import client as oauth2client_client
 
+from grpc.beta import implementations
+from grpc.beta import interfaces
 from grpc.framework.common import cardinality
 from grpc.framework.interfaces.face import face
 
@@ -77,10 +79,11 @@ class TestService(test_pb2.BetaTestServiceServicer):
 
   def FullDuplexCall(self, request_iterator, context):
     for request in request_iterator:
-      yield messages_pb2.StreamingOutputCallResponse(
-          payload=messages_pb2.Payload(
-              type=request.payload.type,
-              body=b'\x00' * request.response_parameters[0].size))
+      for response_parameters in request.response_parameters:
+        yield messages_pb2.StreamingOutputCallResponse(
+            payload=messages_pb2.Payload(
+                type=request.payload.type,
+                body=b'\x00' * response_parameters.size))
 
   # NOTE(nathaniel): Apparently this is the same as the full-duplex call?
   # NOTE(atash): It isn't even called in the interop spec (Oct 22 2015)...
@@ -88,13 +91,15 @@ class TestService(test_pb2.BetaTestServiceServicer):
     return self.FullDuplexCall(request_iterator, context)
 
 
-def _large_unary_common_behavior(stub, fill_username, fill_oauth_scope):
+def _large_unary_common_behavior(stub, fill_username, fill_oauth_scope,
+                                 protocol_options=None):
   with stub:
     request = messages_pb2.SimpleRequest(
         response_type=messages_pb2.COMPRESSABLE, response_size=314159,
         payload=messages_pb2.Payload(body=b'\x00' * 271828),
         fill_username=fill_username, fill_oauth_scope=fill_oauth_scope)
-    response_future = stub.UnaryCall.future(request, _TIMEOUT)
+    response_future = stub.UnaryCall.future(request, _TIMEOUT,
+                                            protocol_options=protocol_options)
     response = response_future.result()
     if response.payload.type is not messages_pb2.COMPRESSABLE:
       raise ValueError(
@@ -303,7 +308,34 @@ def _oauth2_auth_token(stub, args):
   if args.oauth_scope.find(response.oauth_scope) == -1:
     raise ValueError(
         'expected to find oauth scope "%s" in received "%s"' %
-            (response.oauth_scope, args.oauth_scope))
+        (response.oauth_scope, args.oauth_scope))
+
+
+def _jwt_token_creds(stub, args):
+  json_key_filename = os.environ[
+      oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS]
+  wanted_email = json.load(open(json_key_filename, 'rb'))['client_email']
+  response = _large_unary_common_behavior(stub, True, False)
+  if wanted_email != response.username:
+    raise ValueError(
+        'expected username %s, got %s' % (wanted_email, response.username))
+
+
+def _per_rpc_creds(stub, args):
+  json_key_filename = os.environ[
+      oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS]
+  wanted_email = json.load(open(json_key_filename, 'rb'))['client_email']
+  credentials = oauth2client_client.GoogleCredentials.get_application_default()
+  scoped_credentials = credentials.create_scoped([args.oauth_scope])
+  call_creds = implementations.google_call_credentials(scoped_credentials)
+  options = interfaces.grpc_call_options(disable_compression=False,
+                                         credentials=call_creds)
+  response = _large_unary_common_behavior(stub, True, False,
+                                          protocol_options=options)
+  if wanted_email != response.username:
+    raise ValueError(
+        'expected username %s, got %s' % (wanted_email, response.username))
+
 
 @enum.unique
 class TestCase(enum.Enum):
@@ -317,6 +349,8 @@ class TestCase(enum.Enum):
   EMPTY_STREAM = 'empty_stream'
   COMPUTE_ENGINE_CREDS = 'compute_engine_creds'
   OAUTH2_AUTH_TOKEN = 'oauth2_auth_token'
+  JWT_TOKEN_CREDS = 'jwt_token_creds'
+  PER_RPC_CREDS = 'per_rpc_creds'
   TIMEOUT_ON_SLEEPING_SERVER = 'timeout_on_sleeping_server'
 
   def test_interoperability(self, stub, args):
@@ -342,5 +376,9 @@ class TestCase(enum.Enum):
       _compute_engine_creds(stub, args)
     elif self is TestCase.OAUTH2_AUTH_TOKEN:
       _oauth2_auth_token(stub, args)
+    elif self is TestCase.JWT_TOKEN_CREDS:
+      _jwt_token_creds(stub, args)
+    elif self is TestCase.PER_RPC_CREDS:
+      _per_rpc_creds(stub, args)
     else:
       raise NotImplementedError('Test case "%s" not implemented!' % self.name)
diff --git a/src/python/grpcio/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio/tests/protoc_plugin/_python_plugin_test.py
new file mode 100644
index 0000000000..1c9cbb0d0c
--- /dev/null
+++ b/src/python/grpcio/tests/protoc_plugin/_python_plugin_test.py
@@ -0,0 +1,583 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import collections
+from concurrent import futures
+import contextlib
+import distutils.spawn
+import errno
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import unittest
+
+from six import moves
+
+import grpc
+from tests.unit.framework.common import test_constants
+
+# Identifiers of entities we expect to find in the generated module.
+STUB_IDENTIFIER = 'TestServiceStub'
+SERVICER_IDENTIFIER = 'TestServiceServicer'
+ADD_SERVICER_TO_SERVER_IDENTIFIER = 'add_TestServiceServicer_to_server'
+
+
+class _ServicerMethods(object):
+
+  def __init__(self, response_pb2, payload_pb2):
+    self._condition = threading.Condition()
+    self._paused = False
+    self._fail = False
+    self._response_pb2 = response_pb2
+    self._payload_pb2 = payload_pb2
+
+  @contextlib.contextmanager
+  def pause(self):  # pylint: disable=invalid-name
+    with self._condition:
+      self._paused = True
+    yield
+    with self._condition:
+      self._paused = False
+      self._condition.notify_all()
+
+  @contextlib.contextmanager
+  def fail(self):  # pylint: disable=invalid-name
+    with self._condition:
+      self._fail = True
+    yield
+    with self._condition:
+      self._fail = False
+
+  def _control(self):  # pylint: disable=invalid-name
+    with self._condition:
+      if self._fail:
+        raise ValueError()
+      while self._paused:
+        self._condition.wait()
+
+  def UnaryCall(self, request, unused_rpc_context):
+    response = self._response_pb2.SimpleResponse()
+    response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+    response.payload.payload_compressable = 'a' * request.response_size
+    self._control()
+    return response
+
+  def StreamingOutputCall(self, request, unused_rpc_context):
+    for parameter in request.response_parameters:
+      response = self._response_pb2.StreamingOutputCallResponse()
+      response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+      response.payload.payload_compressable = 'a' * parameter.size
+      self._control()
+      yield response
+
+  def StreamingInputCall(self, request_iter, unused_rpc_context):
+    response = self._response_pb2.StreamingInputCallResponse()
+    aggregated_payload_size = 0
+    for request in request_iter:
+      aggregated_payload_size += len(request.payload.payload_compressable)
+    response.aggregated_payload_size = aggregated_payload_size
+    self._control()
+    return response
+
+  def FullDuplexCall(self, request_iter, unused_rpc_context):
+    for request in request_iter:
+      for parameter in request.response_parameters:
+        response = self._response_pb2.StreamingOutputCallResponse()
+        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+        response.payload.payload_compressable = 'a' * parameter.size
+        self._control()
+        yield response
+
+  def HalfDuplexCall(self, request_iter, unused_rpc_context):
+    responses = []
+    for request in request_iter:
+      for parameter in request.response_parameters:
+        response = self._response_pb2.StreamingOutputCallResponse()
+        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+        response.payload.payload_compressable = 'a' * parameter.size
+        self._control()
+        responses.append(response)
+    for response in responses:
+      yield response
+
+
+class _Service(
+    collections.namedtuple(
+      '_Service', ('servicer_methods', 'server', 'stub',))):
+  """A live and running service.
+
+  Attributes:
+    servicer_methods: The _ServicerMethods servicing RPCs.
+    server: The grpc.Server servicing RPCs.
+    stub: A stub on which to invoke RPCs.
+  """
+      
+
+def _CreateService(service_pb2, response_pb2, payload_pb2):
+  """Provides a servicer backend and a stub.
+
+  Args:
+    service_pb2: The service_pb2 module generated by this test.
+    response_pb2: The response_pb2 module generated by this test.
+    payload_pb2: The payload_pb2 module generated by this test.
+
+  Returns:
+    A _Service with which to test RPCs.
+  """
+  servicer_methods = _ServicerMethods(response_pb2, payload_pb2)
+
+  class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)):
+
+    def UnaryCall(self, request, context):
+      return servicer_methods.UnaryCall(request, context)
+
+    def StreamingOutputCall(self, request, context):
+      return servicer_methods.StreamingOutputCall(request, context)
+
+    def StreamingInputCall(self, request_iter, context):
+      return servicer_methods.StreamingInputCall(request_iter, context)
+
+    def FullDuplexCall(self, request_iter, context):
+      return servicer_methods.FullDuplexCall(request_iter, context)
+
+    def HalfDuplexCall(self, request_iter, context):
+      return servicer_methods.HalfDuplexCall(request_iter, context)
+
+  server = grpc.server(
+      (), futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE))
+  getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server)
+  port = server.add_insecure_port('[::]:0')
+  server.start()
+  channel = grpc.insecure_channel('localhost:{}'.format(port))
+  stub = getattr(service_pb2, STUB_IDENTIFIER)(channel)
+  return _Service(servicer_methods, server, stub)
+
+
+def _CreateIncompleteService(service_pb2):
+  """Provides a servicer backend that fails to implement methods and its stub.
+
+  Args:
+    service_pb2: The service_pb2 module generated by this test.
+
+  Returns:
+    A _Service with which to test RPCs. The returned _Service's
+      servicer_methods implements none of the methods required of it.
+  """
+
+  class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)):
+    pass
+
+  server = grpc.server(
+      (), futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE))
+  getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server)
+  port = server.add_insecure_port('[::]:0')
+  server.start()
+  channel = grpc.insecure_channel('localhost:{}'.format(port))
+  stub = getattr(service_pb2, STUB_IDENTIFIER)(channel)
+  return _Service(None, server, stub)
+
+
+def _streaming_input_request_iterator(request_pb2, payload_pb2):
+  for _ in range(3):
+    request = request_pb2.StreamingInputCallRequest()
+    request.payload.payload_type = payload_pb2.COMPRESSABLE
+    request.payload.payload_compressable = 'a'
+    yield request
+
+
+def _streaming_output_request(request_pb2):
+  request = request_pb2.StreamingOutputCallRequest()
+  sizes = [1, 2, 3]
+  request.response_parameters.add(size=sizes[0], interval_us=0)
+  request.response_parameters.add(size=sizes[1], interval_us=0)
+  request.response_parameters.add(size=sizes[2], interval_us=0)
+  return request
+
+
+def _full_duplex_request_iterator(request_pb2):
+  request = request_pb2.StreamingOutputCallRequest()
+  request.response_parameters.add(size=1, interval_us=0)
+  yield request
+  request = request_pb2.StreamingOutputCallRequest()
+  request.response_parameters.add(size=2, interval_us=0)
+  request.response_parameters.add(size=3, interval_us=0)
+  yield request
+
+
+class PythonPluginTest(unittest.TestCase):
+  """Test case for the gRPC Python protoc-plugin.
+
+  While reading these tests, remember that the futures API
+  (`stub.method.future()`) only gives futures for the *response-unary*
+  methods and does not exist for response-streaming methods.
+  """
+
+  def setUp(self):
+    # Assume that the appropriate protoc and grpc_python_plugins are on the
+    # path.
+    protoc_command = 'protoc'
+    protoc_plugin_filename = distutils.spawn.find_executable(
+        'grpc_python_plugin')
+    if not os.path.isfile(protoc_command):
+      # Assume that if we haven't built protoc that it's on the system.
+      protoc_command = 'protoc'
+
+    # Ensure that the output directory exists.
+    self.outdir = tempfile.mkdtemp()
+
+    # Find all proto files
+    paths = []
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    proto_dir = os.path.join(root_dir, 'protos')
+    for walk_root, _, filenames in os.walk(proto_dir):
+      for filename in filenames:
+        if filename.endswith('.proto'):
+          path = os.path.join(walk_root, filename)
+          paths.append(path)
+
+    # Invoke protoc with the plugin.
+    cmd = [
+        protoc_command,
+        '--plugin=protoc-gen-python-grpc=%s' % protoc_plugin_filename,
+        '-I %s' % root_dir,
+        '--python_out=%s' % self.outdir,
+        '--python-grpc_out=%s' % self.outdir
+    ] + paths
+    subprocess.check_call(' '.join(cmd), shell=True, env=os.environ,
+                          cwd=os.path.dirname(os.path.realpath(__file__)))
+
+    # Generated proto directories dont include __init__.py, but
+    # these are needed for python package resolution
+    for walk_root, _, _ in os.walk(os.path.join(self.outdir, 'protos')):
+      path = os.path.join(walk_root, '__init__.py')
+      open(path, 'a').close()
+
+    sys.path.insert(0, self.outdir)
+
+    import protos.payload.test_payload_pb2 as payload_pb2
+    import protos.requests.r.test_requests_pb2 as request_pb2
+    import protos.responses.test_responses_pb2 as response_pb2
+    import protos.service.test_service_pb2 as service_pb2
+    self._payload_pb2 = payload_pb2
+    self._request_pb2 = request_pb2
+    self._response_pb2 = response_pb2
+    self._service_pb2 = service_pb2
+
+  def tearDown(self):
+    try:
+      shutil.rmtree(self.outdir)
+    except OSError as exc:
+      if exc.errno != errno.ENOENT:
+        raise
+    sys.path.remove(self.outdir)
+
+  def testImportAttributes(self):
+    # check that we can access the generated module and its members.
+    self.assertIsNotNone(
+        getattr(self._service_pb2, STUB_IDENTIFIER, None))
+    self.assertIsNotNone(
+        getattr(self._service_pb2, SERVICER_IDENTIFIER, None))
+    self.assertIsNotNone(
+        getattr(self._service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER, None))
+
+  def testUpDown(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    self.assertIsNotNone(service.servicer_methods)
+    self.assertIsNotNone(service.server)
+    self.assertIsNotNone(service.stub)
+
+  def testIncompleteServicer(self):
+    service = _CreateIncompleteService(self._service_pb2)
+    request = self._request_pb2.SimpleRequest(response_size=13)
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      service.stub.UnaryCall(request)
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.UNIMPLEMENTED)
+
+  def testUnaryCall(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = self._request_pb2.SimpleRequest(response_size=13)
+    response = service.stub.UnaryCall(request)
+    expected_response = service.servicer_methods.UnaryCall(
+        request, 'not a real context!')
+    self.assertEqual(expected_response, response)
+
+  def testUnaryCallFuture(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = self._request_pb2.SimpleRequest(response_size=13)
+    # Check that the call does not block waiting for the server to respond.
+    with service.servicer_methods.pause():
+      response_future = service.stub.UnaryCall.future(request)
+    response = response_future.result()
+    expected_response = service.servicer_methods.UnaryCall(
+        request, 'not a real RpcContext!')
+    self.assertEqual(expected_response, response)
+
+  def testUnaryCallFutureExpired(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = self._request_pb2.SimpleRequest(response_size=13)
+    with service.servicer_methods.pause():
+      response_future = service.stub.UnaryCall.future(
+          request, timeout=test_constants.SHORT_TIMEOUT)
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        response_future.result()
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+    self.assertIs(response_future.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+
+  def testUnaryCallFutureCancelled(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = self._request_pb2.SimpleRequest(response_size=13)
+    with service.servicer_methods.pause():
+      response_future = service.stub.UnaryCall.future(request)
+      response_future.cancel()
+    self.assertTrue(response_future.cancelled())
+    self.assertIs(response_future.code(), grpc.StatusCode.CANCELLED)
+
+  def testUnaryCallFutureFailed(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = self._request_pb2.SimpleRequest(response_size=13)
+    with service.servicer_methods.fail():
+      response_future = service.stub.UnaryCall.future(request)
+      self.assertIsNotNone(response_future.exception())
+    self.assertIs(response_future.code(), grpc.StatusCode.UNKNOWN)
+
+  def testStreamingOutputCall(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request(self._request_pb2)
+    responses = service.stub.StreamingOutputCall(request)
+    expected_responses = service.servicer_methods.StreamingOutputCall(
+        request, 'not a real RpcContext!')
+    for expected_response, response in moves.zip_longest(
+        expected_responses, responses):
+      self.assertEqual(expected_response, response)
+
+  def testStreamingOutputCallExpired(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request(self._request_pb2)
+    with service.servicer_methods.pause():
+      responses = service.stub.StreamingOutputCall(
+          request, timeout=test_constants.SHORT_TIMEOUT)
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        list(responses)
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+
+  def testStreamingOutputCallCancelled(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request(self._request_pb2)
+    responses = service.stub.StreamingOutputCall(request)
+    next(responses)
+    responses.cancel()
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      next(responses)
+    self.assertIs(responses.code(), grpc.StatusCode.CANCELLED)
+
+  def testStreamingOutputCallFailed(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request(self._request_pb2)
+    with service.servicer_methods.fail():
+      responses = service.stub.StreamingOutputCall(request)
+      self.assertIsNotNone(responses)
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        next(responses)
+    self.assertIs(exception_context.exception.code(), grpc.StatusCode.UNKNOWN)
+
+  def testStreamingInputCall(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    response = service.stub.StreamingInputCall(
+        _streaming_input_request_iterator(
+            self._request_pb2, self._payload_pb2))
+    expected_response = service.servicer_methods.StreamingInputCall(
+        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        'not a real RpcContext!')
+    self.assertEqual(expected_response, response)
+
+  def testStreamingInputCallFuture(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with service.servicer_methods.pause():
+      response_future = service.stub.StreamingInputCall.future(
+          _streaming_input_request_iterator(
+              self._request_pb2, self._payload_pb2))
+    response = response_future.result()
+    expected_response = service.servicer_methods.StreamingInputCall(
+        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        'not a real RpcContext!')
+    self.assertEqual(expected_response, response)
+
+  def testStreamingInputCallFutureExpired(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with service.servicer_methods.pause():
+      response_future = service.stub.StreamingInputCall.future(
+          _streaming_input_request_iterator(
+              self._request_pb2, self._payload_pb2),
+          timeout=test_constants.SHORT_TIMEOUT)
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        response_future.result()
+    self.assertIsInstance(response_future.exception(), grpc.RpcError)
+    self.assertIs(
+        response_future.exception().code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+
+  def testStreamingInputCallFutureCancelled(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with service.servicer_methods.pause():
+      response_future = service.stub.StreamingInputCall.future(
+          _streaming_input_request_iterator(
+              self._request_pb2, self._payload_pb2))
+      response_future.cancel()
+    self.assertTrue(response_future.cancelled())
+    with self.assertRaises(grpc.FutureCancelledError):
+      response_future.result()
+
+  def testStreamingInputCallFutureFailed(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with service.servicer_methods.fail():
+      response_future = service.stub.StreamingInputCall.future(
+          _streaming_input_request_iterator(
+              self._request_pb2, self._payload_pb2))
+      self.assertIsNotNone(response_future.exception())
+      self.assertIs(response_future.code(), grpc.StatusCode.UNKNOWN)
+
+  def testFullDuplexCall(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    responses = service.stub.FullDuplexCall(
+        _full_duplex_request_iterator(self._request_pb2))
+    expected_responses = service.servicer_methods.FullDuplexCall(
+        _full_duplex_request_iterator(self._request_pb2),
+        'not a real RpcContext!')
+    for expected_response, response in moves.zip_longest(
+        expected_responses, responses):
+      self.assertEqual(expected_response, response)
+
+  def testFullDuplexCallExpired(self):
+    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with service.servicer_methods.pause():
+      responses = service.stub.FullDuplexCall(
+          request_iterator, timeout=test_constants.SHORT_TIMEOUT)
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        list(responses)
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+
+  def testFullDuplexCallCancelled(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    responses = service.stub.FullDuplexCall(request_iterator)
+    next(responses)
+    responses.cancel()
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      next(responses)
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.CANCELLED)
+
+  def testFullDuplexCallFailed(self):
+    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with service.servicer_methods.fail():
+      responses = service.stub.FullDuplexCall(request_iterator)
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        next(responses)
+    self.assertIs(exception_context.exception.code(), grpc.StatusCode.UNKNOWN)
+
+  def testHalfDuplexCall(self):
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    def half_duplex_request_iterator():
+      request = self._request_pb2.StreamingOutputCallRequest()
+      request.response_parameters.add(size=1, interval_us=0)
+      yield request
+      request = self._request_pb2.StreamingOutputCallRequest()
+      request.response_parameters.add(size=2, interval_us=0)
+      request.response_parameters.add(size=3, interval_us=0)
+      yield request
+    responses = service.stub.HalfDuplexCall(half_duplex_request_iterator())
+    expected_responses = service.servicer_methods.HalfDuplexCall(
+        half_duplex_request_iterator(), 'not a real RpcContext!')
+    for expected_response, response in moves.zip_longest(
+        expected_responses, responses):
+      self.assertEqual(expected_response, response)
+
+  def testHalfDuplexCallWedged(self):
+    condition = threading.Condition()
+    wait_cell = [False]
+    @contextlib.contextmanager
+    def wait():  # pylint: disable=invalid-name
+      # Where's Python 3's 'nonlocal' statement when you need it?
+      with condition:
+        wait_cell[0] = True
+      yield
+      with condition:
+        wait_cell[0] = False
+        condition.notify_all()
+    def half_duplex_request_iterator():
+      request = self._request_pb2.StreamingOutputCallRequest()
+      request.response_parameters.add(size=1, interval_us=0)
+      yield request
+      with condition:
+        while wait_cell[0]:
+          condition.wait()
+    service = _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2)
+    with wait():
+      responses = service.stub.HalfDuplexCall(
+          half_duplex_request_iterator(), timeout=test_constants.SHORT_TIMEOUT)
+      # half-duplex waits for the client to send all info
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        next(responses)
+    self.assertIs(
+        exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py b/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py
index 3dc3042e38..7466f88059 100644
--- a/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py
+++ b/src/python/grpcio/tests/protoc_plugin/beta_python_plugin_test.py
@@ -59,11 +59,12 @@ STUB_FACTORY_IDENTIFIER = 'beta_create_TestService_stub'
 
 class _ServicerMethods(object):
 
-  def __init__(self, test_pb2):
+  def __init__(self, response_pb2, payload_pb2):
     self._condition = threading.Condition()
     self._paused = False
     self._fail = False
-    self._test_pb2 = test_pb2
+    self._response_pb2 = response_pb2
+    self._payload_pb2 = payload_pb2
 
   @contextlib.contextmanager
   def pause(self):  # pylint: disable=invalid-name
@@ -90,22 +91,22 @@ class _ServicerMethods(object):
         self._condition.wait()
 
   def UnaryCall(self, request, unused_rpc_context):
-    response = self._test_pb2.SimpleResponse()
-    response.payload.payload_type = self._test_pb2.COMPRESSABLE
+    response = self._response_pb2.SimpleResponse()
+    response.payload.payload_type = self._payload_pb2.COMPRESSABLE
     response.payload.payload_compressable = 'a' * request.response_size
     self._control()
     return response
 
   def StreamingOutputCall(self, request, unused_rpc_context):
     for parameter in request.response_parameters:
-      response = self._test_pb2.StreamingOutputCallResponse()
-      response.payload.payload_type = self._test_pb2.COMPRESSABLE
+      response = self._response_pb2.StreamingOutputCallResponse()
+      response.payload.payload_type = self._payload_pb2.COMPRESSABLE
       response.payload.payload_compressable = 'a' * parameter.size
       self._control()
       yield response
 
   def StreamingInputCall(self, request_iter, unused_rpc_context):
-    response = self._test_pb2.StreamingInputCallResponse()
+    response = self._response_pb2.StreamingInputCallResponse()
     aggregated_payload_size = 0
     for request in request_iter:
       aggregated_payload_size += len(request.payload.payload_compressable)
@@ -116,8 +117,8 @@ class _ServicerMethods(object):
   def FullDuplexCall(self, request_iter, unused_rpc_context):
     for request in request_iter:
       for parameter in request.response_parameters:
-        response = self._test_pb2.StreamingOutputCallResponse()
-        response.payload.payload_type = self._test_pb2.COMPRESSABLE
+        response = self._response_pb2.StreamingOutputCallResponse()
+        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
         response.payload.payload_compressable = 'a' * parameter.size
         self._control()
         yield response
@@ -126,8 +127,8 @@ class _ServicerMethods(object):
     responses = []
     for request in request_iter:
       for parameter in request.response_parameters:
-        response = self._test_pb2.StreamingOutputCallResponse()
-        response.payload.payload_type = self._test_pb2.COMPRESSABLE
+        response = self._response_pb2.StreamingOutputCallResponse()
+        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
         response.payload.payload_compressable = 'a' * parameter.size
         self._control()
         responses.append(response)
@@ -136,23 +137,25 @@ class _ServicerMethods(object):
 
 
 @contextlib.contextmanager
-def _CreateService(test_pb2):
+def _CreateService(service_pb2, response_pb2, payload_pb2):
   """Provides a servicer backend and a stub.
 
   The servicer is just the implementation of the actual servicer passed to the
   face player of the python RPC implementation; the two are detached.
 
   Args:
-    test_pb2: The test_pb2 module generated by this test.
+    service_pb2: The service_pb2 module generated by this test.
+    response_pb2: The response_pb2 module generated by this test
+    payload_pb2: The payload_pb2 module generated by this test
 
   Yields:
     A (servicer_methods, stub) pair where servicer_methods is the back-end of
       the service bound to the stub and and stub is the stub on which to invoke
       RPCs.
   """
-  servicer_methods = _ServicerMethods(test_pb2)
+  servicer_methods = _ServicerMethods(response_pb2, payload_pb2)
 
-  class Servicer(getattr(test_pb2, SERVICER_IDENTIFIER)):
+  class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)):
 
     def UnaryCall(self, request, context):
       return servicer_methods.UnaryCall(request, context)
@@ -170,55 +173,52 @@ def _CreateService(test_pb2):
       return servicer_methods.HalfDuplexCall(request_iter, context)
 
   servicer = Servicer()
-  server = getattr(test_pb2, SERVER_FACTORY_IDENTIFIER)(servicer)
+  server = getattr(service_pb2, SERVER_FACTORY_IDENTIFIER)(servicer)
   port = server.add_insecure_port('[::]:0')
   server.start()
   channel = implementations.insecure_channel('localhost', port)
-  stub = getattr(test_pb2, STUB_FACTORY_IDENTIFIER)(channel)
-  yield servicer_methods, stub
+  stub = getattr(service_pb2, STUB_FACTORY_IDENTIFIER)(channel)
+  yield (servicer_methods, stub)
   server.stop(0)
 
 
 @contextlib.contextmanager
-def _CreateIncompleteService(test_pb2):
+def _CreateIncompleteService(service_pb2):
   """Provides a servicer backend that fails to implement methods and its stub.
 
   The servicer is just the implementation of the actual servicer passed to the
   face player of the python RPC implementation; the two are detached.
-
   Args:
-    test_pb2: The test_pb2 module generated by this test.
-
+    service_pb2: The service_pb2 module generated by this test.
   Yields:
     A (servicer_methods, stub) pair where servicer_methods is the back-end of
       the service bound to the stub and and stub is the stub on which to invoke
       RPCs.
   """
-  servicer_methods = _ServicerMethods(test_pb2)
 
-  class Servicer(getattr(test_pb2, SERVICER_IDENTIFIER)):
+  class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)):
     pass
 
   servicer = Servicer()
-  server = getattr(test_pb2, SERVER_FACTORY_IDENTIFIER)(servicer)
+  server = getattr(service_pb2, SERVER_FACTORY_IDENTIFIER)(servicer)
   port = server.add_insecure_port('[::]:0')
   server.start()
   channel = implementations.insecure_channel('localhost', port)
-  stub = getattr(test_pb2, STUB_FACTORY_IDENTIFIER)(channel)
-  yield servicer_methods, stub
+  stub = getattr(service_pb2, STUB_FACTORY_IDENTIFIER)(channel)
+  yield None, stub
   server.stop(0)
 
 
-def _streaming_input_request_iterator(test_pb2):
+def _streaming_input_request_iterator(request_pb2, payload_pb2):
   for _ in range(3):
-    request = test_pb2.StreamingInputCallRequest()
-    request.payload.payload_type = test_pb2.COMPRESSABLE
+    request = request_pb2.StreamingInputCallRequest()
+    request.payload.payload_type = payload_pb2.COMPRESSABLE
     request.payload.payload_compressable = 'a'
     yield request
 
 
-def _streaming_output_request(test_pb2):
-  request = test_pb2.StreamingOutputCallRequest()
+def _streaming_output_request(request_pb2):
+  request = request_pb2.StreamingOutputCallRequest()
   sizes = [1, 2, 3]
   request.response_parameters.add(size=sizes[0], interval_us=0)
   request.response_parameters.add(size=sizes[1], interval_us=0)
@@ -226,11 +226,11 @@ def _streaming_output_request(test_pb2):
   return request
 
 
-def _full_duplex_request_iterator(test_pb2):
-  request = test_pb2.StreamingOutputCallRequest()
+def _full_duplex_request_iterator(request_pb2):
+  request = request_pb2.StreamingOutputCallRequest()
   request.response_parameters.add(size=1, interval_us=0)
   yield request
-  request = test_pb2.StreamingOutputCallRequest()
+  request = request_pb2.StreamingOutputCallRequest()
   request.response_parameters.add(size=2, interval_us=0)
   request.response_parameters.add(size=3, interval_us=0)
   yield request
@@ -250,8 +250,6 @@ class PythonPluginTest(unittest.TestCase):
     protoc_command = 'protoc'
     protoc_plugin_filename = distutils.spawn.find_executable(
         'grpc_python_plugin')
-    test_proto_filename = pkg_resources.resource_filename(
-        'tests.protoc_plugin', 'protoc_plugin_test.proto')
     if not os.path.isfile(protoc_command):
       # Assume that if we haven't built protoc that it's on the system.
       protoc_command = 'protoc'
@@ -259,19 +257,44 @@ class PythonPluginTest(unittest.TestCase):
     # Ensure that the output directory exists.
     self.outdir = tempfile.mkdtemp()
 
+    # Find all proto files
+    paths = []
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    proto_dir = os.path.join(root_dir, 'protos')
+    for walk_root, _, filenames in os.walk(proto_dir):
+      for filename in filenames:
+        if filename.endswith('.proto'):
+          path = os.path.join(walk_root, filename)
+          paths.append(path)
+
     # Invoke protoc with the plugin.
     cmd = [
         protoc_command,
         '--plugin=protoc-gen-python-grpc=%s' % protoc_plugin_filename,
-        '-I .',
+        '-I %s' % root_dir,
         '--python_out=%s' % self.outdir,
-        '--python-grpc_out=%s' % self.outdir,
-        os.path.basename(test_proto_filename),
-    ]
+        '--python-grpc_out=%s' % self.outdir
+    ] + paths
     subprocess.check_call(' '.join(cmd), shell=True, env=os.environ,
-                          cwd=os.path.dirname(test_proto_filename))
+                          cwd=os.path.dirname(os.path.realpath(__file__)))
+
+    # Generated proto directories dont include __init__.py, but
+    # these are needed for python package resolution
+    for walk_root, _, _ in os.walk(os.path.join(self.outdir, 'protos')):
+      path = os.path.join(walk_root, '__init__.py')
+      open(path, 'a').close()
+
     sys.path.insert(0, self.outdir)
 
+    import protos.payload.test_payload_pb2 as payload_pb2  # pylint: disable=g-import-not-at-top
+    import protos.requests.r.test_requests_pb2 as request_pb2  # pylint: disable=g-import-not-at-top
+    import protos.responses.test_responses_pb2 as response_pb2  # pylint: disable=g-import-not-at-top
+    import protos.service.test_service_pb2 as service_pb2  # pylint: disable=g-import-not-at-top
+    self._payload_pb2 = payload_pb2
+    self._request_pb2 = request_pb2
+    self._response_pb2 = response_pb2
+    self._service_pb2 = service_pb2
+
   def tearDown(self):
     try:
       shutil.rmtree(self.outdir)
@@ -282,43 +305,40 @@ class PythonPluginTest(unittest.TestCase):
 
   def testImportAttributes(self):
     # check that we can access the generated module and its members.
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    self.assertIsNotNone(getattr(test_pb2, SERVICER_IDENTIFIER, None))
-    self.assertIsNotNone(getattr(test_pb2, STUB_IDENTIFIER, None))
-    self.assertIsNotNone(getattr(test_pb2, SERVER_FACTORY_IDENTIFIER, None))
-    self.assertIsNotNone(getattr(test_pb2, STUB_FACTORY_IDENTIFIER, None))
+    self.assertIsNotNone(
+        getattr(self._service_pb2, SERVICER_IDENTIFIER, None))
+    self.assertIsNotNone(
+        getattr(self._service_pb2, STUB_IDENTIFIER, None))
+    self.assertIsNotNone(
+        getattr(self._service_pb2, SERVER_FACTORY_IDENTIFIER, None))
+    self.assertIsNotNone(
+        getattr(self._service_pb2, STUB_FACTORY_IDENTIFIER, None))
 
   def testUpDown(self):
-    import protoc_plugin_test_pb2 as test_pb2
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (servicer, stub):
-      request = test_pb2.SimpleRequest(response_size=13)
+    with _CreateService(
+        self._service_pb2, self._response_pb2, self._payload_pb2):
+      self._request_pb2.SimpleRequest(response_size=13)
 
   def testIncompleteServicer(self):
-    import protoc_plugin_test_pb2 as test_pb2
-    moves.reload_module(test_pb2)
-    with _CreateIncompleteService(test_pb2) as (servicer, stub):
-      request = test_pb2.SimpleRequest(response_size=13)
+    with _CreateIncompleteService(self._service_pb2) as (_, stub):
+      request = self._request_pb2.SimpleRequest(response_size=13)
       try:
-        response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT)
+        stub.UnaryCall(request, test_constants.LONG_TIMEOUT)
       except face.AbortionError as error:
         self.assertEqual(interfaces.StatusCode.UNIMPLEMENTED, error.code)
 
   def testUnaryCall(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
-      request = test_pb2.SimpleRequest(response_size=13)
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = self._request_pb2.SimpleRequest(response_size=13)
       response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT)
     expected_response = methods.UnaryCall(request, 'not a real context!')
     self.assertEqual(expected_response, response)
 
   def testUnaryCallFuture(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = test_pb2.SimpleRequest(response_size=13)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = self._request_pb2.SimpleRequest(response_size=13)
       # Check that the call does not block waiting for the server to respond.
       with methods.pause():
         response_future = stub.UnaryCall.future(
@@ -328,10 +348,9 @@ class PythonPluginTest(unittest.TestCase):
     self.assertEqual(expected_response, response)
 
   def testUnaryCallFutureExpired(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
-      request = test_pb2.SimpleRequest(response_size=13)
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = self._request_pb2.SimpleRequest(response_size=13)
       with methods.pause():
         response_future = stub.UnaryCall.future(
             request, test_constants.SHORT_TIMEOUT)
@@ -339,30 +358,27 @@ class PythonPluginTest(unittest.TestCase):
           response_future.result()
 
   def testUnaryCallFutureCancelled(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = test_pb2.SimpleRequest(response_size=13)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = self._request_pb2.SimpleRequest(response_size=13)
       with methods.pause():
         response_future = stub.UnaryCall.future(request, 1)
         response_future.cancel()
         self.assertTrue(response_future.cancelled())
 
   def testUnaryCallFutureFailed(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = test_pb2.SimpleRequest(response_size=13)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = self._request_pb2.SimpleRequest(response_size=13)
       with methods.fail():
         response_future = stub.UnaryCall.future(
             request, test_constants.LONG_TIMEOUT)
         self.assertIsNotNone(response_future.exception())
 
   def testStreamingOutputCall(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = _streaming_output_request(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request(self._request_pb2)
       responses = stub.StreamingOutputCall(
           request, test_constants.LONG_TIMEOUT)
       expected_responses = methods.StreamingOutputCall(
@@ -372,10 +388,9 @@ class PythonPluginTest(unittest.TestCase):
         self.assertEqual(expected_response, response)
 
   def testStreamingOutputCallExpired(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = _streaming_output_request(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request(self._request_pb2)
       with methods.pause():
         responses = stub.StreamingOutputCall(
             request, test_constants.SHORT_TIMEOUT)
@@ -383,10 +398,9 @@ class PythonPluginTest(unittest.TestCase):
           list(responses)
 
   def testStreamingOutputCallCancelled(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = _streaming_output_request(test_pb2)
-    with _CreateService(test_pb2) as (unused_methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request(self._request_pb2)
       responses = stub.StreamingOutputCall(
           request, test_constants.LONG_TIMEOUT)
       next(responses)
@@ -395,10 +409,9 @@ class PythonPluginTest(unittest.TestCase):
         next(responses)
 
   def testStreamingOutputCallFailed(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request = _streaming_output_request(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request(self._request_pb2)
       with methods.fail():
         responses = stub.StreamingOutputCall(request, 1)
         self.assertIsNotNone(responses)
@@ -406,36 +419,38 @@ class PythonPluginTest(unittest.TestCase):
           next(responses)
 
   def testStreamingInputCall(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       response = stub.StreamingInputCall(
-          _streaming_input_request_iterator(test_pb2),
+          _streaming_input_request_iterator(
+              self._request_pb2, self._payload_pb2),
           test_constants.LONG_TIMEOUT)
     expected_response = methods.StreamingInputCall(
-        _streaming_input_request_iterator(test_pb2), 'not a real RpcContext!')
+        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        'not a real RpcContext!')
     self.assertEqual(expected_response, response)
 
   def testStreamingInputCallFuture(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(test_pb2),
+            _streaming_input_request_iterator(
+                self._request_pb2, self._payload_pb2),
             test_constants.LONG_TIMEOUT)
       response = response_future.result()
     expected_response = methods.StreamingInputCall(
-        _streaming_input_request_iterator(test_pb2), 'not a real RpcContext!')
+        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        'not a real RpcContext!')
     self.assertEqual(expected_response, response)
 
   def testStreamingInputCallFutureExpired(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(test_pb2),
+            _streaming_input_request_iterator(
+                self._request_pb2, self._payload_pb2),
             test_constants.SHORT_TIMEOUT)
         with self.assertRaises(face.ExpirationError):
           response_future.result()
@@ -443,12 +458,12 @@ class PythonPluginTest(unittest.TestCase):
             response_future.exception(), face.ExpirationError)
 
   def testStreamingInputCallFutureCancelled(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(test_pb2),
+            _streaming_input_request_iterator(
+                self._request_pb2, self._payload_pb2),
             test_constants.LONG_TIMEOUT)
         response_future.cancel()
         self.assertTrue(response_future.cancelled())
@@ -456,32 +471,32 @@ class PythonPluginTest(unittest.TestCase):
         response_future.result()
 
   def testStreamingInputCallFutureFailed(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with methods.fail():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(test_pb2),
+            _streaming_input_request_iterator(
+                self._request_pb2, self._payload_pb2),
             test_constants.LONG_TIMEOUT)
         self.assertIsNotNone(response_future.exception())
 
   def testFullDuplexCall(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       responses = stub.FullDuplexCall(
-          _full_duplex_request_iterator(test_pb2), test_constants.LONG_TIMEOUT)
+          _full_duplex_request_iterator(self._request_pb2),
+          test_constants.LONG_TIMEOUT)
       expected_responses = methods.FullDuplexCall(
-          _full_duplex_request_iterator(test_pb2), 'not a real RpcContext!')
+          _full_duplex_request_iterator(self._request_pb2),
+          'not a real RpcContext!')
       for expected_response, response in moves.zip_longest(
           expected_responses, responses):
         self.assertEqual(expected_response, response)
 
   def testFullDuplexCallExpired(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request_iterator = _full_duplex_request_iterator(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         responses = stub.FullDuplexCall(
             request_iterator, test_constants.SHORT_TIMEOUT)
@@ -489,10 +504,9 @@ class PythonPluginTest(unittest.TestCase):
           list(responses)
 
   def testFullDuplexCallCancelled(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
-      request_iterator = _full_duplex_request_iterator(test_pb2)
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
+      request_iterator = _full_duplex_request_iterator(self._request_pb2)
       responses = stub.FullDuplexCall(
           request_iterator, test_constants.LONG_TIMEOUT)
       next(responses)
@@ -501,10 +515,9 @@ class PythonPluginTest(unittest.TestCase):
         next(responses)
 
   def testFullDuplexCallFailed(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    request_iterator = _full_duplex_request_iterator(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with methods.fail():
         responses = stub.FullDuplexCall(
             request_iterator, test_constants.LONG_TIMEOUT)
@@ -513,14 +526,13 @@ class PythonPluginTest(unittest.TestCase):
           next(responses)
 
   def testHalfDuplexCall(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       def half_duplex_request_iterator():
-        request = test_pb2.StreamingOutputCallRequest()
+        request = self._request_pb2.StreamingOutputCallRequest()
         request.response_parameters.add(size=1, interval_us=0)
         yield request
-        request = test_pb2.StreamingOutputCallRequest()
+        request = self._request_pb2.StreamingOutputCallRequest()
         request.response_parameters.add(size=2, interval_us=0)
         request.response_parameters.add(size=3, interval_us=0)
         yield request
@@ -533,8 +545,6 @@ class PythonPluginTest(unittest.TestCase):
         self.assertEqual(expected_response, response)
 
   def testHalfDuplexCallWedged(self):
-    import protoc_plugin_test_pb2 as test_pb2  # pylint: disable=g-import-not-at-top
-    moves.reload_module(test_pb2)
     condition = threading.Condition()
     wait_cell = [False]
     @contextlib.contextmanager
@@ -547,13 +557,14 @@ class PythonPluginTest(unittest.TestCase):
         wait_cell[0] = False
         condition.notify_all()
     def half_duplex_request_iterator():
-      request = test_pb2.StreamingOutputCallRequest()
+      request = self._request_pb2.StreamingOutputCallRequest()
       request.response_parameters.add(size=1, interval_us=0)
       yield request
       with condition:
         while wait_cell[0]:
           condition.wait()
-    with _CreateService(test_pb2) as (methods, stub):
+    with _CreateService(self._service_pb2, self._response_pb2,
+                        self._payload_pb2) as (methods, stub):
       with wait():
         responses = stub.HalfDuplexCall(
             half_duplex_request_iterator(), test_constants.SHORT_TIMEOUT)
@@ -563,5 +574,5 @@ class PythonPluginTest(unittest.TestCase):
 
 
 if __name__ == '__main__':
-  os.chdir(os.path.dirname(sys.argv[0]))
+  #os.chdir(os.path.dirname(sys.argv[0]))
   unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/protoc_plugin/protos/payload/test_payload.proto b/src/python/grpcio/tests/protoc_plugin/protos/payload/test_payload.proto
new file mode 100644
index 0000000000..457543aa79
--- /dev/null
+++ b/src/python/grpcio/tests/protoc_plugin/protos/payload/test_payload.proto
@@ -0,0 +1,51 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package grpc_protoc_plugin;
+
+enum PayloadType {
+  // Compressable text format.
+  COMPRESSABLE= 0;
+
+  // Uncompressable binary format.
+  UNCOMPRESSABLE = 1;
+
+  // Randomly chosen from all other formats defined in this enum.
+  RANDOM = 2;
+}
+
+message Payload {
+  PayloadType payload_type = 1;
+  oneof payload_body {
+    string payload_compressable = 2;
+    bytes payload_uncompressable = 3;
+  }
+}
diff --git a/src/python/grpcio/tests/protoc_plugin/protos/requests/r/test_requests.proto b/src/python/grpcio/tests/protoc_plugin/protos/requests/r/test_requests.proto
new file mode 100644
index 0000000000..54105df6a5
--- /dev/null
+++ b/src/python/grpcio/tests/protoc_plugin/protos/requests/r/test_requests.proto
@@ -0,0 +1,77 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+import "protos/payload/test_payload.proto";
+
+package grpc_protoc_plugin;
+
+message SimpleRequest {
+  // Desired payload type in the response from the server.
+  // If response_type is RANDOM, server randomly chooses one from other formats.
+  PayloadType response_type = 1;
+
+  // Desired payload size in the response from the server.
+  // If response_type is COMPRESSABLE, this denotes the size before compression.
+  int32 response_size = 2;
+
+  // input payload sent along with the request.
+  Payload payload = 3;
+}
+
+message StreamingInputCallRequest {
+  // input payload sent along with the request.
+  Payload payload = 1;
+
+  // Not expecting any payload from the response.
+}
+
+message ResponseParameters {
+  // Desired payload sizes in responses from the server.
+  // If response_type is COMPRESSABLE, this denotes the size before compression.
+  int32 size = 1;
+
+  // Desired interval between consecutive responses in the response stream in
+  // microseconds.
+  int32 interval_us = 2;
+}
+
+message StreamingOutputCallRequest {
+  // Desired payload type in the response from the server.
+  // If response_type is RANDOM, the payload from each response in the stream
+  // might be of different types. This is to simulate a mixed type of payload
+  // stream.
+  PayloadType response_type = 1;
+
+  repeated ResponseParameters response_parameters = 2;
+
+  // input payload sent along with the request.
+  Payload payload = 3;
+}
diff --git a/src/python/grpcio_health_checking/grpc/health/v1/health.proto b/src/python/grpcio/tests/protoc_plugin/protos/responses/test_responses.proto
index b0bac54be9..734fbda86e 100644
--- a/src/python/grpcio_health_checking/grpc/health/v1/health.proto
+++ b/src/python/grpcio/tests/protoc_plugin/protos/responses/test_responses.proto
@@ -29,21 +29,19 @@
 
 syntax = "proto3";
 
-package grpc.health.v1;
+import "protos/payload/test_payload.proto";
 
-message HealthCheckRequest {
-  string service = 1;
+package grpc_protoc_plugin;
+
+message SimpleResponse {
+  Payload payload = 1;
 }
 
-message HealthCheckResponse {
-  enum ServingStatus {
-    UNKNOWN = 0;
-    SERVING = 1;
-    NOT_SERVING = 2;
-  }
-  ServingStatus status = 1;
+message StreamingInputCallResponse {
+  // Aggregated size of payloads received from the client.
+  int32 aggregated_payload_size = 1;
 }
 
-service Health {
-  rpc Check(HealthCheckRequest) returns (HealthCheckResponse);
+message StreamingOutputCallResponse {
+  Payload payload = 1;
 }
diff --git a/src/python/grpcio/tests/protoc_plugin/protoc_plugin_test.proto b/src/python/grpcio/tests/protoc_plugin/protos/service/test_service.proto
index 6762a8e7f3..fe715ee7f9 100644
--- a/src/python/grpcio/tests/protoc_plugin/protoc_plugin_test.proto
+++ b/src/python/grpcio/tests/protoc_plugin/protos/service/test_service.proto
@@ -1,4 +1,4 @@
-// Copyright 2015, Google Inc.
+// Copyright 2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -27,87 +27,12 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-// An integration test service that covers all the method signature permutations
-// of unary/streaming requests/responses.
-// This file is duplicated around the code base. See GitHub issue #526.
-syntax = "proto2";
+syntax = "proto3";
 
-package grpc_protoc_plugin;
-
-enum PayloadType {
-  // Compressable text format.
-  COMPRESSABLE= 1;
-
-  // Uncompressable binary format.
-  UNCOMPRESSABLE = 2;
-
-  // Randomly chosen from all other formats defined in this enum.
-  RANDOM = 3;
-}
-
-message Payload {
-  required PayloadType payload_type = 1;
-  oneof payload_body {
-    string payload_compressable = 2;
-    bytes payload_uncompressable = 3;
-  }
-}
-
-message SimpleRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, server randomly chooses one from other formats.
-  optional PayloadType response_type = 1 [default=COMPRESSABLE];
-
-  // Desired payload size in the response from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  optional int32 response_size = 2;
-
-  // Optional input payload sent along with the request.
-  optional Payload payload = 3;
-}
-
-message SimpleResponse {
-  optional Payload payload = 1;
-}
-
-message StreamingInputCallRequest {
-  // Optional input payload sent along with the request.
-  optional Payload payload = 1;
+import "protos/requests/r/test_requests.proto";
+import "protos/responses/test_responses.proto";
 
-  // Not expecting any payload from the response.
-}
-
-message StreamingInputCallResponse {
-  // Aggregated size of payloads received from the client.
-  optional int32 aggregated_payload_size = 1;
-}
-
-message ResponseParameters {
-  // Desired payload sizes in responses from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  required int32 size = 1;
-
-  // Desired interval between consecutive responses in the response stream in
-  // microseconds.
-  required int32 interval_us = 2;
-}
-
-message StreamingOutputCallRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, the payload from each response in the stream
-  // might be of different types. This is to simulate a mixed type of payload
-  // stream.
-  optional PayloadType response_type = 1 [default=COMPRESSABLE];
-
-  repeated ResponseParameters response_parameters = 2;
-
-  // Optional input payload sent along with the request.
-  optional Payload payload = 3;
-}
-
-message StreamingOutputCallResponse {
-  optional Payload payload = 1;
-}
+package grpc_protoc_plugin;
 
 service TestService {
   // One request followed by one response.
diff --git a/src/python/grpcio/tests/qps/benchmark_client.py b/src/python/grpcio/tests/qps/benchmark_client.py
index eed0b0c6da..080281415d 100644
--- a/src/python/grpcio/tests/qps/benchmark_client.py
+++ b/src/python/grpcio/tests/qps/benchmark_client.py
@@ -30,15 +30,15 @@
 """Defines test client behaviors (UNARY/STREAMING) (SYNC/ASYNC)."""
 
 import abc
+import threading
 import time
-try:
-  import Queue as queue  # Python 2.x
-except ImportError:
-  import queue  # Python 3
 
 from concurrent import futures
+from six.moves import queue
 
+import grpc
 from grpc.beta import implementations
+from grpc.framework.interfaces.face import face
 from src.proto.grpc.testing import messages_pb2
 from src.proto.grpc.testing import services_pb2
 from tests.unit import resources
@@ -64,6 +64,13 @@ class BenchmarkClient:
     else:
       channel = implementations.insecure_channel(host, port)
 
+    connected_event = threading.Event()
+    def wait_for_ready(connectivity):
+      if connectivity == grpc.ChannelConnectivity.READY:
+        connected_event.set()
+    channel.subscribe(wait_for_ready, try_to_connect=True)
+    connected_event.wait()
+
     if config.payload_config.WhichOneof('payload') == 'simple_params':
       self._generic = False
       self._stub = services_pb2.beta_create_BenchmarkService_stub(channel)
@@ -81,6 +88,7 @@ class BenchmarkClient:
     self._response_callbacks = []
 
   def add_response_callback(self, callback):
+    """callback will be invoked as callback(client, query_time)"""
     self._response_callbacks.append(callback)
 
   @abc.abstractmethod
@@ -94,10 +102,10 @@ class BenchmarkClient:
   def stop(self):
     pass
 
-  def _handle_response(self, query_time):
+  def _handle_response(self, client, query_time):
     self._hist.add(query_time * 1e9)  # Report times in nanoseconds
     for callback in self._response_callbacks:
-      callback(query_time)
+      callback(client, query_time)
 
 
 class UnarySyncBenchmarkClient(BenchmarkClient):
@@ -120,7 +128,7 @@ class UnarySyncBenchmarkClient(BenchmarkClient):
     start_time = time.time()
     self._stub.UnaryCall(self._request, _TIMEOUT)
     end_time = time.time()
-    self._handle_response(end_time - start_time)
+    self._handle_response(self, end_time - start_time)
 
 
 class UnaryAsyncBenchmarkClient(BenchmarkClient):
@@ -135,19 +143,20 @@ class UnaryAsyncBenchmarkClient(BenchmarkClient):
   def _response_received(self, start_time, resp):
     resp.result()
     end_time = time.time()
-    self._handle_response(end_time - start_time)
+    self._handle_response(self, end_time - start_time)
 
   def stop(self):
     self._stub = None
 
 
-class StreamingAsyncBenchmarkClient(BenchmarkClient):
+class _SyncStream(object):
 
-  def __init__(self, server, config, hist):
-    super(StreamingAsyncBenchmarkClient, self).__init__(server, config, hist)
+  def __init__(self, stub, generic, request, handle_response):
+    self._stub = stub
+    self._generic = generic
+    self._request = request
+    self._handle_response = handle_response
     self._is_streaming = False
-    self._pool = futures.ThreadPoolExecutor(max_workers=1)
-    # Use a thread-safe queue to put requests on the stream
     self._request_queue = queue.Queue()
     self._send_time_queue = queue.Queue()
 
@@ -157,25 +166,19 @@ class StreamingAsyncBenchmarkClient(BenchmarkClient):
 
   def start(self):
     self._is_streaming = True
-    self._pool.submit(self._request_stream)
-
-  def stop(self):
-    self._is_streaming = False
-    self._pool.shutdown(wait=True)
-    self._stub = None
-
-  def _request_stream(self):
-    self._is_streaming = True
     if self._generic:
-      response_stream = self._stub.inline_stream_stream(
-          'grpc.testing.BenchmarkService', 'StreamingCall',
-          self._request_generator(), _TIMEOUT)
+      stream_callable = self._stub.stream_stream(
+          'grpc.testing.BenchmarkService', 'StreamingCall')
     else:
-      response_stream = self._stub.StreamingCall(self._request_generator(),
-                                                 _TIMEOUT)
+      stream_callable = self._stub.StreamingCall
+
+    response_stream = stream_callable(self._request_generator(), _TIMEOUT)
     for _ in response_stream:
-      end_time = time.time()
-      self._handle_response(end_time - self._send_time_queue.get_nowait())
+      self._handle_response(
+          self, time.time() - self._send_time_queue.get_nowait())
+
+  def stop(self):
+    self._is_streaming = False
 
   def _request_generator(self):
     while self._is_streaming:
@@ -184,3 +187,30 @@ class StreamingAsyncBenchmarkClient(BenchmarkClient):
         yield request
       except queue.Empty:
         pass
+
+
+class StreamingSyncBenchmarkClient(BenchmarkClient):
+
+  def __init__(self, server, config, hist):
+    super(StreamingSyncBenchmarkClient, self).__init__(server, config, hist)
+    self._pool = futures.ThreadPoolExecutor(
+        max_workers=config.outstanding_rpcs_per_channel)
+    self._streams = [_SyncStream(self._stub, self._generic, 
+                                 self._request, self._handle_response)
+                     for _ in xrange(config.outstanding_rpcs_per_channel)]
+    self._curr_stream = 0
+
+  def send_request(self):
+    # Use a round_robin scheduler to determine what stream to send on
+    self._streams[self._curr_stream].send_request()
+    self._curr_stream = (self._curr_stream + 1) % len(self._streams)
+
+  def start(self):
+    for stream in self._streams:
+      self._pool.submit(stream.start)
+
+  def stop(self):
+    for stream in self._streams:
+      stream.stop()
+    self._pool.shutdown(wait=True)
+    self._stub = None
diff --git a/src/python/grpcio/tests/qps/client_runner.py b/src/python/grpcio/tests/qps/client_runner.py
index a36c30ccc0..1fd58687ad 100644
--- a/src/python/grpcio/tests/qps/client_runner.py
+++ b/src/python/grpcio/tests/qps/client_runner.py
@@ -34,7 +34,7 @@ ClientRunner invokes either periodically or in response to some event.
 """
 
 import abc
-import thread
+import threading
 import time
 
 
@@ -61,15 +61,18 @@ class OpenLoopClientRunner(ClientRunner):
     super(OpenLoopClientRunner, self).__init__(client)
     self._is_running = False
     self._interval_generator = interval_generator
+    self._dispatch_thread = threading.Thread(
+        target=self._dispatch_requests, args=())
 
   def start(self):
     self._is_running = True
     self._client.start()
-    thread.start_new_thread(self._dispatch_requests, ())
-
+    self._dispatch_thread.start()
+   
   def stop(self):
     self._is_running = False
     self._client.stop()
+    self._dispatch_thread.join()
     self._client = None
 
   def _dispatch_requests(self):
@@ -89,16 +92,15 @@ class ClosedLoopClientRunner(ClientRunner):
 
   def start(self):
     self._is_running = True
+    self._client.start()
     for _ in xrange(self._request_count):
       self._client.send_request()
-    self._client.start()
 
   def stop(self):
     self._is_running = False
     self._client.stop()
     self._client = None
 
-  def _send_request(self, response_time):
+  def _send_request(self, client, response_time):
     if self._is_running:
-      self._client.send_request()
-
+      client.send_request()
diff --git a/src/python/grpcio/tests/qps/qps_worker.py b/src/python/grpcio/tests/qps/qps_worker.py
index 3dda718638..16926379a5 100644
--- a/src/python/grpcio/tests/qps/qps_worker.py
+++ b/src/python/grpcio/tests/qps/qps_worker.py
@@ -43,9 +43,7 @@ def run_worker_server(port):
   server.add_insecure_port('[::]:{}'.format(port))
   server.start()
   servicer.wait_for_quit()
-  # Drain outstanding requests for clean exit
-  time.sleep(2)
-  server.stop(0)
+  server.stop(2)
 
 
 if __name__ == '__main__':
diff --git a/src/python/grpcio/tests/qps/worker_server.py b/src/python/grpcio/tests/qps/worker_server.py
index 0b3acc14e7..d41f8377c2 100644
--- a/src/python/grpcio/tests/qps/worker_server.py
+++ b/src/python/grpcio/tests/qps/worker_server.py
@@ -146,15 +146,15 @@ class WorkerServer(services_pb2.BetaWorkerServiceServicer):
       if config.rpc_type == control_pb2.UNARY:
         client = benchmark_client.UnarySyncBenchmarkClient(
             server, config, qps_data)
-      else:
-        raise Exception('STREAMING SYNC client not supported')
+      elif config.rpc_type == control_pb2.STREAMING:
+        client = benchmark_client.StreamingSyncBenchmarkClient(
+            server, config, qps_data)
     elif config.client_type == control_pb2.ASYNC_CLIENT:
       if config.rpc_type == control_pb2.UNARY:
         client = benchmark_client.UnaryAsyncBenchmarkClient(
             server, config, qps_data)
-      elif config.rpc_type == control_pb2.STREAMING:
-        client = benchmark_client.StreamingAsyncBenchmarkClient(
-            server, config, qps_data)
+      else:
+        raise Exception('Async streaming client not supported')
     else:
       raise Exception('Unsupported client type {}'.format(config.client_type))
 
diff --git a/src/python/grpcio/tests/stress/client.py b/src/python/grpcio/tests/stress/client.py
index a733741b73..0de2532cd8 100644
--- a/src/python/grpcio/tests/stress/client.py
+++ b/src/python/grpcio/tests/stress/client.py
@@ -30,10 +30,10 @@
 """Entry point for running stress tests."""
 
 import argparse
-import Queue
 import threading
 
 from grpc.beta import implementations
+from six.moves import queue
 from src.proto.grpc.testing import metrics_pb2
 from src.proto.grpc.testing import test_pb2
 
@@ -94,7 +94,7 @@ def run_test(args):
   test_cases = _parse_weighted_test_cases(args.test_cases)
   test_servers = args.server_addresses.split(',')
   # Propagate any client exceptions with a queue
-  exception_queue = Queue.Queue()
+  exception_queue = queue.Queue()
   stop_event = threading.Event()
   hist = histogram.Histogram(1, 1)
   runners = []
@@ -117,8 +117,11 @@ def run_test(args):
   for runner in runners:
     runner.start()
   try:
-    raise exception_queue.get(block=True, timeout=args.test_duration_secs)
-  except Queue.Empty:
+    timeout_secs = args.test_duration_secs
+    if timeout_secs < 0:
+      timeout_secs = None
+    raise exception_queue.get(block=True, timeout=timeout_secs)
+  except queue.Empty:
     # No exceptions thrown, success
     pass
   finally:
diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json
index 84870aaa5c..45eb75b242 100644
--- a/src/python/grpcio/tests/tests.json
+++ b/src/python/grpcio/tests/tests.json
@@ -1,53 +1,42 @@
 [
-  "_base_interface_test.AsyncEasyTest", 
-  "_base_interface_test.AsyncPeasyTest", 
-  "_base_interface_test.SyncEasyTest", 
-  "_base_interface_test.SyncPeasyTest", 
+  "_api_test.AllTest",
+  "_api_test.ChannelConnectivityTest",
+  "_api_test.ChannelTest",
+  "_auth_test.AccessTokenCallCredentialsTest",
+  "_auth_test.GoogleCallCredentialsTest",
   "_beta_features_test.BetaFeaturesTest", 
   "_beta_features_test.ContextManagementAndLifecycleTest", 
+  "_cancel_many_calls_test.CancelManyCallsTest",
+  "_channel_connectivity_test.ChannelConnectivityTest",
+  "_channel_ready_future_test.ChannelReadyFutureTest",
   "_channel_test.ChannelTest", 
-  "_connectivity_channel_test.ChannelConnectivityTest", 
-  "_core_over_links_base_interface_test.AsyncEasyTest", 
-  "_core_over_links_base_interface_test.AsyncPeasyTest", 
-  "_core_over_links_base_interface_test.SyncEasyTest", 
-  "_core_over_links_base_interface_test.SyncPeasyTest", 
-  "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
-  "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
-  "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
-  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_compression_test.CompressionTest",
+  "_connectivity_channel_test.ConnectivityStatesTest",
+  "_empty_message_test.EmptyMessageTest",
+  "_exit_test.ExitTest",
   "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
   "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
   "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
   "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
-  "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest",
+  "_health_servicer_test.HealthServicerTest",
+  "_implementations_test.CallCredentialsTest",
   "_implementations_test.ChannelCredentialsTest", 
   "_insecure_interop_test.InsecureInteropTest", 
-  "_intermediary_low_test.CancellationTest", 
-  "_intermediary_low_test.EchoTest", 
-  "_intermediary_low_test.ExpirationTest", 
-  "_intermediary_low_test.LonelyClientTest", 
-  "_later_test.LaterTest", 
   "_logging_pool_test.LoggingPoolTest", 
-  "_lonely_invocation_link_test.LonelyInvocationLinkTest", 
-  "_low_test.HangingServerShutdown", 
-  "_low_test.InsecureServerInsecureClient", 
+  "_metadata_code_details_test.MetadataCodeDetailsTest",
+  "_metadata_test.MetadataTest",
   "_not_found_test.NotFoundTest", 
+  "_python_plugin_test.PythonPluginTest",
+  "_read_some_but_not_all_responses_test.ReadSomeButNotAllResponsesTest",
+  "_rpc_test.RPCTest",
   "_sanity_test.Sanity", 
   "_secure_interop_test.SecureInteropTest", 
-  "_transmission_test.RoundTripTest", 
-  "_transmission_test.TransmissionTest", 
+  "_thread_cleanup_test.CleanupThreadTest",
   "_utilities_test.ChannelConnectivityTest", 
   "beta_python_plugin_test.PythonPluginTest", 
   "cygrpc_test.InsecureServerInsecureClient", 
   "cygrpc_test.SecureServerSecureClient", 
   "cygrpc_test.TypeSmokeTest"
-]
\ No newline at end of file
+]
diff --git a/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py b/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py
deleted file mode 100644
index 22d4b019c7..0000000000
--- a/src/python/grpcio/tests/unit/_adapter/_intermediary_low_test.py
+++ /dev/null
@@ -1,428 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests for the old '_low'."""
-
-import threading
-import time
-import unittest
-
-import six
-from six.moves import queue
-
-from grpc._adapter import _intermediary_low as _low
-
-_STREAM_LENGTH = 300
-_TIMEOUT = 5
-_AFTER_DELAY = 2
-_FUTURE = time.time() + 60 * 60 * 24
-_BYTE_SEQUENCE = b'\abcdefghijklmnopqrstuvwxyz0123456789' * 200
-_BYTE_SEQUENCE_SEQUENCE = tuple(
-    bytes(bytearray((row + column) % 256 for column in range(row)))
-    for row in range(_STREAM_LENGTH))
-
-
-class LonelyClientTest(unittest.TestCase):
-
-  def testLonelyClient(self):
-    host = 'nosuchhostexists'
-    port = 54321
-    method = 'test method'
-    deadline = time.time() + _TIMEOUT
-    after_deadline = deadline + _AFTER_DELAY
-    metadata_tag = object()
-    finish_tag = object()
-
-    completion_queue = _low.CompletionQueue()
-    channel = _low.Channel('%s:%d' % (host, port), None)
-    client_call = _low.Call(channel, completion_queue, method, host, deadline)
-
-    client_call.invoke(completion_queue, metadata_tag, finish_tag)
-    first_event = completion_queue.get(after_deadline)
-    self.assertIsNotNone(first_event)
-    second_event = completion_queue.get(after_deadline)
-    self.assertIsNotNone(second_event)
-    kinds = [event.kind for event in (first_event, second_event)]
-    six.assertCountEqual(self,
-        (_low.Event.Kind.METADATA_ACCEPTED, _low.Event.Kind.FINISH),
-        kinds)
-
-    self.assertIsNone(completion_queue.get(after_deadline))
-
-    completion_queue.stop()
-    stop_event = completion_queue.get(_FUTURE)
-    self.assertEqual(_low.Event.Kind.STOP, stop_event.kind)
-
-    del client_call
-    del channel
-    del completion_queue
-
-
-def _drive_completion_queue(completion_queue, event_queue):
-  while True:
-    event = completion_queue.get(_FUTURE)
-    if event.kind is _low.Event.Kind.STOP:
-      break
-    event_queue.put(event)
-
-
-class EchoTest(unittest.TestCase):
-
-  def setUp(self):
-    self.host = 'localhost'
-
-    self.server_completion_queue = _low.CompletionQueue()
-    self.server = _low.Server(self.server_completion_queue)
-    port = self.server.add_http2_addr('[::]:0')
-    self.server.start()
-    self.server_events = queue.Queue()
-    self.server_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue,
-        args=(self.server_completion_queue, self.server_events))
-    self.server_completion_queue_thread.start()
-
-    self.client_completion_queue = _low.CompletionQueue()
-    self.channel = _low.Channel('%s:%d' % (self.host, port), None)
-    self.client_events = queue.Queue()
-    self.client_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue,
-        args=(self.client_completion_queue, self.client_events))
-    self.client_completion_queue_thread.start()
-
-  def tearDown(self):
-    self.server.stop()
-    self.server.cancel_all_calls()
-    self.server_completion_queue.stop()
-    self.client_completion_queue.stop()
-    self.server_completion_queue_thread.join()
-    self.client_completion_queue_thread.join()
-    del self.server
-
-  def _perform_echo_test(self, test_data):
-    method = 'test method'
-    details = 'test details'
-    server_leading_metadata_key = 'my_server_leading_key'
-    server_leading_metadata_value = 'my_server_leading_value'
-    server_trailing_metadata_key = 'my_server_trailing_key'
-    server_trailing_metadata_value = 'my_server_trailing_value'
-    client_metadata_key = 'my_client_key'
-    client_metadata_value = 'my_client_value'
-    server_leading_binary_metadata_key = 'my_server_leading_key-bin'
-    server_leading_binary_metadata_value = b'\0'*2047
-    server_trailing_binary_metadata_key = 'my_server_trailing_key-bin'
-    server_trailing_binary_metadata_value = b'\0'*2047
-    client_binary_metadata_key = 'my_client_key-bin'
-    client_binary_metadata_value = b'\0'*2047
-    deadline = _FUTURE
-    metadata_tag = object()
-    finish_tag = object()
-    write_tag = object()
-    complete_tag = object()
-    service_tag = object()
-    read_tag = object()
-    status_tag = object()
-
-    server_data = []
-    client_data = []
-
-    client_call = _low.Call(self.channel, self.client_completion_queue,
-                            method, self.host, deadline)
-    client_call.add_metadata(client_metadata_key, client_metadata_value)
-    client_call.add_metadata(client_binary_metadata_key,
-                             client_binary_metadata_value)
-
-    client_call.invoke(self.client_completion_queue, metadata_tag, finish_tag)
-
-    self.server.service(service_tag)
-    service_accepted = self.server_events.get()
-    self.assertIsNotNone(service_accepted)
-    self.assertIs(service_accepted.kind, _low.Event.Kind.SERVICE_ACCEPTED)
-    self.assertIs(service_accepted.tag, service_tag)
-    self.assertEqual(method, service_accepted.service_acceptance.method)
-    self.assertEqual(self.host, service_accepted.service_acceptance.host)
-    self.assertIsNotNone(service_accepted.service_acceptance.call)
-    metadata = dict(service_accepted.metadata)
-    self.assertIn(client_metadata_key, metadata)
-    self.assertEqual(client_metadata_value, metadata[client_metadata_key])
-    self.assertIn(client_binary_metadata_key, metadata)
-    self.assertEqual(client_binary_metadata_value,
-                     metadata[client_binary_metadata_key])
-    server_call = service_accepted.service_acceptance.call
-    server_call.accept(self.server_completion_queue, finish_tag)
-    server_call.add_metadata(server_leading_metadata_key,
-                             server_leading_metadata_value)
-    server_call.add_metadata(server_leading_binary_metadata_key,
-                             server_leading_binary_metadata_value)
-    server_call.premetadata()
-
-    metadata_accepted = self.client_events.get()
-    self.assertIsNotNone(metadata_accepted)
-    self.assertEqual(_low.Event.Kind.METADATA_ACCEPTED, metadata_accepted.kind)
-    self.assertEqual(metadata_tag, metadata_accepted.tag)
-    metadata = dict(metadata_accepted.metadata)
-    self.assertIn(server_leading_metadata_key, metadata)
-    self.assertEqual(server_leading_metadata_value,
-                     metadata[server_leading_metadata_key])
-    self.assertIn(server_leading_binary_metadata_key, metadata)
-    self.assertEqual(server_leading_binary_metadata_value,
-                     metadata[server_leading_binary_metadata_key])
-
-    for datum in test_data:
-      client_call.write(datum, write_tag, _low.WriteFlags.WRITE_NO_COMPRESS)
-      write_accepted = self.client_events.get()
-      self.assertIsNotNone(write_accepted)
-      self.assertIs(write_accepted.kind, _low.Event.Kind.WRITE_ACCEPTED)
-      self.assertIs(write_accepted.tag, write_tag)
-      self.assertIs(write_accepted.write_accepted, True)
-
-      server_call.read(read_tag)
-      read_accepted = self.server_events.get()
-      self.assertIsNotNone(read_accepted)
-      self.assertEqual(_low.Event.Kind.READ_ACCEPTED, read_accepted.kind)
-      self.assertEqual(read_tag, read_accepted.tag)
-      self.assertIsNotNone(read_accepted.bytes)
-      server_data.append(read_accepted.bytes)
-
-      server_call.write(read_accepted.bytes, write_tag, 0)
-      write_accepted = self.server_events.get()
-      self.assertIsNotNone(write_accepted)
-      self.assertEqual(_low.Event.Kind.WRITE_ACCEPTED, write_accepted.kind)
-      self.assertEqual(write_tag, write_accepted.tag)
-      self.assertTrue(write_accepted.write_accepted)
-
-      client_call.read(read_tag)
-      read_accepted = self.client_events.get()
-      self.assertIsNotNone(read_accepted)
-      self.assertEqual(_low.Event.Kind.READ_ACCEPTED, read_accepted.kind)
-      self.assertEqual(read_tag, read_accepted.tag)
-      self.assertIsNotNone(read_accepted.bytes)
-      client_data.append(read_accepted.bytes)
-
-    client_call.complete(complete_tag)
-    complete_accepted = self.client_events.get()
-    self.assertIsNotNone(complete_accepted)
-    self.assertIs(complete_accepted.kind, _low.Event.Kind.COMPLETE_ACCEPTED)
-    self.assertIs(complete_accepted.tag, complete_tag)
-    self.assertIs(complete_accepted.complete_accepted, True)
-
-    server_call.read(read_tag)
-    read_accepted = self.server_events.get()
-    self.assertIsNotNone(read_accepted)
-    self.assertEqual(_low.Event.Kind.READ_ACCEPTED, read_accepted.kind)
-    self.assertEqual(read_tag, read_accepted.tag)
-    self.assertIsNone(read_accepted.bytes)
-
-    server_call.add_metadata(server_trailing_metadata_key,
-                             server_trailing_metadata_value)
-    server_call.add_metadata(server_trailing_binary_metadata_key,
-                             server_trailing_binary_metadata_value)
-
-    server_call.status(_low.Status(_low.Code.OK, details), status_tag)
-    server_terminal_event_one = self.server_events.get()
-    server_terminal_event_two = self.server_events.get()
-    if server_terminal_event_one.kind == _low.Event.Kind.COMPLETE_ACCEPTED:
-      status_accepted = server_terminal_event_one
-      rpc_accepted = server_terminal_event_two
-    else:
-      status_accepted = server_terminal_event_two
-      rpc_accepted = server_terminal_event_one
-    self.assertIsNotNone(status_accepted)
-    self.assertIsNotNone(rpc_accepted)
-    self.assertEqual(_low.Event.Kind.COMPLETE_ACCEPTED, status_accepted.kind)
-    self.assertEqual(status_tag, status_accepted.tag)
-    self.assertTrue(status_accepted.complete_accepted)
-    self.assertEqual(_low.Event.Kind.FINISH, rpc_accepted.kind)
-    self.assertEqual(finish_tag, rpc_accepted.tag)
-    self.assertEqual(_low.Status(_low.Code.OK, ''), rpc_accepted.status)
-
-    client_call.read(read_tag)
-    client_terminal_event_one = self.client_events.get()
-    client_terminal_event_two = self.client_events.get()
-    if client_terminal_event_one.kind == _low.Event.Kind.READ_ACCEPTED:
-      read_accepted = client_terminal_event_one
-      finish_accepted = client_terminal_event_two
-    else:
-      read_accepted = client_terminal_event_two
-      finish_accepted = client_terminal_event_one
-    self.assertIsNotNone(read_accepted)
-    self.assertIsNotNone(finish_accepted)
-    self.assertEqual(_low.Event.Kind.READ_ACCEPTED, read_accepted.kind)
-    self.assertEqual(read_tag, read_accepted.tag)
-    self.assertIsNone(read_accepted.bytes)
-    self.assertEqual(_low.Event.Kind.FINISH, finish_accepted.kind)
-    self.assertEqual(finish_tag, finish_accepted.tag)
-    self.assertEqual(_low.Status(_low.Code.OK, details), finish_accepted.status)
-    metadata = dict(finish_accepted.metadata)
-    self.assertIn(server_trailing_metadata_key, metadata)
-    self.assertEqual(server_trailing_metadata_value,
-                     metadata[server_trailing_metadata_key])
-    self.assertIn(server_trailing_binary_metadata_key, metadata)
-    self.assertEqual(server_trailing_binary_metadata_value,
-                     metadata[server_trailing_binary_metadata_key])
-    self.assertSetEqual(set(key for key, _ in finish_accepted.metadata),
-                        set((server_trailing_metadata_key,
-                             server_trailing_binary_metadata_key,)))
-
-    self.assertSequenceEqual(test_data, server_data)
-    self.assertSequenceEqual(test_data, client_data)
-
-  def testNoEcho(self):
-    self._perform_echo_test(())
-
-  def testOneByteEcho(self):
-    self._perform_echo_test([b'\x07'])
-
-  def testOneManyByteEcho(self):
-    self._perform_echo_test([_BYTE_SEQUENCE])
-
-  def testManyOneByteEchoes(self):
-    self._perform_echo_test(_BYTE_SEQUENCE)
-
-  def testManyManyByteEchoes(self):
-    self._perform_echo_test(_BYTE_SEQUENCE_SEQUENCE)
-
-
-class CancellationTest(unittest.TestCase):
-
-  def setUp(self):
-    self.host = 'localhost'
-
-    self.server_completion_queue = _low.CompletionQueue()
-    self.server = _low.Server(self.server_completion_queue)
-    port = self.server.add_http2_addr('[::]:0')
-    self.server.start()
-    self.server_events = queue.Queue()
-    self.server_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue,
-        args=(self.server_completion_queue, self.server_events))
-    self.server_completion_queue_thread.start()
-
-    self.client_completion_queue = _low.CompletionQueue()
-    self.channel = _low.Channel('%s:%d' % (self.host, port), None)
-    self.client_events = queue.Queue()
-    self.client_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue,
-        args=(self.client_completion_queue, self.client_events))
-    self.client_completion_queue_thread.start()
-
-  def tearDown(self):
-    self.server.stop()
-    self.server.cancel_all_calls()
-    self.server_completion_queue.stop()
-    self.client_completion_queue.stop()
-    self.server_completion_queue_thread.join()
-    self.client_completion_queue_thread.join()
-    del self.server
-
-  def testCancellation(self):
-    method = 'test method'
-    deadline = _FUTURE
-    metadata_tag = object()
-    finish_tag = object()
-    write_tag = object()
-    service_tag = object()
-    read_tag = object()
-    test_data = _BYTE_SEQUENCE_SEQUENCE
-
-    server_data = []
-    client_data = []
-
-    client_call = _low.Call(self.channel, self.client_completion_queue,
-                            method, self.host, deadline)
-
-    client_call.invoke(self.client_completion_queue, metadata_tag, finish_tag)
-
-    self.server.service(service_tag)
-    service_accepted = self.server_events.get()
-    server_call = service_accepted.service_acceptance.call
-
-    server_call.accept(self.server_completion_queue, finish_tag)
-    server_call.premetadata()
-
-    metadata_accepted = self.client_events.get()
-    self.assertIsNotNone(metadata_accepted)
-
-    for datum in test_data:
-      client_call.write(datum, write_tag, 0)
-      write_accepted = self.client_events.get()
-
-      server_call.read(read_tag)
-      read_accepted = self.server_events.get()
-      server_data.append(read_accepted.bytes)
-
-      server_call.write(read_accepted.bytes, write_tag, 0)
-      write_accepted = self.server_events.get()
-      self.assertIsNotNone(write_accepted)
-
-      client_call.read(read_tag)
-      read_accepted = self.client_events.get()
-      client_data.append(read_accepted.bytes)
-
-    client_call.cancel()
-    # cancel() is idempotent.
-    client_call.cancel()
-    client_call.cancel()
-    client_call.cancel()
-
-    server_call.read(read_tag)
-
-    server_terminal_event_one = self.server_events.get()
-    server_terminal_event_two = self.server_events.get()
-    if server_terminal_event_one.kind == _low.Event.Kind.READ_ACCEPTED:
-      read_accepted = server_terminal_event_one
-      rpc_accepted = server_terminal_event_two
-    else:
-      read_accepted = server_terminal_event_two
-      rpc_accepted = server_terminal_event_one
-    self.assertIsNotNone(read_accepted)
-    self.assertIsNotNone(rpc_accepted)
-    self.assertEqual(_low.Event.Kind.READ_ACCEPTED, read_accepted.kind)
-    self.assertIsNone(read_accepted.bytes)
-    self.assertEqual(_low.Event.Kind.FINISH, rpc_accepted.kind)
-    self.assertEqual(_low.Status(_low.Code.CANCELLED, ''), rpc_accepted.status)
-
-    finish_event = self.client_events.get()
-    self.assertEqual(_low.Event.Kind.FINISH, finish_event.kind)
-    self.assertEqual(_low.Status(_low.Code.CANCELLED, 'Cancelled'),
-                                 finish_event.status)
-
-    self.assertSequenceEqual(test_data, server_data)
-    self.assertSequenceEqual(test_data, client_data)
-
-
-class ExpirationTest(unittest.TestCase):
-
-  @unittest.skip('TODO(nathaniel): Expiration test!')
-  def testExpiration(self):
-    pass
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
-
diff --git a/src/python/grpcio/tests/unit/_adapter/_low_test.py b/src/python/grpcio/tests/unit/_adapter/_low_test.py
deleted file mode 100644
index ec46617996..0000000000
--- a/src/python/grpcio/tests/unit/_adapter/_low_test.py
+++ /dev/null
@@ -1,319 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import threading
-import time
-import unittest
-
-from grpc import _grpcio_metadata
-from grpc._adapter import _types
-from grpc._adapter import _low
-from tests.unit import test_common
-
-
-def wait_for_events(completion_queues, deadline):
-  """
-  Args:
-    completion_queues: list of completion queues to wait for events on
-    deadline: absolute deadline to wait until
-
-  Returns:
-    a sequence of events of length len(completion_queues).
-  """
-
-  results = [None] * len(completion_queues)
-  lock = threading.Lock()
-  threads = []
-  def set_ith_result(i, completion_queue):
-    result = completion_queue.next(deadline)
-    with lock:
-      results[i] = result
-  for i, completion_queue in enumerate(completion_queues):
-    thread = threading.Thread(target=set_ith_result,
-                              args=[i, completion_queue])
-    thread.start()
-    threads.append(thread)
-  for thread in threads:
-    thread.join()
-  return results
-
-
-class InsecureServerInsecureClient(unittest.TestCase):
-
-  def setUp(self):
-    self.server_completion_queue = _low.CompletionQueue()
-    self.server = _low.Server(self.server_completion_queue, [])
-    self.port = self.server.add_http2_port('[::]:0')
-    self.client_completion_queue = _low.CompletionQueue()
-    self.client_channel = _low.Channel('localhost:%d'%self.port, [])
-
-    self.server.start()
-
-  def tearDown(self):
-    self.server.shutdown()
-    del self.client_channel
-
-    self.client_completion_queue.shutdown()
-    while (self.client_completion_queue.next(float('+inf')).type !=
-           _types.EventType.QUEUE_SHUTDOWN):
-      pass
-    self.server_completion_queue.shutdown()
-    while (self.server_completion_queue.next(float('+inf')).type !=
-           _types.EventType.QUEUE_SHUTDOWN):
-      pass
-
-    del self.client_completion_queue
-    del self.server_completion_queue
-    del self.server
-
-  def testEcho(self):
-    deadline = time.time() + 5
-    event_time_tolerance = 2
-    deadline_tolerance = 0.25
-    client_metadata_ascii_key = 'key'
-    client_metadata_ascii_value = 'val'
-    client_metadata_bin_key = 'key-bin'
-    client_metadata_bin_value = b'\0'*1000
-    server_initial_metadata_key = 'init_me_me_me'
-    server_initial_metadata_value = 'whodawha?'
-    server_trailing_metadata_key = 'california_is_in_a_drought'
-    server_trailing_metadata_value = 'zomg it is'
-    server_status_code = _types.StatusCode.OK
-    server_status_details = 'our work is never over'
-    request = 'blarghaflargh'
-    response = 'his name is robert paulson'
-    method = 'twinkies'
-    host = 'hostess'
-    server_request_tag = object()
-    request_call_result = self.server.request_call(self.server_completion_queue,
-                                                   server_request_tag)
-
-    self.assertEqual(_types.CallError.OK, request_call_result)
-
-    client_call_tag = object()
-    client_call = self.client_channel.create_call(
-        self.client_completion_queue, method, host, deadline)
-    client_initial_metadata = [
-        (client_metadata_ascii_key, client_metadata_ascii_value),
-        (client_metadata_bin_key, client_metadata_bin_value)
-    ]
-    client_start_batch_result = client_call.start_batch([
-        _types.OpArgs.send_initial_metadata(client_initial_metadata),
-        _types.OpArgs.send_message(request, 0),
-        _types.OpArgs.send_close_from_client(),
-        _types.OpArgs.recv_initial_metadata(),
-        _types.OpArgs.recv_message(),
-        _types.OpArgs.recv_status_on_client()
-    ], client_call_tag)
-    self.assertEqual(_types.CallError.OK, client_start_batch_result)
-
-    client_no_event, request_event, = wait_for_events(
-        [self.client_completion_queue, self.server_completion_queue],
-        time.time() + event_time_tolerance)
-    self.assertEqual(client_no_event, None)
-    self.assertEqual(_types.EventType.OP_COMPLETE, request_event.type)
-    self.assertIsInstance(request_event.call, _low.Call)
-    self.assertIs(server_request_tag, request_event.tag)
-    self.assertEqual(1, len(request_event.results))
-    received_initial_metadata = request_event.results[0].initial_metadata
-    # Check that our metadata were transmitted
-    self.assertTrue(test_common.metadata_transmitted(client_initial_metadata,
-                                                     received_initial_metadata))
-    # Check that Python's user agent string is a part of the full user agent
-    # string
-    received_initial_metadata_dict = dict(received_initial_metadata)
-    self.assertIn('user-agent', received_initial_metadata_dict)
-    self.assertIn('Python-gRPC-{}'.format(_grpcio_metadata.__version__),
-                  received_initial_metadata_dict['user-agent'])
-    self.assertEqual(method, request_event.call_details.method)
-    self.assertEqual(host, request_event.call_details.host)
-    self.assertLess(abs(deadline - request_event.call_details.deadline),
-                    deadline_tolerance)
-
-    # Check that the channel is connected, and that both it and the call have
-    # the proper target and peer; do this after the first flurry of messages to
-    # avoid the possibility that connection was delayed by the core until the
-    # first message was sent.
-    self.assertEqual(_types.ConnectivityState.READY,
-                     self.client_channel.check_connectivity_state(False))
-    self.assertIsNotNone(self.client_channel.target())
-    self.assertIsNotNone(client_call.peer())
-
-    server_call_tag = object()
-    server_call = request_event.call
-    server_initial_metadata = [
-        (server_initial_metadata_key, server_initial_metadata_value)
-    ]
-    server_trailing_metadata = [
-        (server_trailing_metadata_key, server_trailing_metadata_value)
-    ]
-    server_start_batch_result = server_call.start_batch([
-        _types.OpArgs.send_initial_metadata(server_initial_metadata),
-        _types.OpArgs.recv_message(),
-        _types.OpArgs.send_message(response, 0),
-        _types.OpArgs.recv_close_on_server(),
-        _types.OpArgs.send_status_from_server(
-            server_trailing_metadata, server_status_code, server_status_details)
-    ], server_call_tag)
-    self.assertEqual(_types.CallError.OK, server_start_batch_result)
-
-    client_event, server_event, = wait_for_events(
-        [self.client_completion_queue, self.server_completion_queue],
-        time.time() + event_time_tolerance)
-
-    self.assertEqual(6, len(client_event.results))
-    found_client_op_types = set()
-    for client_result in client_event.results:
-      # we expect each op type to be unique
-      self.assertNotIn(client_result.type, found_client_op_types)
-      found_client_op_types.add(client_result.type)
-      if client_result.type == _types.OpType.RECV_INITIAL_METADATA:
-        self.assertTrue(
-            test_common.metadata_transmitted(server_initial_metadata,
-                                             client_result.initial_metadata))
-      elif client_result.type == _types.OpType.RECV_MESSAGE:
-        self.assertEqual(response, client_result.message)
-      elif client_result.type == _types.OpType.RECV_STATUS_ON_CLIENT:
-        self.assertTrue(
-            test_common.metadata_transmitted(server_trailing_metadata,
-                                             client_result.trailing_metadata))
-        self.assertEqual(server_status_details, client_result.status.details)
-        self.assertEqual(server_status_code, client_result.status.code)
-    self.assertEqual(set([
-          _types.OpType.SEND_INITIAL_METADATA,
-          _types.OpType.SEND_MESSAGE,
-          _types.OpType.SEND_CLOSE_FROM_CLIENT,
-          _types.OpType.RECV_INITIAL_METADATA,
-          _types.OpType.RECV_MESSAGE,
-          _types.OpType.RECV_STATUS_ON_CLIENT
-      ]), found_client_op_types)
-
-    self.assertEqual(5, len(server_event.results))
-    found_server_op_types = set()
-    for server_result in server_event.results:
-      self.assertNotIn(client_result.type, found_server_op_types)
-      found_server_op_types.add(server_result.type)
-      if server_result.type == _types.OpType.RECV_MESSAGE:
-        self.assertEqual(request, server_result.message)
-      elif server_result.type == _types.OpType.RECV_CLOSE_ON_SERVER:
-        self.assertFalse(server_result.cancelled)
-    self.assertEqual(set([
-          _types.OpType.SEND_INITIAL_METADATA,
-          _types.OpType.RECV_MESSAGE,
-          _types.OpType.SEND_MESSAGE,
-          _types.OpType.RECV_CLOSE_ON_SERVER,
-          _types.OpType.SEND_STATUS_FROM_SERVER
-      ]), found_server_op_types)
-
-    del client_call
-    del server_call
-
-
-class HangingServerShutdown(unittest.TestCase):
-
-  def setUp(self):
-    self.server_completion_queue = _low.CompletionQueue()
-    self.server = _low.Server(self.server_completion_queue, [])
-    self.port = self.server.add_http2_port('[::]:0')
-    self.client_completion_queue = _low.CompletionQueue()
-    self.client_channel = _low.Channel('localhost:%d'%self.port, [])
-
-    self.server.start()
-
-  def tearDown(self):
-    self.server.shutdown()
-    del self.client_channel
-
-    self.client_completion_queue.shutdown()
-    self.server_completion_queue.shutdown()
-    while True:
-      client_event, server_event = wait_for_events(
-          [self.client_completion_queue, self.server_completion_queue],
-          float("+inf"))
-      if (client_event.type == _types.EventType.QUEUE_SHUTDOWN and
-          server_event.type == _types.EventType.QUEUE_SHUTDOWN):
-        break
-
-    del self.client_completion_queue
-    del self.server_completion_queue
-    del self.server
-
-  def testHangingServerCall(self):
-    deadline = time.time() + 5
-    deadline_tolerance = 0.25
-    event_time_tolerance = 2
-    cancel_all_calls_time_tolerance = 0.5
-    request = 'blarghaflargh'
-    method = 'twinkies'
-    host = 'hostess'
-    server_request_tag = object()
-    request_call_result = self.server.request_call(self.server_completion_queue,
-                                                   server_request_tag)
-
-    client_call_tag = object()
-    client_call = self.client_channel.create_call(self.client_completion_queue,
-                                                  method, host, deadline)
-    client_start_batch_result = client_call.start_batch([
-        _types.OpArgs.send_initial_metadata([]),
-        _types.OpArgs.send_message(request, 0),
-        _types.OpArgs.send_close_from_client(),
-        _types.OpArgs.recv_initial_metadata(),
-        _types.OpArgs.recv_message(),
-        _types.OpArgs.recv_status_on_client()
-    ], client_call_tag)
-
-    client_no_event, request_event, = wait_for_events(
-        [self.client_completion_queue, self.server_completion_queue],
-        time.time() + event_time_tolerance)
-
-    # Now try to shutdown the server and expect that we see server shutdown
-    # almost immediately after calling cancel_all_calls.
-
-    # First attempt to cancel all calls before shutting down, and expect
-    # our state machine to catch the erroneous API use.
-    with self.assertRaises(RuntimeError):
-      self.server.cancel_all_calls()
-
-    shutdown_tag = object()
-    self.server.shutdown(shutdown_tag)
-    pre_cancel_timestamp = time.time()
-    self.server.cancel_all_calls()
-    finish_shutdown_timestamp = None
-    client_call_event, server_shutdown_event = wait_for_events(
-        [self.client_completion_queue, self.server_completion_queue],
-        time.time() + event_time_tolerance)
-    self.assertIs(shutdown_tag, server_shutdown_event.tag)
-    self.assertGreater(pre_cancel_timestamp + cancel_all_calls_time_tolerance,
-                       time.time())
-
-    del client_call
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_api_test.py b/src/python/grpcio/tests/unit/_api_test.py
new file mode 100644
index 0000000000..2fe89499f5
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_api_test.py
@@ -0,0 +1,111 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test of gRPC Python's application-layer API."""
+
+import unittest
+
+import six
+
+import grpc
+
+from tests.unit import _from_grpc_import_star
+
+
+class AllTest(unittest.TestCase):
+
+  def testAll(self):
+    expected_grpc_code_elements = (
+        'FutureTimeoutError',
+        'FutureCancelledError',
+        'Future',
+        'ChannelConnectivity',
+        'StatusCode',
+        'RpcError',
+        'RpcContext',
+        'Call',
+        'ChannelCredentials',
+        'CallCredentials',
+        'AuthMetadataContext',
+        'AuthMetadataPluginCallback',
+        'AuthMetadataPlugin',
+        'ServerCredentials',
+        'UnaryUnaryMultiCallable',
+        'UnaryStreamMultiCallable',
+        'StreamUnaryMultiCallable',
+        'StreamStreamMultiCallable',
+        'Channel',
+        'ServicerContext',
+        'RpcMethodHandler',
+        'HandlerCallDetails',
+        'GenericRpcHandler',
+        'Server',
+        'unary_unary_rpc_method_handler',
+        'unary_stream_rpc_method_handler',
+        'stream_unary_rpc_method_handler',
+        'stream_stream_rpc_method_handler',
+        'method_handlers_generic_handler',
+        'ssl_channel_credentials',
+        'metadata_call_credentials',
+        'access_token_call_credentials',
+        'composite_call_credentials',
+        'composite_channel_credentials',
+        'ssl_server_credentials',
+        'channel_ready_future',
+        'insecure_channel',
+        'secure_channel',
+        'server',
+    )
+
+    six.assertCountEqual(
+        self, expected_grpc_code_elements,
+        _from_grpc_import_star.GRPC_ELEMENTS)
+
+
+class ChannelConnectivityTest(unittest.TestCase):
+
+  def testChannelConnectivity(self):
+    self.assertSequenceEqual(
+        (grpc.ChannelConnectivity.IDLE,
+         grpc.ChannelConnectivity.CONNECTING,
+         grpc.ChannelConnectivity.READY,
+         grpc.ChannelConnectivity.TRANSIENT_FAILURE,
+         grpc.ChannelConnectivity.SHUTDOWN,),
+        tuple(grpc.ChannelConnectivity))
+
+
+class ChannelTest(unittest.TestCase):
+
+  def test_secure_channel(self):
+    channel_credentials = grpc.ssl_channel_credentials()
+    channel = grpc.secure_channel('google.com:443', channel_credentials)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_auth_test.py b/src/python/grpcio/tests/unit/_auth_test.py
new file mode 100644
index 0000000000..c31f7b06f7
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_auth_test.py
@@ -0,0 +1,96 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests of standard AuthMetadataPlugins."""
+
+import collections
+import threading
+import unittest
+
+from grpc import _auth
+
+
+class MockGoogleCreds(object):
+
+  def get_access_token(self):
+    token = collections.namedtuple('MockAccessTokenInfo',
+                                   ('access_token', 'expires_in'))
+    token.access_token = 'token'
+    return token
+
+
+class MockExceptionGoogleCreds(object):
+
+  def get_access_token(self):
+    raise Exception()
+
+
+class GoogleCallCredentialsTest(unittest.TestCase):
+
+  def test_google_call_credentials_success(self):
+    callback_event = threading.Event()
+
+    def mock_callback(metadata, error):
+      self.assertEqual(metadata, (('authorization', 'Bearer token'),))
+      self.assertIsNone(error)
+      callback_event.set()
+
+    call_creds = _auth.GoogleCallCredentials(MockGoogleCreds())
+    call_creds(None, mock_callback)
+    self.assertTrue(callback_event.wait(1.0))
+
+  def test_google_call_credentials_error(self):
+    callback_event = threading.Event()
+
+    def mock_callback(metadata, error):
+      self.assertIsNotNone(error)
+      callback_event.set()
+
+    call_creds = _auth.GoogleCallCredentials(MockExceptionGoogleCreds())
+    call_creds(None, mock_callback)
+    self.assertTrue(callback_event.wait(1.0))
+
+
+class AccessTokenCallCredentialsTest(unittest.TestCase):
+
+  def test_google_call_credentials_success(self):
+    callback_event = threading.Event()
+
+    def mock_callback(metadata, error):
+      self.assertEqual(metadata, (('authorization', 'Bearer token'),))
+      self.assertIsNone(error)
+      callback_event.set()
+
+    call_creds = _auth.AccessTokenCallCredentials('token')
+    call_creds(None, mock_callback)
+    self.assertTrue(callback_event.wait(1.0))
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_channel_connectivity_test.py b/src/python/grpcio/tests/unit/_channel_connectivity_test.py
new file mode 100644
index 0000000000..ae8de523ec
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_channel_connectivity_test.py
@@ -0,0 +1,161 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests of grpc._channel.Channel connectivity."""
+
+import threading
+import time
+import unittest
+from concurrent import futures
+
+import grpc
+from grpc import _channel
+from grpc import _server
+from tests.unit.framework.common import test_constants
+
+
+def _ready_in_connectivities(connectivities):
+  return grpc.ChannelConnectivity.READY in connectivities
+
+
+def _last_connectivity_is_not_ready(connectivities):
+  return connectivities[-1] is not grpc.ChannelConnectivity.READY
+
+
+class _Callback(object):
+
+  def __init__(self):
+    self._condition = threading.Condition()
+    self._connectivities = []
+
+  def update(self, connectivity):
+    with self._condition:
+      self._connectivities.append(connectivity)
+      self._condition.notify()
+
+  def connectivities(self):
+    with self._condition:
+      return tuple(self._connectivities)
+
+  def block_until_connectivities_satisfy(self, predicate):
+    with self._condition:
+      while True:
+        connectivities = tuple(self._connectivities)
+        if predicate(connectivities):
+          return connectivities
+        else:
+          self._condition.wait()
+
+
+class ChannelConnectivityTest(unittest.TestCase):
+
+  def test_lonely_channel_connectivity(self):
+    callback = _Callback()
+
+    channel = _channel.Channel('localhost:12345', None, None)
+    channel.subscribe(callback.update, try_to_connect=False)
+    first_connectivities = callback.block_until_connectivities_satisfy(bool)
+    channel.subscribe(callback.update, try_to_connect=True)
+    second_connectivities = callback.block_until_connectivities_satisfy(
+        lambda connectivities: 2 <= len(connectivities))
+    # Wait for a connection that will never happen.
+    time.sleep(test_constants.SHORT_TIMEOUT)
+    third_connectivities = callback.connectivities()
+    channel.unsubscribe(callback.update)
+    fourth_connectivities = callback.connectivities()
+    channel.unsubscribe(callback.update)
+    fifth_connectivities = callback.connectivities()
+
+    self.assertSequenceEqual(
+        (grpc.ChannelConnectivity.IDLE,), first_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.READY, second_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.READY, third_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.READY, fourth_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.READY, fifth_connectivities)
+
+  def test_immediately_connectable_channel_connectivity(self):
+    server = _server.Server((), futures.ThreadPoolExecutor(max_workers=0))
+    port = server.add_insecure_port('[::]:0')
+    server.start()
+    first_callback = _Callback()
+    second_callback = _Callback()
+
+    channel = _channel.Channel('localhost:{}'.format(port), None, None)
+    channel.subscribe(first_callback.update, try_to_connect=False)
+    first_connectivities = first_callback.block_until_connectivities_satisfy(
+        bool)
+    # Wait for a connection that will never happen because try_to_connect=True
+    # has not yet been passed.
+    time.sleep(test_constants.SHORT_TIMEOUT)
+    second_connectivities = first_callback.connectivities()
+    channel.subscribe(second_callback.update, try_to_connect=True)
+    third_connectivities = first_callback.block_until_connectivities_satisfy(
+        lambda connectivities: 2 <= len(connectivities))
+    fourth_connectivities = second_callback.block_until_connectivities_satisfy(
+        bool)
+    # Wait for a connection that will happen (or may already have happened).
+    first_callback.block_until_connectivities_satisfy(_ready_in_connectivities)
+    second_callback.block_until_connectivities_satisfy(_ready_in_connectivities)
+    del channel
+
+    self.assertSequenceEqual(
+        (grpc.ChannelConnectivity.IDLE,), first_connectivities)
+    self.assertSequenceEqual(
+        (grpc.ChannelConnectivity.IDLE,), second_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.TRANSIENT_FAILURE, third_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.SHUTDOWN, third_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.TRANSIENT_FAILURE,
+        fourth_connectivities)
+    self.assertNotIn(
+        grpc.ChannelConnectivity.SHUTDOWN, fourth_connectivities)
+
+  def test_reachable_then_unreachable_channel_connectivity(self):
+    server = _server.Server((), futures.ThreadPoolExecutor(max_workers=0))
+    port = server.add_insecure_port('[::]:0')
+    server.start()
+    callback = _Callback()
+
+    channel = _channel.Channel('localhost:{}'.format(port), None, None)
+    channel.subscribe(callback.update, try_to_connect=True)
+    callback.block_until_connectivities_satisfy(_ready_in_connectivities)
+    # Now take down the server and confirm that channel readiness is repudiated.
+    server.stop(None)
+    callback.block_until_connectivities_satisfy(_last_connectivity_is_not_ready)
+    channel.unsubscribe(callback.update)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_channel_ready_future_test.py b/src/python/grpcio/tests/unit/_channel_ready_future_test.py
new file mode 100644
index 0000000000..b84bc0197a
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_channel_ready_future_test.py
@@ -0,0 +1,103 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests of grpc.channel_ready_future."""
+
+import threading
+import unittest
+from concurrent import futures
+
+import grpc
+from grpc import _channel
+from grpc import _server
+from tests.unit.framework.common import test_constants
+
+
+class _Callback(object):
+
+  def __init__(self):
+    self._condition = threading.Condition()
+    self._value = None
+
+  def accept_value(self, value):
+    with self._condition:
+      self._value = value
+      self._condition.notify_all()
+
+  def block_until_called(self):
+    with self._condition:
+      while self._value is None:
+        self._condition.wait()
+      return self._value
+
+
+class ChannelReadyFutureTest(unittest.TestCase):
+
+  def test_lonely_channel_connectivity(self):
+    channel = grpc.insecure_channel('localhost:12345')
+    callback = _Callback()
+
+    ready_future = grpc.channel_ready_future(channel)
+    ready_future.add_done_callback(callback.accept_value)
+    with self.assertRaises(grpc.FutureTimeoutError):
+      ready_future.result(test_constants.SHORT_TIMEOUT)
+    self.assertFalse(ready_future.cancelled())
+    self.assertFalse(ready_future.done())
+    self.assertTrue(ready_future.running())
+    ready_future.cancel()
+    value_passed_to_callback = callback.block_until_called()
+    self.assertIs(ready_future, value_passed_to_callback)
+    self.assertTrue(ready_future.cancelled())
+    self.assertTrue(ready_future.done())
+    self.assertFalse(ready_future.running())
+
+  def test_immediately_connectable_channel_connectivity(self):
+    server = _server.Server((), futures.ThreadPoolExecutor(max_workers=0))
+    port = server.add_insecure_port('[::]:0')
+    server.start()
+    channel = grpc.insecure_channel('localhost:{}'.format(port))
+    callback = _Callback()
+
+    ready_future = grpc.channel_ready_future(channel)
+    ready_future.add_done_callback(callback.accept_value)
+    self.assertIsNone(ready_future.result(test_constants.SHORT_TIMEOUT))
+    value_passed_to_callback = callback.block_until_called()
+    self.assertIs(ready_future, value_passed_to_callback)
+    self.assertFalse(ready_future.cancelled())
+    self.assertTrue(ready_future.done())
+    self.assertFalse(ready_future.running())
+    # Cancellation after maturity has no effect.
+    ready_future.cancel()
+    self.assertFalse(ready_future.cancelled())
+    self.assertTrue(ready_future.done())
+    self.assertFalse(ready_future.running())
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_compression_test.py b/src/python/grpcio/tests/unit/_compression_test.py
new file mode 100644
index 0000000000..9e8b8578c1
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_compression_test.py
@@ -0,0 +1,133 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Tests server and client side compression."""
+
+import unittest
+
+import grpc
+from grpc import _grpcio_metadata
+from grpc.framework.foundation import logging_pool
+
+from tests.unit import test_common
+from tests.unit.framework.common import test_constants
+
+_UNARY_UNARY = '/test/UnaryUnary'
+_STREAM_STREAM = '/test/StreamStream'
+
+
+def handle_unary(request, servicer_context):
+  servicer_context.send_initial_metadata([
+    ('grpc-internal-encoding-request', 'gzip')])
+  return request
+
+
+def handle_stream(request_iterator, servicer_context):
+  # TODO(issue:#6891) We should be able to remove this loop,
+  # and replace with return; yield
+  servicer_context.send_initial_metadata([
+    ('grpc-internal-encoding-request', 'gzip')])
+  for request in request_iterator:
+    yield request
+
+
+class _MethodHandler(grpc.RpcMethodHandler):
+
+  def __init__(self, request_streaming, response_streaming):
+    self.request_streaming = request_streaming
+    self.response_streaming = response_streaming
+    self.request_deserializer = None
+    self.response_serializer = None
+    self.unary_unary = None
+    self.unary_stream = None
+    self.stream_unary = None
+    self.stream_stream = None
+    if self.request_streaming and self.response_streaming:
+      self.stream_stream = lambda x, y: handle_stream(x, y)
+    elif not self.request_streaming and not self.response_streaming:
+      self.unary_unary = lambda x, y: handle_unary(x, y)
+
+
+class _GenericHandler(grpc.GenericRpcHandler):
+
+  def service(self, handler_call_details):
+    if handler_call_details.method == _UNARY_UNARY:
+      return _MethodHandler(False, False)
+    elif handler_call_details.method == _STREAM_STREAM:
+      return _MethodHandler(True, True)
+    else:
+      return None
+
+
+class CompressionTest(unittest.TestCase):
+
+  def setUp(self):
+    self._server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+    self._server = grpc.server((_GenericHandler(),), self._server_pool)
+    self._port = self._server.add_insecure_port('[::]:0')
+    self._server.start()
+
+  def testUnary(self):
+    request = b'\x00' * 100
+
+    # Client -> server compressed through default client channel compression
+    # settings. Server -> client compressed via server-side metadata setting.
+    # TODO(https://github.com/grpc/grpc/issues/4078): replace the "1" integer
+    # literal with proper use of the public API.
+    compressed_channel = grpc.insecure_channel('localhost:%d' % self._port,
+        options=[('grpc.default_compression_algorithm', 1)])
+    multi_callable = compressed_channel.unary_unary(_UNARY_UNARY)
+    response = multi_callable(request)
+    self.assertEqual(request, response)
+
+    # Client -> server compressed through client metadata setting. Server ->
+    # client compressed via server-side metadata setting.
+    # TODO(https://github.com/grpc/grpc/issues/4078): replace the "0" integer
+    # literal with proper use of the public API.
+    uncompressed_channel = grpc.insecure_channel('localhost:%d' % self._port,
+        options=[('grpc.default_compression_algorithm', 0)])
+    multi_callable = compressed_channel.unary_unary(_UNARY_UNARY)
+    response = multi_callable(request, metadata=[
+      ('grpc-internal-encoding-request', 'gzip')])
+    self.assertEqual(request, response)
+
+  def testStreaming(self):
+    request = b'\x00' * 100
+
+    # TODO(https://github.com/grpc/grpc/issues/4078): replace the "1" integer
+    # literal with proper use of the public API.
+    compressed_channel = grpc.insecure_channel('localhost:%d' % self._port,
+        options=[('grpc.default_compression_algorithm', 1)])
+    multi_callable = compressed_channel.stream_stream(_STREAM_STREAM)
+    call = multi_callable([request] * test_constants.STREAM_LENGTH)
+    for response in call:
+      self.assertEqual(request, response)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py b/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py
deleted file mode 100644
index 2b8981c752..0000000000
--- a/src/python/grpcio/tests/unit/_core_over_links_base_interface_test.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests Base interface compliance of the core-over-gRPC-links stack."""
-
-import collections
-import logging
-import random
-import time
-import unittest
-
-import six
-
-from grpc._adapter import _intermediary_low
-from grpc._links import invocation
-from grpc._links import service
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.core import implementations
-from grpc.framework.interfaces.base import utilities
-from tests.unit import test_common as grpc_test_common
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.interfaces.base import test_cases
-from tests.unit.framework.interfaces.base import test_interfaces
-
-
-class _SerializationBehaviors(
-    collections.namedtuple(
-        '_SerializationBehaviors',
-        ('request_serializers', 'request_deserializers', 'response_serializers',
-         'response_deserializers',))):
-  pass
-
-
-class _Links(
-    collections.namedtuple(
-        '_Links',
-        ('invocation_end_link', 'invocation_grpc_link', 'service_grpc_link',
-         'service_end_link'))):
-  pass
-
-
-def _serialization_behaviors_from_serializations(serializations):
-  request_serializers = {}
-  request_deserializers = {}
-  response_serializers = {}
-  response_deserializers = {}
-  for (group, method), serialization in six.iteritems(serializations):
-    request_serializers[group, method] = serialization.serialize_request
-    request_deserializers[group, method] = serialization.deserialize_request
-    response_serializers[group, method] = serialization.serialize_response
-    response_deserializers[group, method] = serialization.deserialize_response
-  return _SerializationBehaviors(
-      request_serializers, request_deserializers, response_serializers,
-      response_deserializers)
-
-
-class _Implementation(test_interfaces.Implementation):
-
-  def instantiate(self, serializations, servicer):
-    serialization_behaviors = _serialization_behaviors_from_serializations(
-        serializations)
-    invocation_end_link = implementations.invocation_end_link()
-    service_end_link = implementations.service_end_link(
-        servicer, test_constants.DEFAULT_TIMEOUT,
-        test_constants.MAXIMUM_TIMEOUT)
-    service_grpc_link = service.service_link(
-        serialization_behaviors.request_deserializers,
-        serialization_behaviors.response_serializers)
-    port = service_grpc_link.add_port('[::]:0', None)
-    channel = _intermediary_low.Channel('localhost:%d' % port, None)
-    invocation_grpc_link = invocation.invocation_link(
-        channel, b'localhost', None,
-        serialization_behaviors.request_serializers,
-        serialization_behaviors.response_deserializers)
-
-    invocation_end_link.join_link(invocation_grpc_link)
-    invocation_grpc_link.join_link(invocation_end_link)
-    service_end_link.join_link(service_grpc_link)
-    service_grpc_link.join_link(service_end_link)
-    invocation_grpc_link.start()
-    service_grpc_link.start()
-    return invocation_end_link, service_end_link, (
-        invocation_grpc_link, service_grpc_link)
-
-  def destantiate(self, memo):
-    invocation_grpc_link, service_grpc_link = memo
-    invocation_grpc_link.stop()
-    service_grpc_link.begin_stop()
-    service_grpc_link.end_stop()
-
-  def invocation_initial_metadata(self):
-    return grpc_test_common.INVOCATION_INITIAL_METADATA
-
-  def service_initial_metadata(self):
-    return grpc_test_common.SERVICE_INITIAL_METADATA
-
-  def invocation_completion(self):
-    return utilities.completion(None, None, None)
-
-  def service_completion(self):
-    return utilities.completion(
-        grpc_test_common.SERVICE_TERMINAL_METADATA,
-        beta_interfaces.StatusCode.OK, grpc_test_common.DETAILS)
-
-  def metadata_transmitted(self, original_metadata, transmitted_metadata):
-    return original_metadata is None or grpc_test_common.metadata_transmitted(
-        original_metadata, transmitted_metadata)
-
-  def completion_transmitted(self, original_completion, transmitted_completion):
-    if (original_completion.terminal_metadata is not None and
-        not grpc_test_common.metadata_transmitted(
-            original_completion.terminal_metadata,
-            transmitted_completion.terminal_metadata)):
-        return False
-    elif original_completion.code is not transmitted_completion.code:
-      return False
-    elif original_completion.message != transmitted_completion.message:
-      return False
-    else:
-      return True
-
-
-def load_tests(loader, tests, pattern):
-  return unittest.TestSuite(
-      tests=tuple(
-          loader.loadTestsFromTestCase(test_case_class)
-          for test_case_class in test_cases.test_cases(_Implementation())))
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py b/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py
deleted file mode 100644
index 50b9a5a824..0000000000
--- a/src/python/grpcio/tests/unit/_crust_over_core_over_links_face_interface_test.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests Face compliance of the crust-over-core-over-gRPC-links stack."""
-
-import collections
-import unittest
-
-import six
-
-from grpc._adapter import _intermediary_low
-from grpc._links import invocation
-from grpc._links import service
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.core import implementations as core_implementations
-from grpc.framework.crust import implementations as crust_implementations
-from grpc.framework.foundation import logging_pool
-from grpc.framework.interfaces.links import utilities
-from tests.unit import test_common as grpc_test_common
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.interfaces.face import test_cases
-from tests.unit.framework.interfaces.face import test_interfaces
-
-
-class _SerializationBehaviors(
-    collections.namedtuple(
-        '_SerializationBehaviors',
-        ('request_serializers', 'request_deserializers', 'response_serializers',
-         'response_deserializers',))):
-  pass
-
-
-def _serialization_behaviors_from_test_methods(test_methods):
-  request_serializers = {}
-  request_deserializers = {}
-  response_serializers = {}
-  response_deserializers = {}
-  for (group, method), test_method in six.iteritems(test_methods):
-    request_serializers[group, method] = test_method.serialize_request
-    request_deserializers[group, method] = test_method.deserialize_request
-    response_serializers[group, method] = test_method.serialize_response
-    response_deserializers[group, method] = test_method.deserialize_response
-  return _SerializationBehaviors(
-      request_serializers, request_deserializers, response_serializers,
-      response_deserializers)
-
-
-class _Implementation(test_interfaces.Implementation):
-
-  def instantiate(
-      self, methods, method_implementations, multi_method_implementation):
-    pool = logging_pool.pool(test_constants.POOL_SIZE)
-    servicer = crust_implementations.servicer(
-        method_implementations, multi_method_implementation, pool)
-    serialization_behaviors = _serialization_behaviors_from_test_methods(
-        methods)
-    invocation_end_link = core_implementations.invocation_end_link()
-    service_end_link = core_implementations.service_end_link(
-        servicer, test_constants.DEFAULT_TIMEOUT,
-        test_constants.MAXIMUM_TIMEOUT)
-    service_grpc_link = service.service_link(
-        serialization_behaviors.request_deserializers,
-        serialization_behaviors.response_serializers)
-    port = service_grpc_link.add_port('[::]:0', None)
-    channel = _intermediary_low.Channel('localhost:%d' % port, None)
-    invocation_grpc_link = invocation.invocation_link(
-        channel, b'localhost', None,
-        serialization_behaviors.request_serializers,
-        serialization_behaviors.response_deserializers)
-
-    invocation_end_link.join_link(invocation_grpc_link)
-    invocation_grpc_link.join_link(invocation_end_link)
-    service_grpc_link.join_link(service_end_link)
-    service_end_link.join_link(service_grpc_link)
-    service_end_link.start()
-    invocation_end_link.start()
-    invocation_grpc_link.start()
-    service_grpc_link.start()
-
-    generic_stub = crust_implementations.generic_stub(invocation_end_link, pool)
-    # TODO(nathaniel): Add a "groups" attribute to _digest.TestServiceDigest.
-    group = next(iter(methods))[0]
-    # TODO(nathaniel): Add a "cardinalities_by_group" attribute to
-    # _digest.TestServiceDigest.
-    cardinalities = {
-        method: method_object.cardinality()
-        for (group, method), method_object in six.iteritems(methods)}
-    dynamic_stub = crust_implementations.dynamic_stub(
-        invocation_end_link, group, cardinalities, pool)
-
-    return generic_stub, {group: dynamic_stub}, (
-        invocation_end_link, invocation_grpc_link, service_grpc_link,
-        service_end_link, pool)
-
-  def destantiate(self, memo):
-    (invocation_end_link, invocation_grpc_link, service_grpc_link,
-     service_end_link, pool) = memo
-    invocation_end_link.stop(0).wait()
-    invocation_grpc_link.stop()
-    service_grpc_link.begin_stop()
-    service_end_link.stop(0).wait()
-    service_grpc_link.end_stop()
-    invocation_end_link.join_link(utilities.NULL_LINK)
-    invocation_grpc_link.join_link(utilities.NULL_LINK)
-    service_grpc_link.join_link(utilities.NULL_LINK)
-    service_end_link.join_link(utilities.NULL_LINK)
-    pool.shutdown(wait=True)
-
-  def invocation_metadata(self):
-    return grpc_test_common.INVOCATION_INITIAL_METADATA
-
-  def initial_metadata(self):
-    return grpc_test_common.SERVICE_INITIAL_METADATA
-
-  def terminal_metadata(self):
-    return grpc_test_common.SERVICE_TERMINAL_METADATA
-
-  def code(self):
-    return beta_interfaces.StatusCode.OK
-
-  def details(self):
-    return grpc_test_common.DETAILS
-
-  def metadata_transmitted(self, original_metadata, transmitted_metadata):
-    return original_metadata is None or grpc_test_common.metadata_transmitted(
-        original_metadata, transmitted_metadata)
-
-
-def load_tests(loader, tests, pattern):
-  return unittest.TestSuite(
-      tests=tuple(
-          loader.loadTestsFromTestCase(test_case_class)
-          for test_case_class in test_cases.test_cases(_Implementation())))
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py
new file mode 100644
index 0000000000..cac0c8b3b9
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py
@@ -0,0 +1,222 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test making many calls and immediately cancelling most of them."""
+
+import threading
+import unittest
+
+from grpc._cython import cygrpc
+from grpc.framework.foundation import logging_pool
+from tests.unit.framework.common import test_constants
+
+_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
+_EMPTY_FLAGS = 0
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+_SERVER_SHUTDOWN_TAG = 'server_shutdown'
+_REQUEST_CALL_TAG = 'request_call'
+_RECEIVE_CLOSE_ON_SERVER_TAG = 'receive_close_on_server'
+_RECEIVE_MESSAGE_TAG = 'receive_message'
+_SERVER_COMPLETE_CALL_TAG = 'server_complete_call'
+
+_SUCCESS_CALL_FRACTION = 1.0 / 8.0
+
+
+class _State(object):
+
+  def __init__(self):
+    self.condition = threading.Condition()
+    self.handlers_released = False
+    self.parked_handlers = 0
+    self.handled_rpcs = 0
+
+
+def _is_cancellation_event(event):
+  return (
+      event.tag is _RECEIVE_CLOSE_ON_SERVER_TAG and
+      event.batch_operations[0].received_cancelled)
+
+
+class _Handler(object):
+
+  def __init__(self, state, completion_queue, rpc_event):
+    self._state = state
+    self._lock = threading.Lock()
+    self._completion_queue = completion_queue
+    self._call = rpc_event.operation_call
+
+  def __call__(self):
+    with self._state.condition:
+      self._state.parked_handlers += 1
+      if self._state.parked_handlers == test_constants.THREAD_CONCURRENCY:
+        self._state.condition.notify_all()
+      while not self._state.handlers_released:
+        self._state.condition.wait()
+
+    with self._lock:
+      self._call.start_batch(
+          cygrpc.Operations(
+              (cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),)),
+          _RECEIVE_CLOSE_ON_SERVER_TAG)
+      self._call.start_batch(
+          cygrpc.Operations((cygrpc.operation_receive_message(_EMPTY_FLAGS),)),
+          _RECEIVE_MESSAGE_TAG)
+    first_event = self._completion_queue.poll()
+    if _is_cancellation_event(first_event):
+      self._completion_queue.poll()
+    else:
+      with self._lock:
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(b'\x79\x57', _EMPTY_FLAGS),
+            cygrpc.operation_send_status_from_server(
+                _EMPTY_METADATA, cygrpc.StatusCode.ok, b'test details!',
+                _EMPTY_FLAGS),
+        )
+        self._call.start_batch(
+            cygrpc.Operations(operations), _SERVER_COMPLETE_CALL_TAG)
+      self._completion_queue.poll()
+      self._completion_queue.poll()
+
+
+def _serve(state, server, server_completion_queue, thread_pool):
+  for _ in range(test_constants.RPC_CONCURRENCY):
+    call_completion_queue = cygrpc.CompletionQueue()
+    server.request_call(
+        call_completion_queue, server_completion_queue, _REQUEST_CALL_TAG)
+    rpc_event = server_completion_queue.poll()
+    thread_pool.submit(_Handler(state, call_completion_queue, rpc_event))
+    with state.condition:
+      state.handled_rpcs += 1
+      if test_constants.RPC_CONCURRENCY <= state.handled_rpcs:
+        state.condition.notify_all()
+  server_completion_queue.poll()
+
+
+class _QueueDriver(object):
+
+  def __init__(self, condition, completion_queue, due):
+    self._condition = condition
+    self._completion_queue = completion_queue
+    self._due = due
+    self._events = []
+    self._returned = False
+
+  def start(self):
+    def in_thread():
+      while True:
+        event = self._completion_queue.poll()
+        with self._condition:
+          self._events.append(event)
+          self._due.remove(event.tag)
+          self._condition.notify_all()
+          if not self._due:
+            self._returned = True
+            return
+    thread = threading.Thread(target=in_thread)
+    thread.start()
+
+  def events(self, at_least):
+    with self._condition:
+      while len(self._events) < at_least:
+        self._condition.wait()
+      return tuple(self._events)
+
+
+class CancelManyCallsTest(unittest.TestCase):
+
+  def testCancelManyCalls(self):
+    server_thread_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+
+    server_completion_queue = cygrpc.CompletionQueue()
+    server = cygrpc.Server()
+    server.register_completion_queue(server_completion_queue)
+    port = server.add_http2_port(b'[::]:0')
+    server.start()
+    channel = cygrpc.Channel('localhost:{}'.format(port).encode())
+
+    state = _State()
+
+    server_thread_args = (
+        state, server, server_completion_queue, server_thread_pool,)
+    server_thread = threading.Thread(target=_serve, args=server_thread_args)
+    server_thread.start()
+
+    client_condition = threading.Condition()
+    client_due = set()
+    client_completion_queue = cygrpc.CompletionQueue()
+    client_driver = _QueueDriver(
+        client_condition, client_completion_queue, client_due)
+    client_driver.start()
+
+    with client_condition:
+      client_calls = []
+      for index in range(test_constants.RPC_CONCURRENCY):
+        client_call = channel.create_call(
+            None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', None,
+            _INFINITE_FUTURE)
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(b'\x45\x56', _EMPTY_FLAGS),
+            cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+            cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+            cygrpc.operation_receive_message(_EMPTY_FLAGS),
+            cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+        )
+        tag = 'client_complete_call_{0:04d}_tag'.format(index)
+        client_call.start_batch(cygrpc.Operations(operations), tag)
+        client_due.add(tag)
+        client_calls.append(client_call)
+
+    with state.condition:
+      while True:
+        if state.parked_handlers < test_constants.THREAD_CONCURRENCY:
+          state.condition.wait()
+        elif state.handled_rpcs < test_constants.RPC_CONCURRENCY:
+          state.condition.wait()
+        else:
+          state.handlers_released = True
+          state.condition.notify_all()
+          break
+
+    client_driver.events(
+        test_constants.RPC_CONCURRENCY * _SUCCESS_CALL_FRACTION)
+    with client_condition:
+      for client_call in client_calls:
+        client_call.cancel()
+
+    with state.condition:
+      server.shutdown(server_completion_queue, _SERVER_SHUTDOWN_TAG)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_cython/_channel_test.py b/src/python/grpcio/tests/unit/_cython/_channel_test.py
index b414f8e6f6..f9c8a3ac62 100644
--- a/src/python/grpcio/tests/unit/_cython/_channel_test.py
+++ b/src/python/grpcio/tests/unit/_cython/_channel_test.py
@@ -33,12 +33,11 @@ import unittest
 
 from grpc._cython import cygrpc
 
-# TODO(nathaniel): This should be at least one hundred. Why not one thousand?
-_PARALLELISM = 4
+from tests.unit.framework.common import test_constants
 
 
 def _channel_and_completion_queue():
-  channel = cygrpc.Channel('localhost:54321', cygrpc.ChannelArgs(()))
+  channel = cygrpc.Channel(b'localhost:54321', cygrpc.ChannelArgs(()))
   completion_queue = cygrpc.CompletionQueue()
   return channel, completion_queue
 
@@ -61,7 +60,7 @@ def _create_loop_destroy():
 def _in_parallel(behavior, arguments):
   threads = tuple(
       threading.Thread(target=behavior, args=arguments)
-      for _ in range(_PARALLELISM))
+      for _ in range(test_constants.THREAD_CONCURRENCY))
   for thread in threads:
     thread.start()
   for thread in threads:
diff --git a/src/python/grpcio/tests/unit/_cython/_read_some_but_not_all_responses_test.py b/src/python/grpcio/tests/unit/_cython/_read_some_but_not_all_responses_test.py
new file mode 100644
index 0000000000..27fcee0d6f
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_cython/_read_some_but_not_all_responses_test.py
@@ -0,0 +1,251 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test a corner-case at the level of the Cython API."""
+
+import threading
+import unittest
+
+from grpc._cython import cygrpc
+
+_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
+_EMPTY_FLAGS = 0
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+
+class _ServerDriver(object):
+
+  def __init__(self, completion_queue, shutdown_tag):
+    self._condition = threading.Condition()
+    self._completion_queue = completion_queue
+    self._shutdown_tag = shutdown_tag
+    self._events = []
+    self._saw_shutdown_tag = False
+
+  def start(self):
+    def in_thread():
+      while True:
+        event = self._completion_queue.poll()
+        with self._condition:
+          self._events.append(event)
+          self._condition.notify()
+          if event.tag is self._shutdown_tag:
+            self._saw_shutdown_tag = True
+            break
+    thread = threading.Thread(target=in_thread)
+    thread.start()
+
+  def done(self):
+    with self._condition:
+      return self._saw_shutdown_tag
+
+  def first_event(self):
+    with self._condition:
+      while not self._events:
+        self._condition.wait()
+      return self._events[0]
+
+  def events(self):
+    with self._condition:
+      while not self._saw_shutdown_tag:
+        self._condition.wait()
+      return tuple(self._events)
+
+
+class _QueueDriver(object):
+
+  def __init__(self, condition, completion_queue, due):
+    self._condition = condition
+    self._completion_queue = completion_queue
+    self._due = due
+    self._events = []
+    self._returned = False
+
+  def start(self):
+    def in_thread():
+      while True:
+        event = self._completion_queue.poll()
+        with self._condition:
+          self._events.append(event)
+          self._due.remove(event.tag)
+          self._condition.notify_all()
+          if not self._due:
+            self._returned = True
+            return
+    thread = threading.Thread(target=in_thread)
+    thread.start()
+
+  def done(self):
+    with self._condition:
+      return self._returned
+
+  def event_with_tag(self, tag):
+    with self._condition:
+      while True:
+        for event in self._events:
+          if event.tag is tag:
+            return event
+        self._condition.wait()
+
+  def events(self):
+    with self._condition:
+      while not self._returned:
+        self._condition.wait()
+      return tuple(self._events)
+
+
+class ReadSomeButNotAllResponsesTest(unittest.TestCase):
+
+  def testReadSomeButNotAllResponses(self):
+    server_completion_queue = cygrpc.CompletionQueue()
+    server = cygrpc.Server()
+    server.register_completion_queue(server_completion_queue)
+    port = server.add_http2_port(b'[::]:0')
+    server.start()
+    channel = cygrpc.Channel('localhost:{}'.format(port).encode())
+
+    server_shutdown_tag = 'server_shutdown_tag'
+    server_driver = _ServerDriver(server_completion_queue, server_shutdown_tag)
+    server_driver.start()
+
+    client_condition = threading.Condition()
+    client_due = set()
+    client_completion_queue = cygrpc.CompletionQueue()
+    client_driver = _QueueDriver(
+        client_condition, client_completion_queue, client_due)
+    client_driver.start()
+
+    server_call_condition = threading.Condition()
+    server_send_initial_metadata_tag = 'server_send_initial_metadata_tag'
+    server_send_first_message_tag = 'server_send_first_message_tag'
+    server_send_second_message_tag = 'server_send_second_message_tag'
+    server_complete_rpc_tag = 'server_complete_rpc_tag'
+    server_call_due = set((
+        server_send_initial_metadata_tag,
+        server_send_first_message_tag,
+        server_send_second_message_tag,
+        server_complete_rpc_tag,
+    ))
+    server_call_completion_queue = cygrpc.CompletionQueue()
+    server_call_driver = _QueueDriver(
+        server_call_condition, server_call_completion_queue, server_call_due)
+    server_call_driver.start()
+
+    server_rpc_tag = 'server_rpc_tag'
+    request_call_result = server.request_call(
+        server_call_completion_queue, server_completion_queue, server_rpc_tag)
+
+    client_call = channel.create_call(
+        None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', None,
+        _INFINITE_FUTURE)
+    client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag'
+    client_complete_rpc_tag = 'client_complete_rpc_tag'
+    with client_condition:
+      client_receive_initial_metadata_start_batch_result = (
+          client_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+          ]), client_receive_initial_metadata_tag))
+      client_due.add(client_receive_initial_metadata_tag)
+      client_complete_rpc_start_batch_result = (
+          client_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_send_initial_metadata(
+                  _EMPTY_METADATA, _EMPTY_FLAGS),
+              cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+              cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+          ]), client_complete_rpc_tag))
+      client_due.add(client_complete_rpc_tag)
+
+    server_rpc_event = server_driver.first_event()
+
+    with server_call_condition:
+      server_send_initial_metadata_start_batch_result = (
+          server_rpc_event.operation_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_send_initial_metadata(
+                  _EMPTY_METADATA, _EMPTY_FLAGS),
+          ]), server_send_initial_metadata_tag))
+      server_send_first_message_start_batch_result = (
+          server_rpc_event.operation_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_send_message(b'\x07', _EMPTY_FLAGS),
+          ]), server_send_first_message_tag))
+    server_send_initial_metadata_event = server_call_driver.event_with_tag(
+        server_send_initial_metadata_tag)
+    server_send_first_message_event = server_call_driver.event_with_tag(
+        server_send_first_message_tag)
+    with server_call_condition:
+      server_send_second_message_start_batch_result = (
+          server_rpc_event.operation_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_send_message(b'\x07', _EMPTY_FLAGS),
+          ]), server_send_second_message_tag))
+      server_complete_rpc_start_batch_result = (
+          server_rpc_event.operation_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),
+              cygrpc.operation_send_status_from_server(
+                  cygrpc.Metadata(()), cygrpc.StatusCode.ok, b'test details',
+                  _EMPTY_FLAGS),
+          ]), server_complete_rpc_tag))
+    server_send_second_message_event = server_call_driver.event_with_tag(
+        server_send_second_message_tag)
+    server_complete_rpc_event = server_call_driver.event_with_tag(
+        server_complete_rpc_tag)
+    server_call_driver.events()
+
+    with client_condition:
+      client_receive_first_message_tag = 'client_receive_first_message_tag'
+      client_receive_first_message_start_batch_result = (
+          client_call.start_batch(cygrpc.Operations([
+              cygrpc.operation_receive_message(_EMPTY_FLAGS),
+          ]), client_receive_first_message_tag))
+      client_due.add(client_receive_first_message_tag)
+    client_receive_first_message_event = client_driver.event_with_tag(
+        client_receive_first_message_tag)
+
+    client_call_cancel_result = client_call.cancel()
+    client_driver.events()
+
+    server.shutdown(server_completion_queue, server_shutdown_tag)
+    server.cancel_all_calls()
+    server_driver.events()
+
+    self.assertEqual(cygrpc.CallError.ok, request_call_result)
+    self.assertEqual(
+        cygrpc.CallError.ok, server_send_initial_metadata_start_batch_result)
+    self.assertEqual(
+        cygrpc.CallError.ok, client_receive_initial_metadata_start_batch_result)
+    self.assertEqual(
+        cygrpc.CallError.ok, client_complete_rpc_start_batch_result)
+    self.assertEqual(cygrpc.CallError.ok, client_call_cancel_result)
+    self.assertIs(server_rpc_tag, server_rpc_event.tag)
+    self.assertEqual(
+        cygrpc.CompletionType.operation_complete, server_rpc_event.type)
+    self.assertIsInstance(server_rpc_event.operation_call, cygrpc.Call)
+    self.assertEqual(0, len(server_rpc_event.batch_operations))
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio/tests/unit/_cython/cygrpc_test.py
index 876da88de9..b740695e35 100644
--- a/src/python/grpcio/tests/unit/_cython/cygrpc_test.py
+++ b/src/python/grpcio/tests/unit/_cython/cygrpc_test.py
@@ -37,46 +37,47 @@ from tests.unit import test_common
 from tests.unit import resources
 
 
-_SSL_HOST_OVERRIDE = 'foo.test.google.fr'
+_SSL_HOST_OVERRIDE = b'foo.test.google.fr'
 _CALL_CREDENTIALS_METADATA_KEY = 'call-creds-key'
 _CALL_CREDENTIALS_METADATA_VALUE = 'call-creds-value'
+_EMPTY_FLAGS = 0
 
 def _metadata_plugin_callback(context, callback):
   callback(cygrpc.Metadata(
       [cygrpc.Metadatum(_CALL_CREDENTIALS_METADATA_KEY,
                         _CALL_CREDENTIALS_METADATA_VALUE)]),
-      cygrpc.StatusCode.ok, '')
+      cygrpc.StatusCode.ok, b'')
 
 
 class TypeSmokeTest(unittest.TestCase):
 
   def testStringsInUtilitiesUpDown(self):
     self.assertEqual(0, cygrpc.StatusCode.ok)
-    metadatum = cygrpc.Metadatum('a', 'b')
-    self.assertEqual('a'.encode(), metadatum.key)
-    self.assertEqual('b'.encode(), metadatum.value)
+    metadatum = cygrpc.Metadatum(b'a', b'b')
+    self.assertEqual(b'a', metadatum.key)
+    self.assertEqual(b'b', metadatum.value)
     metadata = cygrpc.Metadata([metadatum])
     self.assertEqual(1, len(metadata))
     self.assertEqual(metadatum.key, metadata[0].key)
 
   def testMetadataIteration(self):
     metadata = cygrpc.Metadata([
-        cygrpc.Metadatum('a', 'b'), cygrpc.Metadatum('c', 'd')])
+        cygrpc.Metadatum(b'a', b'b'), cygrpc.Metadatum(b'c', b'd')])
     iterator = iter(metadata)
     metadatum = next(iterator)
     self.assertIsInstance(metadatum, cygrpc.Metadatum)
-    self.assertEqual(metadatum.key, 'a'.encode())
-    self.assertEqual(metadatum.value, 'b'.encode())
+    self.assertEqual(metadatum.key, b'a')
+    self.assertEqual(metadatum.value, b'b')
     metadatum = next(iterator)
     self.assertIsInstance(metadatum, cygrpc.Metadatum)
-    self.assertEqual(metadatum.key, 'c'.encode())
-    self.assertEqual(metadatum.value, 'd'.encode())
+    self.assertEqual(metadatum.key, b'c')
+    self.assertEqual(metadatum.value, b'd')
     with self.assertRaises(StopIteration):
       next(iterator)
 
   def testOperationsIteration(self):
     operations = cygrpc.Operations([
-        cygrpc.operation_send_message('asdf')])
+        cygrpc.operation_send_message(b'asdf', _EMPTY_FLAGS)])
     iterator = iter(operations)
     operation = next(iterator)
     self.assertIsInstance(operation, cygrpc.Operation)
@@ -85,6 +86,11 @@ class TypeSmokeTest(unittest.TestCase):
     with self.assertRaises(StopIteration):
       next(iterator)
 
+  def testOperationFlags(self):
+    operation = cygrpc.operation_send_message(b'asdf',
+                                              cygrpc.WriteFlag.no_compress)
+    self.assertEqual(cygrpc.WriteFlag.no_compress, operation.flags)
+
   def testTimespec(self):
     now = time.time()
     timespec = cygrpc.Timespec(now)
@@ -99,16 +105,16 @@ class TypeSmokeTest(unittest.TestCase):
     del server
 
   def testChannelUpDown(self):
-    channel = cygrpc.Channel('[::]:0', cygrpc.ChannelArgs([]))
+    channel = cygrpc.Channel(b'[::]:0', cygrpc.ChannelArgs([]))
     del channel
 
   def testCredentialsMetadataPluginUpDown(self):
     plugin = cygrpc.CredentialsMetadataPlugin(
-        lambda ignored_a, ignored_b: None, '')
+        lambda ignored_a, ignored_b: None, b'')
     del plugin
 
   def testCallCredentialsFromPluginUpDown(self):
-    plugin = cygrpc.CredentialsMetadataPlugin(_metadata_plugin_callback, '')
+    plugin = cygrpc.CredentialsMetadataPlugin(_metadata_plugin_callback, b'')
     call_credentials = cygrpc.call_credentials_metadata_plugin(plugin)
     del plugin
     del call_credentials
@@ -117,7 +123,7 @@ class TypeSmokeTest(unittest.TestCase):
     server = cygrpc.Server()
     completion_queue = cygrpc.CompletionQueue()
     server.register_completion_queue(completion_queue)
-    port = server.add_http2_port('[::]:0')
+    port = server.add_http2_port(b'[::]:0')
     self.assertIsInstance(port, int)
     server.start()
     del server
@@ -125,7 +131,7 @@ class TypeSmokeTest(unittest.TestCase):
   def testServerStartShutdown(self):
     completion_queue = cygrpc.CompletionQueue()
     server = cygrpc.Server()
-    server.add_http2_port('[::]:0')
+    server.add_http2_port(b'[::]:0')
     server.register_completion_queue(completion_queue)
     server.start()
     shutdown_tag = object()
@@ -137,22 +143,60 @@ class TypeSmokeTest(unittest.TestCase):
     del completion_queue
 
 
-class InsecureServerInsecureClient(unittest.TestCase):
+class ServerClientMixin(object):
 
-  def setUp(self):
+  def setUpMixin(self, server_credentials, client_credentials, host_override):
     self.server_completion_queue = cygrpc.CompletionQueue()
     self.server = cygrpc.Server()
     self.server.register_completion_queue(self.server_completion_queue)
-    self.port = self.server.add_http2_port('[::]:0')
+    if server_credentials:
+      self.port = self.server.add_http2_port(b'[::]:0', server_credentials)
+    else:
+      self.port = self.server.add_http2_port(b'[::]:0')
     self.server.start()
     self.client_completion_queue = cygrpc.CompletionQueue()
-    self.client_channel = cygrpc.Channel('localhost:{}'.format(self.port))
-
-  def tearDown(self):
+    if client_credentials:
+      client_channel_arguments = cygrpc.ChannelArgs([
+          cygrpc.ChannelArg(cygrpc.ChannelArgKey.ssl_target_name_override,
+                            host_override)])
+      self.client_channel = cygrpc.Channel(
+          'localhost:{}'.format(self.port).encode(), client_channel_arguments,
+          client_credentials)
+    else:
+      self.client_channel = cygrpc.Channel('localhost:{}'.format(self.port).encode())
+    if host_override:
+      self.host_argument = None  # default host
+      self.expected_host = host_override
+    else:
+      # arbitrary host name necessitating no further identification
+      self.host_argument = b'hostess'
+      self.expected_host = self.host_argument
+
+  def tearDownMixin(self):
     del self.server
     del self.client_completion_queue
     del self.server_completion_queue
 
+  def _perform_operations(self, operations, call, queue, deadline, description):
+    """Perform the list of operations with given call, queue, and deadline.
+
+    Invocation errors are reported with as an exception with `description` in
+    the message. Performs the operations asynchronously, returning a future.
+    """
+    def performer():
+      tag = object()
+      try:
+        call_result = call.start_batch(cygrpc.Operations(operations), tag)
+        self.assertEqual(cygrpc.CallError.ok, call_result)
+        event = queue.poll(deadline)
+        self.assertEqual(cygrpc.CompletionType.operation_complete, event.type)
+        self.assertTrue(event.success)
+        self.assertIs(tag, event.tag)
+      except Exception as error:
+        raise Exception("Error in '{}': {}".format(description, error.message))
+      return event
+    return test_utilities.SimpleFuture(performer)
+
   def testEcho(self):
     DEADLINE = time.time()+5
     DEADLINE_TOLERANCE = 0.25
@@ -169,7 +213,6 @@ class InsecureServerInsecureClient(unittest.TestCase):
     REQUEST = b'in death a member of project mayhem has a name'
     RESPONSE = b'his name is robert paulson'
     METHOD = b'twinkies'
-    HOST = b'hostess'
 
     cygrpc_deadline = cygrpc.Timespec(DEADLINE)
 
@@ -182,18 +225,20 @@ class InsecureServerInsecureClient(unittest.TestCase):
 
     client_call_tag = object()
     client_call = self.client_channel.create_call(
-        None, 0, self.client_completion_queue, METHOD, HOST, cygrpc_deadline)
+        None, 0, self.client_completion_queue, METHOD, self.host_argument,
+        cygrpc_deadline)
     client_initial_metadata = cygrpc.Metadata([
         cygrpc.Metadatum(CLIENT_METADATA_ASCII_KEY,
                          CLIENT_METADATA_ASCII_VALUE),
         cygrpc.Metadatum(CLIENT_METADATA_BIN_KEY, CLIENT_METADATA_BIN_VALUE)])
     client_start_batch_result = client_call.start_batch(cygrpc.Operations([
-        cygrpc.operation_send_initial_metadata(client_initial_metadata),
-        cygrpc.operation_send_message(REQUEST),
-        cygrpc.operation_send_close_from_client(),
-        cygrpc.operation_receive_initial_metadata(),
-        cygrpc.operation_receive_message(),
-        cygrpc.operation_receive_status_on_client()
+        cygrpc.operation_send_initial_metadata(client_initial_metadata,
+                                               _EMPTY_FLAGS),
+        cygrpc.operation_send_message(REQUEST, _EMPTY_FLAGS),
+        cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+        cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+        cygrpc.operation_receive_message(_EMPTY_FLAGS),
+        cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS)
     ]), client_call_tag)
     self.assertEqual(cygrpc.CallError.ok, client_start_batch_result)
     client_event_future = test_utilities.CompletionQueuePollFuture(
@@ -209,7 +254,8 @@ class InsecureServerInsecureClient(unittest.TestCase):
         test_common.metadata_transmitted(client_initial_metadata,
                                          request_event.request_metadata))
     self.assertEqual(METHOD, request_event.request_call_details.method)
-    self.assertEqual(HOST, request_event.request_call_details.host)
+    self.assertEqual(self.expected_host,
+                     request_event.request_call_details.host)
     self.assertLess(
         abs(DEADLINE - float(request_event.request_call_details.deadline)),
         DEADLINE_TOLERANCE)
@@ -223,12 +269,14 @@ class InsecureServerInsecureClient(unittest.TestCase):
         cygrpc.Metadatum(SERVER_TRAILING_METADATA_KEY,
                          SERVER_TRAILING_METADATA_VALUE)])
     server_start_batch_result = server_call.start_batch([
-        cygrpc.operation_send_initial_metadata(server_initial_metadata),
-        cygrpc.operation_receive_message(),
-        cygrpc.operation_send_message(RESPONSE),
-        cygrpc.operation_receive_close_on_server(),
+        cygrpc.operation_send_initial_metadata(server_initial_metadata,
+                                               _EMPTY_FLAGS),
+        cygrpc.operation_receive_message(_EMPTY_FLAGS),
+        cygrpc.operation_send_message(RESPONSE, _EMPTY_FLAGS),
+        cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),
         cygrpc.operation_send_status_from_server(
-            server_trailing_metadata, SERVER_STATUS_CODE, SERVER_STATUS_DETAILS)
+            server_trailing_metadata, SERVER_STATUS_CODE,
+            SERVER_STATUS_DETAILS, _EMPTY_FLAGS)
     ], server_call_tag)
     self.assertEqual(cygrpc.CallError.ok, server_start_batch_result)
 
@@ -283,169 +331,101 @@ class InsecureServerInsecureClient(unittest.TestCase):
     del client_call
     del server_call
 
-
-class SecureServerSecureClient(unittest.TestCase):
-
-  def setUp(self):
-    server_credentials = cygrpc.server_credentials_ssl(
-        None, [cygrpc.SslPemKeyCertPair(resources.private_key(),
-                                        resources.certificate_chain())], False)
-    channel_credentials = cygrpc.channel_credentials_ssl(
-        resources.test_root_certificates(), None)
-    self.server_completion_queue = cygrpc.CompletionQueue()
-    self.server = cygrpc.Server()
-    self.server.register_completion_queue(self.server_completion_queue)
-    self.port = self.server.add_http2_port('[::]:0', server_credentials)
-    self.server.start()
-    self.client_completion_queue = cygrpc.CompletionQueue()
-    client_channel_arguments = cygrpc.ChannelArgs([
-        cygrpc.ChannelArg(cygrpc.ChannelArgKey.ssl_target_name_override,
-                          _SSL_HOST_OVERRIDE)])
-    self.client_channel = cygrpc.Channel(
-        'localhost:{}'.format(self.port), client_channel_arguments,
-        channel_credentials)
-
-  def tearDown(self):
-    del self.server
-    del self.client_completion_queue
-    del self.server_completion_queue
-
-  def testEcho(self):
+  def test6522(self):
     DEADLINE = time.time()+5
     DEADLINE_TOLERANCE = 0.25
-    CLIENT_METADATA_ASCII_KEY = b'key'
-    CLIENT_METADATA_ASCII_VALUE = b'val'
-    CLIENT_METADATA_BIN_KEY = b'key-bin'
-    CLIENT_METADATA_BIN_VALUE = b'\0'*1000
-    SERVER_INITIAL_METADATA_KEY = b'init_me_me_me'
-    SERVER_INITIAL_METADATA_VALUE = b'whodawha?'
-    SERVER_TRAILING_METADATA_KEY = b'california_is_in_a_drought'
-    SERVER_TRAILING_METADATA_VALUE = b'zomg it is'
-    SERVER_STATUS_CODE = cygrpc.StatusCode.ok
-    SERVER_STATUS_DETAILS = b'our work is never over'
-    REQUEST = b'in death a member of project mayhem has a name'
-    RESPONSE = b'his name is robert paulson'
-    METHOD = b'/twinkies'
-    HOST = None  # Default host
+    METHOD = b'twinkies'
 
     cygrpc_deadline = cygrpc.Timespec(DEADLINE)
+    empty_metadata = cygrpc.Metadata([])
 
     server_request_tag = object()
-    request_call_result = self.server.request_call(
+    self.server.request_call(
         self.server_completion_queue, self.server_completion_queue,
         server_request_tag)
+    client_call = self.client_channel.create_call(
+        None, 0, self.client_completion_queue, METHOD, self.host_argument,
+        cygrpc_deadline)
 
-    self.assertEqual(cygrpc.CallError.ok, request_call_result)
+    # Prologue
+    def perform_client_operations(operations, description):
+      return self._perform_operations(
+          operations, client_call,
+          self.client_completion_queue, cygrpc_deadline, description)
 
-    plugin = cygrpc.CredentialsMetadataPlugin(_metadata_plugin_callback, '')
-    call_credentials = cygrpc.call_credentials_metadata_plugin(plugin)
-
-    client_call_tag = object()
-    client_call = self.client_channel.create_call(
-        None, 0, self.client_completion_queue, METHOD, HOST, cygrpc_deadline)
-    client_call.set_credentials(call_credentials)
-    client_initial_metadata = cygrpc.Metadata([
-        cygrpc.Metadatum(CLIENT_METADATA_ASCII_KEY,
-                         CLIENT_METADATA_ASCII_VALUE),
-        cygrpc.Metadatum(CLIENT_METADATA_BIN_KEY, CLIENT_METADATA_BIN_VALUE)])
-    client_start_batch_result = client_call.start_batch(cygrpc.Operations([
-        cygrpc.operation_send_initial_metadata(client_initial_metadata),
-        cygrpc.operation_send_message(REQUEST),
-        cygrpc.operation_send_close_from_client(),
-        cygrpc.operation_receive_initial_metadata(),
-        cygrpc.operation_receive_message(),
-        cygrpc.operation_receive_status_on_client()
-    ]), client_call_tag)
-    self.assertEqual(cygrpc.CallError.ok, client_start_batch_result)
-    client_event_future = test_utilities.CompletionQueuePollFuture(
-        self.client_completion_queue, cygrpc_deadline)
+    client_event_future = perform_client_operations([
+            cygrpc.operation_send_initial_metadata(empty_metadata,
+                                                   _EMPTY_FLAGS),
+            cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+        ], "Client prologue")
 
     request_event = self.server_completion_queue.poll(cygrpc_deadline)
-    self.assertEqual(cygrpc.CompletionType.operation_complete,
-                      request_event.type)
-    self.assertIsInstance(request_event.operation_call, cygrpc.Call)
-    self.assertIs(server_request_tag, request_event.tag)
-    self.assertEqual(0, len(request_event.batch_operations))
-    client_metadata_with_credentials = list(client_initial_metadata) + [
-        (_CALL_CREDENTIALS_METADATA_KEY, _CALL_CREDENTIALS_METADATA_VALUE)]
-    self.assertTrue(
-        test_common.metadata_transmitted(client_metadata_with_credentials,
-                                         request_event.request_metadata))
-    self.assertEqual(METHOD, request_event.request_call_details.method)
-    self.assertEqual(_SSL_HOST_OVERRIDE,
-                     request_event.request_call_details.host)
-    self.assertLess(
-        abs(DEADLINE - float(request_event.request_call_details.deadline)),
-        DEADLINE_TOLERANCE)
-
-    server_call_tag = object()
     server_call = request_event.operation_call
-    server_initial_metadata = cygrpc.Metadata([
-        cygrpc.Metadatum(SERVER_INITIAL_METADATA_KEY,
-                         SERVER_INITIAL_METADATA_VALUE)])
-    server_trailing_metadata = cygrpc.Metadata([
-        cygrpc.Metadatum(SERVER_TRAILING_METADATA_KEY,
-                         SERVER_TRAILING_METADATA_VALUE)])
-    server_start_batch_result = server_call.start_batch([
-        cygrpc.operation_send_initial_metadata(server_initial_metadata),
-        cygrpc.operation_receive_message(),
-        cygrpc.operation_send_message(RESPONSE),
-        cygrpc.operation_receive_close_on_server(),
-        cygrpc.operation_send_status_from_server(
-            server_trailing_metadata, SERVER_STATUS_CODE, SERVER_STATUS_DETAILS)
-    ], server_call_tag)
-    self.assertEqual(cygrpc.CallError.ok, server_start_batch_result)
 
-    client_event = client_event_future.result()
-    server_event = self.server_completion_queue.poll(cygrpc_deadline)
+    def perform_server_operations(operations, description):
+      return self._perform_operations(
+          operations, server_call,
+          self.server_completion_queue, cygrpc_deadline, description)
 
-    self.assertEqual(6, len(client_event.batch_operations))
-    found_client_op_types = set()
-    for client_result in client_event.batch_operations:
-      # we expect each op type to be unique
-      self.assertNotIn(client_result.type, found_client_op_types)
-      found_client_op_types.add(client_result.type)
-      if client_result.type == cygrpc.OperationType.receive_initial_metadata:
-        self.assertTrue(
-            test_common.metadata_transmitted(server_initial_metadata,
-                                             client_result.received_metadata))
-      elif client_result.type == cygrpc.OperationType.receive_message:
-        self.assertEqual(RESPONSE, client_result.received_message.bytes())
-      elif client_result.type == cygrpc.OperationType.receive_status_on_client:
-        self.assertTrue(
-            test_common.metadata_transmitted(server_trailing_metadata,
-                                             client_result.received_metadata))
-        self.assertEqual(SERVER_STATUS_DETAILS,
-                         client_result.received_status_details)
-        self.assertEqual(SERVER_STATUS_CODE, client_result.received_status_code)
-    self.assertEqual(set([
-          cygrpc.OperationType.send_initial_metadata,
-          cygrpc.OperationType.send_message,
-          cygrpc.OperationType.send_close_from_client,
-          cygrpc.OperationType.receive_initial_metadata,
-          cygrpc.OperationType.receive_message,
-          cygrpc.OperationType.receive_status_on_client
-      ]), found_client_op_types)
+    server_event_future = perform_server_operations([
+            cygrpc.operation_send_initial_metadata(empty_metadata,
+                                                   _EMPTY_FLAGS),
+        ], "Server prologue")
 
-    self.assertEqual(5, len(server_event.batch_operations))
-    found_server_op_types = set()
-    for server_result in server_event.batch_operations:
-      self.assertNotIn(client_result.type, found_server_op_types)
-      found_server_op_types.add(server_result.type)
-      if server_result.type == cygrpc.OperationType.receive_message:
-        self.assertEqual(REQUEST, server_result.received_message.bytes())
-      elif server_result.type == cygrpc.OperationType.receive_close_on_server:
-        self.assertFalse(server_result.received_cancelled)
-    self.assertEqual(set([
-          cygrpc.OperationType.send_initial_metadata,
-          cygrpc.OperationType.receive_message,
-          cygrpc.OperationType.send_message,
-          cygrpc.OperationType.receive_close_on_server,
-          cygrpc.OperationType.send_status_from_server
-      ]), found_server_op_types)
+    client_event_future.result()  # force completion
+    server_event_future.result()
 
-    del client_call
-    del server_call
+    # Messaging
+    for _ in range(10):
+      client_event_future = perform_client_operations([
+              cygrpc.operation_send_message(b'', _EMPTY_FLAGS),
+              cygrpc.operation_receive_message(_EMPTY_FLAGS),
+          ], "Client message")
+      server_event_future = perform_server_operations([
+              cygrpc.operation_send_message(b'', _EMPTY_FLAGS),
+              cygrpc.operation_receive_message(_EMPTY_FLAGS),
+          ], "Server receive")
+
+      client_event_future.result()  # force completion
+      server_event_future.result()
+
+    # Epilogue
+    client_event_future = perform_client_operations([
+            cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+            cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS)
+        ], "Client epilogue")
+
+    server_event_future = perform_server_operations([
+            cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),
+            cygrpc.operation_send_status_from_server(
+                empty_metadata, cygrpc.StatusCode.ok, b'', _EMPTY_FLAGS)
+        ], "Server epilogue")
+
+    client_event_future.result()  # force completion
+    server_event_future.result()
+
+
+class InsecureServerInsecureClient(unittest.TestCase, ServerClientMixin):
+
+  def setUp(self):
+    self.setUpMixin(None, None, None)
+
+  def tearDown(self):
+    self.tearDownMixin()
+
+
+class SecureServerSecureClient(unittest.TestCase, ServerClientMixin):
+
+  def setUp(self):
+    server_credentials = cygrpc.server_credentials_ssl(
+        None, [cygrpc.SslPemKeyCertPair(resources.private_key(),
+                                        resources.certificate_chain())], False)
+    client_credentials = cygrpc.channel_credentials_ssl(
+        resources.test_root_certificates(), None)
+    self.setUpMixin(server_credentials, client_credentials, _SSL_HOST_OVERRIDE)
+
+  def tearDown(self):
+    self.tearDownMixin()
 
 
 if __name__ == '__main__':
diff --git a/src/python/grpcio/tests/unit/_cython/test_utilities.py b/src/python/grpcio/tests/unit/_cython/test_utilities.py
index 6a09739643..6280ce74c4 100644
--- a/src/python/grpcio/tests/unit/_cython/test_utilities.py
+++ b/src/python/grpcio/tests/unit/_cython/test_utilities.py
@@ -32,15 +32,35 @@ import threading
 from grpc._cython import cygrpc
 
 
-class CompletionQueuePollFuture:
+class SimpleFuture(object):
+  """A simple future mechanism."""
 
-  def __init__(self, completion_queue, deadline):
-    def poller_function():
-      self._event_result = completion_queue.poll(deadline)
-    self._event_result = None
-    self._thread = threading.Thread(target=poller_function)
+  def __init__(self, function, *args, **kwargs):
+    def wrapped_function():
+      try:
+        self._result = function(*args, **kwargs)
+      except Exception as error:
+        self._error = error
+    self._result = None
+    self._error = None
+    self._thread = threading.Thread(target=wrapped_function)
     self._thread.start()
 
   def result(self):
+    """The resulting value of this future.
+
+    Re-raises any exceptions.
+    """
     self._thread.join()
-    return self._event_result
+    if self._error:
+      # TODO(atash): re-raise exceptions in a way that preserves tracebacks
+      raise self._error
+    return self._result
+
+
+class CompletionQueuePollFuture(SimpleFuture):
+
+  def __init__(self, completion_queue, deadline):
+    super(CompletionQueuePollFuture, self).__init__(
+        lambda: completion_queue.poll(deadline))
+
diff --git a/src/python/grpcio/tests/unit/_empty_message_test.py b/src/python/grpcio/tests/unit/_empty_message_test.py
new file mode 100644
index 0000000000..8c7d697728
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_empty_message_test.py
@@ -0,0 +1,137 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+import grpc
+from grpc.framework.foundation import logging_pool
+
+from tests.unit.framework.common import test_constants
+
+_REQUEST = b''
+_RESPONSE = b''
+
+_UNARY_UNARY = '/test/UnaryUnary'
+_UNARY_STREAM = '/test/UnaryStream'
+_STREAM_UNARY = '/test/StreamUnary'
+_STREAM_STREAM = '/test/StreamStream'
+
+
+def handle_unary_unary(request, servicer_context):
+  return _RESPONSE
+
+
+def handle_unary_stream(request, servicer_context):
+  for _ in range(test_constants.STREAM_LENGTH):
+    yield _RESPONSE
+
+
+def handle_stream_unary(request_iterator, servicer_context):
+  for request in request_iterator:
+    pass
+  return _RESPONSE
+
+
+def handle_stream_stream(request_iterator, servicer_context):
+  for request in request_iterator:
+    yield _RESPONSE
+
+
+class _MethodHandler(grpc.RpcMethodHandler):
+
+  def __init__(self, request_streaming, response_streaming):
+    self.request_streaming = request_streaming
+    self.response_streaming = response_streaming
+    self.request_deserializer = None
+    self.response_serializer = None
+    self.unary_unary = None
+    self.unary_stream = None
+    self.stream_unary = None
+    self.stream_stream = None
+    if self.request_streaming and self.response_streaming:
+      self.stream_stream = handle_stream_stream
+    elif self.request_streaming:
+      self.stream_unary = handle_stream_unary
+    elif self.response_streaming:
+      self.unary_stream = handle_unary_stream
+    else:
+      self.unary_unary = handle_unary_unary
+
+
+class _GenericHandler(grpc.GenericRpcHandler):
+
+  def service(self, handler_call_details):
+    if handler_call_details.method == _UNARY_UNARY:
+      return _MethodHandler(False, False)
+    elif handler_call_details.method == _UNARY_STREAM:
+      return _MethodHandler(False, True)
+    elif handler_call_details.method == _STREAM_UNARY:
+      return _MethodHandler(True, False)
+    elif handler_call_details.method == _STREAM_STREAM:
+      return _MethodHandler(True, True)
+    else:
+      return None
+
+
+class EmptyMessageTest(unittest.TestCase):
+
+  def setUp(self):
+    self._server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+    self._server = grpc.server((_GenericHandler(),), self._server_pool)
+    port = self._server.add_insecure_port('[::]:0')
+    self._server.start()
+    self._channel = grpc.insecure_channel('localhost:%d' % port)
+
+  def tearDown(self):
+    self._server.stop(0)
+
+  def testUnaryUnary(self):
+    response = self._channel.unary_unary(_UNARY_UNARY)(_REQUEST)
+    self.assertEqual(_RESPONSE, response)
+
+  def testUnaryStream(self):
+    response_iterator = self._channel.unary_stream(_UNARY_STREAM)(_REQUEST)
+    self.assertSequenceEqual(
+        [_RESPONSE] * test_constants.STREAM_LENGTH, list(response_iterator))
+
+  def testStreamUnary(self):
+    response = self._channel.stream_unary(_STREAM_UNARY)(
+        [_REQUEST] * test_constants.STREAM_LENGTH)
+    self.assertEqual(_RESPONSE, response)
+
+  def testStreamStream(self):
+    response_iterator = self._channel.stream_stream(_STREAM_STREAM)(
+        [_REQUEST] * test_constants.STREAM_LENGTH)
+    self.assertSequenceEqual(
+        [_RESPONSE] * test_constants.STREAM_LENGTH, list(response_iterator))
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
+
diff --git a/src/python/grpcio/tests/unit/_exit_scenarios.py b/src/python/grpcio/tests/unit/_exit_scenarios.py
new file mode 100644
index 0000000000..24a2faef85
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_exit_scenarios.py
@@ -0,0 +1,249 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Defines a number of module-scope gRPC scenarios to test clean exit."""
+
+import argparse
+import threading
+import time
+
+import grpc
+
+from tests.unit.framework.common import test_constants
+
+WAIT_TIME = 1000
+
+REQUEST = b'request'
+
+UNSTARTED_SERVER = 'unstarted_server'
+RUNNING_SERVER = 'running_server'
+POLL_CONNECTIVITY_NO_SERVER = 'poll_connectivity_no_server'
+POLL_CONNECTIVITY = 'poll_connectivity'
+IN_FLIGHT_UNARY_UNARY_CALL = 'in_flight_unary_unary_call'
+IN_FLIGHT_UNARY_STREAM_CALL = 'in_flight_unary_stream_call'
+IN_FLIGHT_STREAM_UNARY_CALL = 'in_flight_stream_unary_call'
+IN_FLIGHT_STREAM_STREAM_CALL = 'in_flight_stream_stream_call'
+IN_FLIGHT_PARTIAL_UNARY_STREAM_CALL = 'in_flight_partial_unary_stream_call'
+IN_FLIGHT_PARTIAL_STREAM_UNARY_CALL = 'in_flight_partial_stream_unary_call'
+IN_FLIGHT_PARTIAL_STREAM_STREAM_CALL = 'in_flight_partial_stream_stream_call'
+
+UNARY_UNARY = b'/test/UnaryUnary'
+UNARY_STREAM = b'/test/UnaryStream'
+STREAM_UNARY = b'/test/StreamUnary'
+STREAM_STREAM = b'/test/StreamStream'
+PARTIAL_UNARY_STREAM = b'/test/PartialUnaryStream'
+PARTIAL_STREAM_UNARY = b'/test/PartialStreamUnary'
+PARTIAL_STREAM_STREAM = b'/test/PartialStreamStream'
+
+TEST_TO_METHOD = {
+    IN_FLIGHT_UNARY_UNARY_CALL: UNARY_UNARY,
+    IN_FLIGHT_UNARY_STREAM_CALL: UNARY_STREAM,
+    IN_FLIGHT_STREAM_UNARY_CALL: STREAM_UNARY,
+    IN_FLIGHT_STREAM_STREAM_CALL: STREAM_STREAM,
+    IN_FLIGHT_PARTIAL_UNARY_STREAM_CALL: PARTIAL_UNARY_STREAM,
+    IN_FLIGHT_PARTIAL_STREAM_UNARY_CALL: PARTIAL_STREAM_UNARY,
+    IN_FLIGHT_PARTIAL_STREAM_STREAM_CALL: PARTIAL_STREAM_STREAM,
+}
+
+
+def hang_unary_unary(request, servicer_context):
+  time.sleep(WAIT_TIME)
+
+
+def hang_unary_stream(request, servicer_context):
+  time.sleep(WAIT_TIME)
+
+
+def hang_partial_unary_stream(request, servicer_context):
+  for _ in range(test_constants.STREAM_LENGTH // 2):
+    yield request
+  time.sleep(WAIT_TIME)
+
+
+def hang_stream_unary(request_iterator, servicer_context):
+  time.sleep(WAIT_TIME)
+
+
+def hang_partial_stream_unary(request_iterator, servicer_context):
+  for _ in range(test_constants.STREAM_LENGTH // 2):
+    next(request_iterator)
+  time.sleep(WAIT_TIME)
+
+
+def hang_stream_stream(request_iterator, servicer_context):
+  time.sleep(WAIT_TIME)
+
+
+def hang_partial_stream_stream(request_iterator, servicer_context):
+  for _ in range(test_constants.STREAM_LENGTH // 2):
+    yield next(request_iterator)
+  time.sleep(WAIT_TIME)
+
+
+class MethodHandler(grpc.RpcMethodHandler):
+
+  def __init__(self, request_streaming, response_streaming, partial_hang):
+    self.request_streaming = request_streaming
+    self.response_streaming = response_streaming
+    self.request_deserializer = None
+    self.response_serializer = None
+    self.unary_unary = None
+    self.unary_stream = None
+    self.stream_unary = None
+    self.stream_stream = None
+    if self.request_streaming and self.response_streaming:
+      if partial_hang:
+        self.stream_stream = hang_partial_stream_stream
+      else:
+        self.stream_stream = hang_stream_stream
+    elif self.request_streaming:
+      if partial_hang:
+        self.stream_unary = hang_partial_stream_unary
+      else:
+        self.stream_unary = hang_stream_unary
+    elif self.response_streaming:
+      if partial_hang:
+        self.unary_stream = hang_partial_unary_stream
+      else:
+        self.unary_stream = hang_unary_stream
+    else:
+      self.unary_unary = hang_unary_unary
+
+
+class GenericHandler(grpc.GenericRpcHandler):
+
+  def service(self, handler_call_details):
+    if handler_call_details.method == UNARY_UNARY:
+      return MethodHandler(False, False, False)
+    elif handler_call_details.method == UNARY_STREAM:
+      return MethodHandler(False, True, False)
+    elif handler_call_details.method == STREAM_UNARY:
+      return MethodHandler(True, False, False)
+    elif handler_call_details.method == STREAM_STREAM:
+      return MethodHandler(True, True, False)
+    elif handler_call_details.method == PARTIAL_UNARY_STREAM:
+      return MethodHandler(False, True, True)
+    elif handler_call_details.method == PARTIAL_STREAM_UNARY:
+      return MethodHandler(True, False, True)
+    elif handler_call_details.method == PARTIAL_STREAM_STREAM:
+      return MethodHandler(True, True, True)
+    else:
+      return None
+
+
+# Traditional executors will not exit until all their
+# current jobs complete.  Because we submit jobs that will
+# never finish, we don't want to block exit on these jobs.
+class DaemonPool(object):
+
+  def submit(self, fn, *args, **kwargs):
+    thread = threading.Thread(target=fn, args=args, kwargs=kwargs)
+    thread.daemon = True
+    thread.start()
+
+  def shutdown(self, wait=True):
+    pass
+
+
+def infinite_request_iterator():
+  while True:
+    yield REQUEST
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('scenario', type=str)
+  parser.add_argument(
+      '--wait_for_interrupt', dest='wait_for_interrupt', action='store_true')
+  args = parser.parse_args()
+
+  if args.scenario == UNSTARTED_SERVER:
+    server = grpc.server((), DaemonPool())
+    if args.wait_for_interrupt:
+      time.sleep(WAIT_TIME)
+  elif args.scenario == RUNNING_SERVER:
+    server = grpc.server((), DaemonPool())
+    port = server.add_insecure_port('[::]:0')
+    server.start()
+    if args.wait_for_interrupt:
+      time.sleep(WAIT_TIME)
+  elif args.scenario == POLL_CONNECTIVITY_NO_SERVER:
+    channel = grpc.insecure_channel('localhost:12345')
+
+    def connectivity_callback(connectivity):
+      pass
+
+    channel.subscribe(connectivity_callback, try_to_connect=True)
+    if args.wait_for_interrupt:
+      time.sleep(WAIT_TIME)
+  elif args.scenario == POLL_CONNECTIVITY:
+    server = grpc.server((), DaemonPool())
+    port = server.add_insecure_port('[::]:0')
+    server.start()
+    channel = grpc.insecure_channel('localhost:%d' % port)
+
+    def connectivity_callback(connectivity):
+      pass
+
+    channel.subscribe(connectivity_callback, try_to_connect=True)
+    if args.wait_for_interrupt:
+      time.sleep(WAIT_TIME)
+
+  else:
+    handler = GenericHandler()
+    server = grpc.server((), DaemonPool())
+    port = server.add_insecure_port('[::]:0')
+    server.add_generic_rpc_handlers((handler,))
+    server.start()
+    channel = grpc.insecure_channel('localhost:%d' % port)
+
+    method = TEST_TO_METHOD[args.scenario]
+
+    if args.scenario == IN_FLIGHT_UNARY_UNARY_CALL:
+      multi_callable = channel.unary_unary(method)
+      future = multi_callable.future(REQUEST)
+      result, call = multi_callable.with_call(REQUEST)
+    elif (args.scenario == IN_FLIGHT_UNARY_STREAM_CALL or
+          args.scenario == IN_FLIGHT_PARTIAL_UNARY_STREAM_CALL):
+      multi_callable = channel.unary_stream(method)
+      response_iterator = multi_callable(REQUEST)
+      for response in response_iterator:
+        pass
+    elif (args.scenario == IN_FLIGHT_STREAM_UNARY_CALL or
+          args.scenario == IN_FLIGHT_PARTIAL_STREAM_UNARY_CALL):
+      multi_callable = channel.stream_unary(method)
+      future = multi_callable.future(infinite_request_iterator())
+      result, call = multi_callable.with_call(
+          [REQUEST] * test_constants.STREAM_LENGTH)
+    elif (args.scenario == IN_FLIGHT_STREAM_STREAM_CALL or
+          args.scenario == IN_FLIGHT_PARTIAL_STREAM_STREAM_CALL):
+      multi_callable = channel.stream_stream(method)
+      response_iterator = multi_callable(infinite_request_iterator())
+      for response in response_iterator:
+        pass
diff --git a/src/python/grpcio/tests/unit/_exit_test.py b/src/python/grpcio/tests/unit/_exit_test.py
new file mode 100644
index 0000000000..b0d6af73e5
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_exit_test.py
@@ -0,0 +1,185 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests clean exit of server/client on Python Interpreter exit/sigint.
+
+The tests in this module spawn a subprocess for each test case, the
+test is considered successful if it doesn't hang/timeout.
+"""
+
+import atexit
+import os
+import signal
+import six
+import subprocess
+import sys
+import threading
+import time
+import unittest
+
+from tests.unit import _exit_scenarios
+
+SCENARIO_FILE = os.path.abspath(os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), '_exit_scenarios.py'))
+INTERPRETER = sys.executable
+BASE_COMMAND = [INTERPRETER, SCENARIO_FILE]
+BASE_SIGTERM_COMMAND = BASE_COMMAND + ['--wait_for_interrupt']
+
+INIT_TIME = 1.0
+
+
+processes = []
+process_lock = threading.Lock()
+
+
+# Make sure we attempt to clean up any
+# processes we may have left running
+def cleanup_processes():
+  with process_lock:
+    for process in processes:
+      try:
+        process.kill()
+      except Exception:
+        pass
+atexit.register(cleanup_processes)
+
+
+def interrupt_and_wait(process):
+  with process_lock:
+    processes.append(process)
+  time.sleep(INIT_TIME)
+  os.kill(process.pid, signal.SIGINT)
+  process.wait()
+
+
+def wait(process):
+  with process_lock:
+    processes.append(process)
+  process.wait()
+
+
+class ExitTest(unittest.TestCase):
+
+  def test_unstarted_server(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.UNSTARTED_SERVER],
+        stdout=sys.stdout, stderr=sys.stderr)
+    wait(process)
+
+  def test_unstarted_server_terminate(self):
+    process = subprocess.Popen(
+        BASE_SIGTERM_COMMAND + [_exit_scenarios.UNSTARTED_SERVER],
+        stdout=sys.stdout)
+    interrupt_and_wait(process)
+
+  def test_running_server(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.RUNNING_SERVER],
+        stdout=sys.stdout, stderr=sys.stderr)
+    wait(process)
+
+  def test_running_server_terminate(self):
+    process = subprocess.Popen(
+        BASE_SIGTERM_COMMAND + [_exit_scenarios.RUNNING_SERVER],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  def test_poll_connectivity_no_server(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.POLL_CONNECTIVITY_NO_SERVER],
+        stdout=sys.stdout, stderr=sys.stderr)
+    wait(process)
+
+  def test_poll_connectivity_no_server_terminate(self):
+    process = subprocess.Popen(
+        BASE_SIGTERM_COMMAND + [_exit_scenarios.POLL_CONNECTIVITY_NO_SERVER],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  def test_poll_connectivity(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.POLL_CONNECTIVITY],
+        stdout=sys.stdout, stderr=sys.stderr)
+    wait(process)
+
+  def test_poll_connectivity_terminate(self):
+    process = subprocess.Popen(
+        BASE_SIGTERM_COMMAND + [_exit_scenarios.POLL_CONNECTIVITY],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  def test_in_flight_unary_unary_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_UNARY_UNARY_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999')
+  def test_in_flight_unary_stream_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_UNARY_STREAM_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  def test_in_flight_stream_unary_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_STREAM_UNARY_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999')
+  def test_in_flight_stream_stream_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_STREAM_STREAM_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999')
+  def test_in_flight_partial_unary_stream_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_PARTIAL_UNARY_STREAM_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  def test_in_flight_partial_stream_unary_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_PARTIAL_STREAM_UNARY_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+  @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999')
+  def test_in_flight_partial_stream_stream_call(self):
+    process = subprocess.Popen(
+        BASE_COMMAND + [_exit_scenarios.IN_FLIGHT_PARTIAL_STREAM_STREAM_CALL],
+        stdout=sys.stdout, stderr=sys.stderr)
+    interrupt_and_wait(process)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_from_grpc_import_star.py b/src/python/grpcio/tests/unit/_from_grpc_import_star.py
new file mode 100644
index 0000000000..78d2fb7dc5
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_from_grpc_import_star.py
@@ -0,0 +1,38 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+_BEFORE_IMPORT = tuple(globals())
+
+from grpc import *
+
+_AFTER_IMPORT = tuple(globals())
+
+GRPC_ELEMENTS = tuple(
+    element for element in _AFTER_IMPORT
+    if element not in _BEFORE_IMPORT and element != '_BEFORE_IMPORT')
diff --git a/src/python/grpcio/tests/unit/_links/_lonely_invocation_link_test.py b/src/python/grpcio/tests/unit/_links/_lonely_invocation_link_test.py
deleted file mode 100644
index 890755f81c..0000000000
--- a/src/python/grpcio/tests/unit/_links/_lonely_invocation_link_test.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""A test of invocation-side code unconnected to an RPC server."""
-
-import unittest
-
-from grpc._adapter import _intermediary_low
-from grpc._links import invocation
-from grpc.framework.interfaces.links import links
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.interfaces.links import test_cases
-from tests.unit.framework.interfaces.links import test_utilities
-
-_NULL_BEHAVIOR = lambda unused_argument: None
-
-
-class LonelyInvocationLinkTest(unittest.TestCase):
-
-  def testUpAndDown(self):
-    channel = _intermediary_low.Channel('nonexistent:54321', None)
-    invocation_link = invocation.invocation_link(
-        channel, 'nonexistent', None, {}, {})
-
-    invocation_link.start()
-    invocation_link.stop()
-
-  def _test_lonely_invocation_with_termination(self, termination):
-    test_operation_id = object()
-    test_group = 'test package.Test Service'
-    test_method = 'test method'
-    invocation_link_mate = test_utilities.RecordingLink()
-
-    channel = _intermediary_low.Channel('nonexistent:54321', None)
-    invocation_link = invocation.invocation_link(
-        channel, 'nonexistent', None, {}, {})
-    invocation_link.join_link(invocation_link_mate)
-    invocation_link.start()
-
-    ticket = links.Ticket(
-        test_operation_id, 0, test_group, test_method,
-        links.Ticket.Subscription.FULL, test_constants.SHORT_TIMEOUT, 1, None,
-        None, None, None, None, termination, None)
-    invocation_link.accept_ticket(ticket)
-    invocation_link_mate.block_until_tickets_satisfy(test_cases.terminated)
-
-    invocation_link.stop()
-
-    self.assertIsNot(
-        invocation_link_mate.tickets()[-1].termination,
-        links.Ticket.Termination.COMPLETION)
-
-  def testLonelyInvocationLinkWithCommencementTicket(self):
-    self._test_lonely_invocation_with_termination(None)
-
-  def testLonelyInvocationLinkWithEntireTicket(self):
-    self._test_lonely_invocation_with_termination(
-        links.Ticket.Termination.COMPLETION)
-
-
-if __name__ == '__main__':
-  unittest.main()
diff --git a/src/python/grpcio/tests/unit/_links/_transmission_test.py b/src/python/grpcio/tests/unit/_links/_transmission_test.py
deleted file mode 100644
index 888684d197..0000000000
--- a/src/python/grpcio/tests/unit/_links/_transmission_test.py
+++ /dev/null
@@ -1,239 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests transmission of tickets across gRPC-on-the-wire."""
-
-import unittest
-
-from grpc._adapter import _intermediary_low
-from grpc._links import invocation
-from grpc._links import service
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.interfaces.links import links
-from tests.unit import test_common
-from tests.unit._links import _proto_scenarios
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.interfaces.links import test_cases
-from tests.unit.framework.interfaces.links import test_utilities
-
-_IDENTITY = lambda x: x
-
-
-class TransmissionTest(test_cases.TransmissionTest, unittest.TestCase):
-
-  def create_transmitting_links(self):
-    service_link = service.service_link(
-        {self.group_and_method(): self.deserialize_request},
-        {self.group_and_method(): self.serialize_response})
-    port = service_link.add_port('[::]:0', None)
-    service_link.start()
-    channel = _intermediary_low.Channel('localhost:%d' % port, None)
-    invocation_link = invocation.invocation_link(
-        channel, 'localhost', None,
-        {self.group_and_method(): self.serialize_request},
-        {self.group_and_method(): self.deserialize_response})
-    invocation_link.start()
-    return invocation_link, service_link
-
-  def destroy_transmitting_links(self, invocation_side_link, service_side_link):
-    invocation_side_link.stop()
-    service_side_link.begin_stop()
-    service_side_link.end_stop()
-
-  def create_invocation_initial_metadata(self):
-    return (
-        ('first_invocation_initial_metadata_key', 'just a string value'),
-        ('second_invocation_initial_metadata_key', '0123456789'),
-        ('third_invocation_initial_metadata_key-bin', '\x00\x57' * 100),
-    )
-
-  def create_invocation_terminal_metadata(self):
-    return None
-
-  def create_service_initial_metadata(self):
-    return (
-        ('first_service_initial_metadata_key', 'just another string value'),
-        ('second_service_initial_metadata_key', '9876543210'),
-        ('third_service_initial_metadata_key-bin', '\x00\x59\x02' * 100),
-    )
-
-  def create_service_terminal_metadata(self):
-    return (
-        ('first_service_terminal_metadata_key', 'yet another string value'),
-        ('second_service_terminal_metadata_key', 'abcdefghij'),
-        ('third_service_terminal_metadata_key-bin', '\x00\x37' * 100),
-    )
-
-  def create_invocation_completion(self):
-    return None, None
-
-  def create_service_completion(self):
-    return (
-        beta_interfaces.StatusCode.OK, b'An exuberant test "details" message!')
-
-  def assertMetadataTransmitted(self, original_metadata, transmitted_metadata):
-    self.assertTrue(
-        test_common.metadata_transmitted(
-            original_metadata, transmitted_metadata),
-        '%s erroneously transmitted as %s' % (
-            original_metadata, transmitted_metadata))
-
-
-class RoundTripTest(unittest.TestCase):
-
-  def testZeroMessageRoundTrip(self):
-    test_operation_id = object()
-    test_group = 'test package.Test Group'
-    test_method = 'test method'
-    identity_transformation = {(test_group, test_method): _IDENTITY}
-    test_code = beta_interfaces.StatusCode.OK
-    test_message = 'a test message'
-
-    service_link = service.service_link(
-        identity_transformation, identity_transformation)
-    service_mate = test_utilities.RecordingLink()
-    service_link.join_link(service_mate)
-    port = service_link.add_port('[::]:0', None)
-    service_link.start()
-    channel = _intermediary_low.Channel('localhost:%d' % port, None)
-    invocation_link = invocation.invocation_link(
-        channel, None, None, identity_transformation, identity_transformation)
-    invocation_mate = test_utilities.RecordingLink()
-    invocation_link.join_link(invocation_mate)
-    invocation_link.start()
-
-    invocation_ticket = links.Ticket(
-        test_operation_id, 0, test_group, test_method,
-        links.Ticket.Subscription.FULL, test_constants.LONG_TIMEOUT, None, None,
-        None, None, None, None, links.Ticket.Termination.COMPLETION, None)
-    invocation_link.accept_ticket(invocation_ticket)
-    service_mate.block_until_tickets_satisfy(test_cases.terminated)
-
-    service_ticket = links.Ticket(
-        service_mate.tickets()[-1].operation_id, 0, None, None, None, None,
-        None, None, None, None, test_code, test_message,
-        links.Ticket.Termination.COMPLETION, None)
-    service_link.accept_ticket(service_ticket)
-    invocation_mate.block_until_tickets_satisfy(test_cases.terminated)
-
-    invocation_link.stop()
-    service_link.begin_stop()
-    service_link.end_stop()
-
-    self.assertIs(
-        service_mate.tickets()[-1].termination,
-        links.Ticket.Termination.COMPLETION)
-    self.assertIs(
-        invocation_mate.tickets()[-1].termination,
-        links.Ticket.Termination.COMPLETION)
-    self.assertIs(invocation_mate.tickets()[-1].code, test_code)
-    self.assertEqual(invocation_mate.tickets()[-1].message, test_message)
-
-  def _perform_scenario_test(self, scenario):
-    test_operation_id = object()
-    test_group, test_method = scenario.group_and_method()
-    test_code = beta_interfaces.StatusCode.OK
-    test_message = 'a scenario test message'
-
-    service_link = service.service_link(
-        {(test_group, test_method): scenario.deserialize_request},
-        {(test_group, test_method): scenario.serialize_response})
-    service_mate = test_utilities.RecordingLink()
-    service_link.join_link(service_mate)
-    port = service_link.add_port('[::]:0', None)
-    service_link.start()
-    channel = _intermediary_low.Channel('localhost:%d' % port, None)
-    invocation_link = invocation.invocation_link(
-        channel, 'localhost', None,
-        {(test_group, test_method): scenario.serialize_request},
-        {(test_group, test_method): scenario.deserialize_response})
-    invocation_mate = test_utilities.RecordingLink()
-    invocation_link.join_link(invocation_mate)
-    invocation_link.start()
-
-    invocation_ticket = links.Ticket(
-        test_operation_id, 0, test_group, test_method,
-        links.Ticket.Subscription.FULL, test_constants.LONG_TIMEOUT, None, None,
-        None, None, None, None, None, None)
-    invocation_link.accept_ticket(invocation_ticket)
-    requests = scenario.requests()
-    for request_index, request in enumerate(requests):
-      request_ticket = links.Ticket(
-          test_operation_id, 1 + request_index, None, None, None, None, 1, None,
-          request, None, None, None, None, None)
-      invocation_link.accept_ticket(request_ticket)
-      service_mate.block_until_tickets_satisfy(
-          test_cases.at_least_n_payloads_received_predicate(1 + request_index))
-      response_ticket = links.Ticket(
-          service_mate.tickets()[0].operation_id, request_index, None, None,
-          None, None, 1, None, scenario.response_for_request(request), None,
-          None, None, None, None)
-      service_link.accept_ticket(response_ticket)
-      invocation_mate.block_until_tickets_satisfy(
-          test_cases.at_least_n_payloads_received_predicate(1 + request_index))
-    request_count = len(requests)
-    invocation_completion_ticket = links.Ticket(
-        test_operation_id, request_count + 1, None, None, None, None, None,
-        None, None, None, None, None, links.Ticket.Termination.COMPLETION,
-        None)
-    invocation_link.accept_ticket(invocation_completion_ticket)
-    service_mate.block_until_tickets_satisfy(test_cases.terminated)
-    service_completion_ticket = links.Ticket(
-        service_mate.tickets()[0].operation_id, request_count, None, None, None,
-        None, None, None, None, None, test_code, test_message,
-        links.Ticket.Termination.COMPLETION, None)
-    service_link.accept_ticket(service_completion_ticket)
-    invocation_mate.block_until_tickets_satisfy(test_cases.terminated)
-
-    invocation_link.stop()
-    service_link.begin_stop()
-    service_link.end_stop()
-
-    observed_requests = tuple(
-        ticket.payload for ticket in service_mate.tickets()
-        if ticket.payload is not None)
-    observed_responses = tuple(
-        ticket.payload for ticket in invocation_mate.tickets()
-        if ticket.payload is not None)
-    self.assertTrue(scenario.verify_requests(observed_requests))
-    self.assertTrue(scenario.verify_responses(observed_responses))
-
-  def testEmptyScenario(self):
-    self._perform_scenario_test(_proto_scenarios.EmptyScenario())
-
-  def testBidirectionallyUnaryScenario(self):
-    self._perform_scenario_test(_proto_scenarios.BidirectionallyUnaryScenario())
-
-  def testBidirectionallyStreamingScenario(self):
-    self._perform_scenario_test(
-        _proto_scenarios.BidirectionallyStreamingScenario())
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_metadata_code_details_test.py b/src/python/grpcio/tests/unit/_metadata_code_details_test.py
new file mode 100644
index 0000000000..0fd02d2a22
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_metadata_code_details_test.py
@@ -0,0 +1,523 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests application-provided metadata, status code, and details."""
+
+import threading
+import unittest
+
+import grpc
+from grpc.framework.foundation import logging_pool
+
+from tests.unit import test_common
+from tests.unit.framework.common import test_constants
+from tests.unit.framework.common import test_control
+
+_SERIALIZED_REQUEST = b'\x46\x47\x48'
+_SERIALIZED_RESPONSE = b'\x49\x50\x51'
+
+_REQUEST_SERIALIZER = lambda unused_request: _SERIALIZED_REQUEST
+_REQUEST_DESERIALIZER = lambda unused_serialized_request: object()
+_RESPONSE_SERIALIZER = lambda unused_response: _SERIALIZED_RESPONSE
+_RESPONSE_DESERIALIZER = lambda unused_serialized_resopnse: object()
+
+_SERVICE = 'test.TestService'
+_UNARY_UNARY = 'UnaryUnary'
+_UNARY_STREAM = 'UnaryStream'
+_STREAM_UNARY = 'StreamUnary'
+_STREAM_STREAM = 'StreamStream'
+
+_CLIENT_METADATA = (
+    ('client-md-key', 'client-md-key'),
+    ('client-md-key-bin', b'\x00\x01')
+)
+
+_SERVER_INITIAL_METADATA = (
+    ('server-initial-md-key', 'server-initial-md-value'),
+    ('server-initial-md-key-bin', b'\x00\x02')
+)
+
+_SERVER_TRAILING_METADATA = (
+    ('server-trailing-md-key', 'server-trailing-md-value'),
+    ('server-trailing-md-key-bin', b'\x00\x03')
+)
+
+_NON_OK_CODE = grpc.StatusCode.NOT_FOUND
+_DETAILS = 'Test details!'
+
+
+class _Servicer(object):
+
+  def __init__(self):
+    self._lock = threading.Lock()
+    self._code = None
+    self._details = None
+    self._exception = False
+    self._return_none = False
+    self._received_client_metadata = None
+
+  def unary_unary(self, request, context):
+    with self._lock:
+      self._received_client_metadata = context.invocation_metadata()
+      context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+      context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+      if self._code is not None:
+        context.set_code(self._code)
+      if self._details is not None:
+        context.set_details(self._details)
+      if self._exception:
+        raise test_control.Defect()
+      else:
+        return None if self._return_none else object()
+
+  def unary_stream(self, request, context):
+    with self._lock:
+      self._received_client_metadata = context.invocation_metadata()
+      context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+      context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+      if self._code is not None:
+        context.set_code(self._code)
+      if self._details is not None:
+        context.set_details(self._details)
+      for _ in range(test_constants.STREAM_LENGTH // 2):
+        yield _SERIALIZED_RESPONSE
+      if self._exception:
+        raise test_control.Defect()
+
+  def stream_unary(self, request_iterator, context):
+    with self._lock:
+      self._received_client_metadata = context.invocation_metadata()
+      context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+      context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+      if self._code is not None:
+        context.set_code(self._code)
+      if self._details is not None:
+        context.set_details(self._details)
+      # TODO(https://github.com/grpc/grpc/issues/6891): just ignore the
+      # request iterator.
+      for ignored_request in request_iterator:
+        pass
+      if self._exception:
+        raise test_control.Defect()
+      else:
+        return None if self._return_none else _SERIALIZED_RESPONSE
+
+  def stream_stream(self, request_iterator, context):
+    with self._lock:
+      self._received_client_metadata = context.invocation_metadata()
+      context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+      context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+      if self._code is not None:
+        context.set_code(self._code)
+      if self._details is not None:
+        context.set_details(self._details)
+      # TODO(https://github.com/grpc/grpc/issues/6891): just ignore the
+      # request iterator.
+      for ignored_request in request_iterator:
+        pass
+      for _ in range(test_constants.STREAM_LENGTH // 3):
+        yield object()
+      if self._exception:
+        raise test_control.Defect()
+
+  def set_code(self, code):
+    with self._lock:
+      self._code = code
+
+  def set_details(self, details):
+    with self._lock:
+      self._details = details
+
+  def set_exception(self):
+    with self._lock:
+      self._exception = True
+
+  def set_return_none(self):
+    with self._lock:
+      self._return_none = True
+
+  def received_client_metadata(self):
+    with self._lock:
+      return self._received_client_metadata
+
+
+def _generic_handler(servicer):
+  method_handlers = {
+      _UNARY_UNARY: grpc.unary_unary_rpc_method_handler(
+          servicer.unary_unary, request_deserializer=_REQUEST_DESERIALIZER,
+          response_serializer=_RESPONSE_SERIALIZER),
+      _UNARY_STREAM: grpc.unary_stream_rpc_method_handler(
+          servicer.unary_stream),
+      _STREAM_UNARY: grpc.stream_unary_rpc_method_handler(
+          servicer.stream_unary),
+      _STREAM_STREAM: grpc.stream_stream_rpc_method_handler(
+          servicer.stream_stream, request_deserializer=_REQUEST_DESERIALIZER,
+          response_serializer=_RESPONSE_SERIALIZER),
+  }
+  return grpc.method_handlers_generic_handler(_SERVICE, method_handlers)
+
+
+class MetadataCodeDetailsTest(unittest.TestCase):
+
+  def setUp(self):
+    self._servicer = _Servicer()
+    self._server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+    self._server = grpc.server(
+        (_generic_handler(self._servicer),), self._server_pool)
+    port = self._server.add_insecure_port('[::]:0')
+    self._server.start()
+
+    channel = grpc.insecure_channel('localhost:{}'.format(port))
+    self._unary_unary = channel.unary_unary(
+        '/'.join(('', _SERVICE, _UNARY_UNARY,)),
+        request_serializer=_REQUEST_SERIALIZER,
+        response_deserializer=_RESPONSE_DESERIALIZER,)
+    self._unary_stream = channel.unary_stream(
+        '/'.join(('', _SERVICE, _UNARY_STREAM,)),)
+    self._stream_unary = channel.stream_unary(
+        '/'.join(('', _SERVICE, _STREAM_UNARY,)),)
+    self._stream_stream = channel.stream_stream(
+        '/'.join(('', _SERVICE, _STREAM_STREAM,)),
+        request_serializer=_REQUEST_SERIALIZER,
+        response_deserializer=_RESPONSE_DESERIALIZER,)
+
+
+  def testSuccessfulUnaryUnary(self):
+    self._servicer.set_details(_DETAILS)
+
+    unused_response, call = self._unary_unary.with_call(
+        object(), metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, call.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(grpc.StatusCode.OK, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testSuccessfulUnaryStream(self):
+    self._servicer.set_details(_DETAILS)
+
+    call = self._unary_stream(_SERIALIZED_REQUEST, metadata=_CLIENT_METADATA)
+    received_initial_metadata = call.initial_metadata()
+    for _ in call:
+      pass
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, received_initial_metadata))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(grpc.StatusCode.OK, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testSuccessfulStreamUnary(self):
+    self._servicer.set_details(_DETAILS)
+
+    unused_response, call = self._stream_unary.with_call(
+        iter([_SERIALIZED_REQUEST] * test_constants.STREAM_LENGTH),
+        metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, call.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(grpc.StatusCode.OK, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testSuccessfulStreamStream(self):
+    self._servicer.set_details(_DETAILS)
+
+    call = self._stream_stream(
+        iter([object()] * test_constants.STREAM_LENGTH),
+        metadata=_CLIENT_METADATA)
+    received_initial_metadata = call.initial_metadata()
+    for _ in call:
+      pass
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, received_initial_metadata))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(grpc.StatusCode.OK, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testCustomCodeUnaryUnary(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      self._unary_unary.with_call(object(), metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA,
+            exception_context.exception.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA,
+            exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+  def testCustomCodeUnaryStream(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+
+    call = self._unary_stream(_SERIALIZED_REQUEST, metadata=_CLIENT_METADATA)
+    received_initial_metadata = call.initial_metadata()
+    with self.assertRaises(grpc.RpcError):
+      for _ in call:
+        pass
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, received_initial_metadata))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testCustomCodeStreamUnary(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      self._stream_unary.with_call(
+          iter([_SERIALIZED_REQUEST] * test_constants.STREAM_LENGTH),
+          metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+          _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+          _SERVER_INITIAL_METADATA,
+          exception_context.exception.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+          _SERVER_TRAILING_METADATA,
+          exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+  def testCustomCodeStreamStream(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+
+    call = self._stream_stream(
+        iter([object()] * test_constants.STREAM_LENGTH),
+        metadata=_CLIENT_METADATA)
+    received_initial_metadata = call.initial_metadata()
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      for _ in call:
+        pass
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+          _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, received_initial_metadata))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA,
+            exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+  def testCustomCodeExceptionUnaryUnary(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+    self._servicer.set_exception()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+     self._unary_unary.with_call(object(), metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA,
+            exception_context.exception.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA,
+            exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+  def testCustomCodeExceptionUnaryStream(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+    self._servicer.set_exception()
+
+    call = self._unary_stream(_SERIALIZED_REQUEST, metadata=_CLIENT_METADATA)
+    received_initial_metadata = call.initial_metadata()
+    with self.assertRaises(grpc.RpcError):
+      for _ in call:
+        pass
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, received_initial_metadata))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testCustomCodeExceptionStreamUnary(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+    self._servicer.set_exception()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      self._stream_unary.with_call(
+          iter([_SERIALIZED_REQUEST] * test_constants.STREAM_LENGTH),
+          metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA,
+            exception_context.exception.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA,
+            exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+  def testCustomCodeExceptionStreamStream(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+    self._servicer.set_exception()
+
+    call = self._stream_stream(
+        iter([object()] * test_constants.STREAM_LENGTH),
+        metadata=_CLIENT_METADATA)
+    received_initial_metadata = call.initial_metadata()
+    with self.assertRaises(grpc.RpcError):
+      for _ in call:
+        pass
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA, received_initial_metadata))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, call.code())
+    self.assertEqual(_DETAILS, call.details())
+
+  def testCustomCodeReturnNoneUnaryUnary(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+    self._servicer.set_return_none()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      self._unary_unary.with_call(object(), metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA,
+            exception_context.exception.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA,
+            exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+  def testCustomCodeReturnNoneStreamUnary(self):
+    self._servicer.set_code(_NON_OK_CODE)
+    self._servicer.set_details(_DETAILS)
+    self._servicer.set_return_none()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      self._stream_unary.with_call(
+          iter([_SERIALIZED_REQUEST] * test_constants.STREAM_LENGTH),
+          metadata=_CLIENT_METADATA)
+
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _CLIENT_METADATA, self._servicer.received_client_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_INITIAL_METADATA,
+            exception_context.exception.initial_metadata()))
+    self.assertTrue(
+        test_common.metadata_transmitted(
+            _SERVER_TRAILING_METADATA,
+            exception_context.exception.trailing_metadata()))
+    self.assertIs(_NON_OK_CODE, exception_context.exception.code())
+    self.assertEqual(_DETAILS, exception_context.exception.details())
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_metadata_test.py b/src/python/grpcio/tests/unit/_metadata_test.py
new file mode 100644
index 0000000000..c637a28039
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_metadata_test.py
@@ -0,0 +1,216 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Tests server and client side metadata API."""
+
+import unittest
+import weakref
+
+import grpc
+from grpc import _grpcio_metadata
+from grpc.framework.foundation import logging_pool
+
+from tests.unit import test_common
+from tests.unit.framework.common import test_constants
+
+_CHANNEL_ARGS = (('grpc.primary_user_agent', 'primary-agent'),
+                 ('grpc.secondary_user_agent', 'secondary-agent'))
+
+_REQUEST = b'\x00\x00\x00'
+_RESPONSE = b'\x00\x00\x00'
+
+_UNARY_UNARY = '/test/UnaryUnary'
+_UNARY_STREAM = '/test/UnaryStream'
+_STREAM_UNARY = '/test/StreamUnary'
+_STREAM_STREAM = '/test/StreamStream'
+
+_USER_AGENT = 'Python-gRPC-{}'.format(_grpcio_metadata.__version__)
+
+_CLIENT_METADATA = (
+    ('client-md-key', 'client-md-key'),
+    ('client-md-key-bin', b'\x00\x01')
+)
+
+_SERVER_INITIAL_METADATA = (
+    ('server-initial-md-key', 'server-initial-md-value'),
+    ('server-initial-md-key-bin', b'\x00\x02')
+)
+
+_SERVER_TRAILING_METADATA = (
+    ('server-trailing-md-key', 'server-trailing-md-value'),
+    ('server-trailing-md-key-bin', b'\x00\x03')
+)
+
+
+def user_agent(metadata):
+  for key, val in metadata:
+    if key == 'user-agent':
+      return val
+  raise KeyError('No user agent!')
+
+
+def validate_client_metadata(test, servicer_context):
+  test.assertTrue(test_common.metadata_transmitted(
+      _CLIENT_METADATA, servicer_context.invocation_metadata()))
+  test.assertTrue(user_agent(servicer_context.invocation_metadata())
+                  .startswith('primary-agent ' + _USER_AGENT))
+  test.assertTrue(user_agent(servicer_context.invocation_metadata())
+                  .endswith('secondary-agent'))
+
+
+def handle_unary_unary(test, request, servicer_context):
+  validate_client_metadata(test, servicer_context)
+  servicer_context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+  servicer_context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+  return _RESPONSE
+
+
+def handle_unary_stream(test, request, servicer_context):
+  validate_client_metadata(test, servicer_context)
+  servicer_context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+  servicer_context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+  for _ in range(test_constants.STREAM_LENGTH):
+    yield _RESPONSE
+
+
+def handle_stream_unary(test, request_iterator, servicer_context):
+  validate_client_metadata(test, servicer_context)
+  servicer_context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+  servicer_context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+  # TODO(issue:#6891) We should be able to remove this loop
+  for request in request_iterator:
+    pass
+  return _RESPONSE
+
+
+def handle_stream_stream(test, request_iterator, servicer_context):
+  validate_client_metadata(test, servicer_context)
+  servicer_context.send_initial_metadata(_SERVER_INITIAL_METADATA)
+  servicer_context.set_trailing_metadata(_SERVER_TRAILING_METADATA)
+  # TODO(issue:#6891) We should be able to remove this loop,
+  # and replace with return; yield
+  for request in request_iterator:
+    yield _RESPONSE
+
+
+class _MethodHandler(grpc.RpcMethodHandler):
+
+  def __init__(self, test, request_streaming, response_streaming):
+    self.request_streaming = request_streaming
+    self.response_streaming = response_streaming
+    self.request_deserializer = None
+    self.response_serializer = None
+    self.unary_unary = None
+    self.unary_stream = None
+    self.stream_unary = None
+    self.stream_stream = None
+    if self.request_streaming and self.response_streaming:
+      self.stream_stream = lambda x, y: handle_stream_stream(test, x, y)
+    elif self.request_streaming:
+      self.stream_unary = lambda x, y: handle_stream_unary(test, x, y)
+    elif self.response_streaming:
+      self.unary_stream = lambda x, y: handle_unary_stream(test, x, y)
+    else:
+      self.unary_unary = lambda x, y: handle_unary_unary(test, x, y)
+
+
+class _GenericHandler(grpc.GenericRpcHandler):
+
+  def __init__(self, test):
+    self._test = test
+
+  def service(self, handler_call_details):
+    if handler_call_details.method == _UNARY_UNARY:
+      return _MethodHandler(self._test, False, False)
+    elif handler_call_details.method == _UNARY_STREAM:
+      return _MethodHandler(self._test, False, True)
+    elif handler_call_details.method == _STREAM_UNARY:
+      return _MethodHandler(self._test, True, False)
+    elif handler_call_details.method == _STREAM_STREAM:
+      return _MethodHandler(self._test, True, True)
+    else:
+      return None
+
+
+class MetadataTest(unittest.TestCase):
+
+  def setUp(self):
+    self._server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+    self._server = grpc.server((_GenericHandler(weakref.proxy(self)),),
+                               self._server_pool)
+    port = self._server.add_insecure_port('[::]:0')
+    self._server.start()
+    self._channel = grpc.insecure_channel('localhost:%d' % port,
+                                          options=_CHANNEL_ARGS)
+
+  def tearDown(self):
+    self._server.stop(0)
+
+  def testUnaryUnary(self):
+    multi_callable = self._channel.unary_unary(_UNARY_UNARY)
+    unused_response, call = multi_callable.with_call(
+        _REQUEST, metadata=_CLIENT_METADATA)
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_INITIAL_METADATA, call.initial_metadata()))
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+
+  def testUnaryStream(self):
+    multi_callable = self._channel.unary_stream(_UNARY_STREAM)
+    call = multi_callable(_REQUEST, metadata=_CLIENT_METADATA)
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_INITIAL_METADATA, call.initial_metadata()))
+    for _ in call:
+      pass
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+
+  def testStreamUnary(self):
+    multi_callable = self._channel.stream_unary(_STREAM_UNARY)
+    unused_response, call = multi_callable.with_call(
+        [_REQUEST] * test_constants.STREAM_LENGTH,
+        metadata=_CLIENT_METADATA)
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_INITIAL_METADATA, call.initial_metadata()))
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+
+  def testStreamStream(self):
+    multi_callable = self._channel.stream_stream(_STREAM_STREAM)
+    call = multi_callable([_REQUEST] * test_constants.STREAM_LENGTH,
+                          metadata=_CLIENT_METADATA)
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_INITIAL_METADATA, call.initial_metadata()))
+    for _ in call:
+      pass
+    self.assertTrue(test_common.metadata_transmitted(
+        _SERVER_TRAILING_METADATA, call.trailing_metadata()))
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_rpc_test.py b/src/python/grpcio/tests/unit/_rpc_test.py
new file mode 100644
index 0000000000..c70d65a6df
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_rpc_test.py
@@ -0,0 +1,765 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test of RPCs made against gRPC Python's application-layer API."""
+
+import itertools
+import threading
+import unittest
+from concurrent import futures
+
+import grpc
+from grpc.framework.foundation import logging_pool
+
+from tests.unit.framework.common import test_constants
+from tests.unit.framework.common import test_control
+
+_SERIALIZE_REQUEST = lambda bytestring: bytestring * 2
+_DESERIALIZE_REQUEST = lambda bytestring: bytestring[len(bytestring) // 2:]
+_SERIALIZE_RESPONSE = lambda bytestring: bytestring * 3
+_DESERIALIZE_RESPONSE = lambda bytestring: bytestring[:len(bytestring) // 3]
+
+_UNARY_UNARY = '/test/UnaryUnary'
+_UNARY_STREAM = '/test/UnaryStream'
+_STREAM_UNARY = '/test/StreamUnary'
+_STREAM_STREAM = '/test/StreamStream'
+
+
+class _Callback(object):
+
+  def __init__(self):
+    self._condition = threading.Condition()
+    self._value = None
+    self._called = False
+
+  def __call__(self, value):
+    with self._condition:
+      self._value = value
+      self._called = True
+      self._condition.notify_all()
+
+  def value(self):
+    with self._condition:
+      while not self._called:
+        self._condition.wait()
+      return self._value
+
+
+class _Handler(object):
+
+  def __init__(self, control):
+    self._control = control
+
+  def handle_unary_unary(self, request, servicer_context):
+    self._control.control()
+    if servicer_context is not None:
+      servicer_context.set_trailing_metadata((('testkey', 'testvalue',),))
+    return request
+
+  def handle_unary_stream(self, request, servicer_context):
+    for _ in range(test_constants.STREAM_LENGTH):
+      self._control.control()
+      yield request
+    self._control.control()
+    if servicer_context is not None:
+      servicer_context.set_trailing_metadata((('testkey', 'testvalue',),))
+
+  def handle_stream_unary(self, request_iterator, servicer_context):
+    if servicer_context is not None:
+      servicer_context.invocation_metadata()
+    self._control.control()
+    response_elements = []
+    for request in request_iterator:
+      self._control.control()
+      response_elements.append(request)
+    self._control.control()
+    if servicer_context is not None:
+      servicer_context.set_trailing_metadata((('testkey', 'testvalue',),))
+    return b''.join(response_elements)
+
+  def handle_stream_stream(self, request_iterator, servicer_context):
+    self._control.control()
+    if servicer_context is not None:
+      servicer_context.set_trailing_metadata((('testkey', 'testvalue',),))
+    for request in request_iterator:
+      self._control.control()
+      yield request
+    self._control.control()
+
+
+class _MethodHandler(grpc.RpcMethodHandler):
+
+  def __init__(
+      self, request_streaming, response_streaming, request_deserializer,
+      response_serializer, unary_unary, unary_stream, stream_unary,
+      stream_stream):
+    self.request_streaming = request_streaming
+    self.response_streaming = response_streaming
+    self.request_deserializer = request_deserializer
+    self.response_serializer = response_serializer
+    self.unary_unary = unary_unary
+    self.unary_stream = unary_stream
+    self.stream_unary = stream_unary
+    self.stream_stream = stream_stream
+
+
+class _GenericHandler(grpc.GenericRpcHandler):
+
+  def __init__(self, handler):
+    self._handler = handler
+
+  def service(self, handler_call_details):
+    if handler_call_details.method == _UNARY_UNARY:
+      return _MethodHandler(
+          False, False, None, None, self._handler.handle_unary_unary, None,
+          None, None)
+    elif handler_call_details.method == _UNARY_STREAM:
+      return _MethodHandler(
+        False, True, _DESERIALIZE_REQUEST, _SERIALIZE_RESPONSE, None,
+        self._handler.handle_unary_stream, None, None)
+    elif handler_call_details.method == _STREAM_UNARY:
+      return _MethodHandler(
+          True, False, _DESERIALIZE_REQUEST, _SERIALIZE_RESPONSE, None, None,
+          self._handler.handle_stream_unary, None)
+    elif handler_call_details.method == _STREAM_STREAM:
+      return _MethodHandler(
+          True, True, None, None, None, None, None,
+          self._handler.handle_stream_stream)
+    else:
+      return None
+
+
+def _unary_unary_multi_callable(channel):
+  return channel.unary_unary(_UNARY_UNARY)
+
+
+def _unary_stream_multi_callable(channel):
+  return channel.unary_stream(
+      _UNARY_STREAM,
+      request_serializer=_SERIALIZE_REQUEST,
+      response_deserializer=_DESERIALIZE_RESPONSE)
+
+
+def _stream_unary_multi_callable(channel):
+  return channel.stream_unary(
+      _STREAM_UNARY,
+      request_serializer=_SERIALIZE_REQUEST,
+      response_deserializer=_DESERIALIZE_RESPONSE)
+
+
+def _stream_stream_multi_callable(channel):
+  return channel.stream_stream(_STREAM_STREAM)
+
+
+class RPCTest(unittest.TestCase):
+
+  def setUp(self):
+    self._control = test_control.PauseFailControl()
+    self._handler = _Handler(self._control)
+    self._server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+
+    self._server = grpc.server((), self._server_pool)
+    port = self._server.add_insecure_port('[::]:0')
+    self._server.add_generic_rpc_handlers((_GenericHandler(self._handler),))
+    self._server.start()
+
+    self._channel = grpc.insecure_channel('localhost:%d' % port)
+
+  def testUnrecognizedMethod(self):
+    request = b'abc'
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      self._channel.unary_unary('NoSuchMethod')(request)
+
+    self.assertEqual(
+        grpc.StatusCode.UNIMPLEMENTED, exception_context.exception.code())
+
+  def testSuccessfulUnaryRequestBlockingUnaryResponse(self):
+    request = b'\x07\x08'
+    expected_response = self._handler.handle_unary_unary(request, None)
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    response = multi_callable(
+        request, metadata=(
+            ('test', 'SuccessfulUnaryRequestBlockingUnaryResponse'),))
+
+    self.assertEqual(expected_response, response)
+
+  def testSuccessfulUnaryRequestBlockingUnaryResponseWithCall(self):
+    request = b'\x07\x08'
+    expected_response = self._handler.handle_unary_unary(request, None)
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    response, call = multi_callable.with_call(
+        request, metadata=(
+            ('test', 'SuccessfulUnaryRequestBlockingUnaryResponseWithCall'),))
+
+    self.assertEqual(expected_response, response)
+    self.assertIs(grpc.StatusCode.OK, call.code())
+
+  def testSuccessfulUnaryRequestFutureUnaryResponse(self):
+    request = b'\x07\x08'
+    expected_response = self._handler.handle_unary_unary(request, None)
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    response_future = multi_callable.future(
+        request, metadata=(
+            ('test', 'SuccessfulUnaryRequestFutureUnaryResponse'),))
+    response = response_future.result()
+
+    self.assertEqual(expected_response, response)
+
+  def testSuccessfulUnaryRequestStreamResponse(self):
+    request = b'\x37\x58'
+    expected_responses = tuple(self._handler.handle_unary_stream(request, None))
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    response_iterator = multi_callable(
+        request,
+        metadata=(('test', 'SuccessfulUnaryRequestStreamResponse'),))
+    responses = tuple(response_iterator)
+
+    self.assertSequenceEqual(expected_responses, responses)
+
+  def testSuccessfulStreamRequestBlockingUnaryResponse(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    expected_response = self._handler.handle_stream_unary(iter(requests), None)
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    response = multi_callable(
+        request_iterator,
+        metadata=(('test', 'SuccessfulStreamRequestBlockingUnaryResponse'),))
+
+    self.assertEqual(expected_response, response)
+
+  def testSuccessfulStreamRequestBlockingUnaryResponseWithCall(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    expected_response = self._handler.handle_stream_unary(iter(requests), None)
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    response, call = multi_callable.with_call(
+        request_iterator,
+        metadata=(
+            ('test', 'SuccessfulStreamRequestBlockingUnaryResponseWithCall'),
+        ))
+
+    self.assertEqual(expected_response, response)
+    self.assertIs(grpc.StatusCode.OK, call.code())
+
+  def testSuccessfulStreamRequestFutureUnaryResponse(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    expected_response = self._handler.handle_stream_unary(iter(requests), None)
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    response_future = multi_callable.future(
+        request_iterator,
+        metadata=(
+            ('test', 'SuccessfulStreamRequestFutureUnaryResponse'),))
+    response = response_future.result()
+
+    self.assertEqual(expected_response, response)
+
+  def testSuccessfulStreamRequestStreamResponse(self):
+    requests = tuple(b'\x77\x58' for _ in range(test_constants.STREAM_LENGTH))
+    expected_responses = tuple(
+        self._handler.handle_stream_stream(iter(requests), None))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    response_iterator = multi_callable(
+        request_iterator,
+        metadata=(('test', 'SuccessfulStreamRequestStreamResponse'),))
+    responses = tuple(response_iterator)
+
+    self.assertSequenceEqual(expected_responses, responses)
+
+  def testSequentialInvocations(self):
+    first_request = b'\x07\x08'
+    second_request = b'\x0809'
+    expected_first_response = self._handler.handle_unary_unary(
+        first_request, None)
+    expected_second_response = self._handler.handle_unary_unary(
+        second_request, None)
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    first_response = multi_callable(
+        first_request, metadata=(('test', 'SequentialInvocations'),))
+    second_response = multi_callable(
+        second_request, metadata=(('test', 'SequentialInvocations'),))
+
+    self.assertEqual(expected_first_response, first_response)
+    self.assertEqual(expected_second_response, second_response)
+
+  def testConcurrentBlockingInvocations(self):
+    pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    expected_response = self._handler.handle_stream_unary(iter(requests), None)
+    expected_responses = [expected_response] * test_constants.THREAD_CONCURRENCY
+    response_futures = [None] * test_constants.THREAD_CONCURRENCY
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    for index in range(test_constants.THREAD_CONCURRENCY):
+      request_iterator = iter(requests)
+      response_future = pool.submit(
+          multi_callable, request_iterator,
+          metadata=(('test', 'ConcurrentBlockingInvocations'),))
+      response_futures[index] = response_future
+    responses = tuple(
+        response_future.result() for response_future in response_futures)
+
+    pool.shutdown(wait=True)
+    self.assertSequenceEqual(expected_responses, responses)
+
+  def testConcurrentFutureInvocations(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    expected_response = self._handler.handle_stream_unary(iter(requests), None)
+    expected_responses = [expected_response] * test_constants.THREAD_CONCURRENCY
+    response_futures = [None] * test_constants.THREAD_CONCURRENCY
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    for index in range(test_constants.THREAD_CONCURRENCY):
+      request_iterator = iter(requests)
+      response_future = multi_callable.future(
+          request_iterator,
+          metadata=(('test', 'ConcurrentFutureInvocations'),))
+      response_futures[index] = response_future
+    responses = tuple(
+        response_future.result() for response_future in response_futures)
+
+    self.assertSequenceEqual(expected_responses, responses)
+
+  def testWaitingForSomeButNotAllConcurrentFutureInvocations(self):
+    pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+    request = b'\x67\x68'
+    expected_response = self._handler.handle_unary_unary(request, None)
+    response_futures = [None] * test_constants.THREAD_CONCURRENCY
+    lock = threading.Lock()
+    test_is_running_cell = [True]
+    def wrap_future(future):
+      def wrap():
+        try:
+          return future.result()
+        except grpc.RpcError:
+          with lock:
+            if test_is_running_cell[0]:
+              raise
+          return None
+      return wrap
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    for index in range(test_constants.THREAD_CONCURRENCY):
+      inner_response_future = multi_callable.future(
+          request,
+          metadata=(
+              ('test',
+               'WaitingForSomeButNotAllConcurrentFutureInvocations'),))
+      outer_response_future = pool.submit(wrap_future(inner_response_future))
+      response_futures[index] = outer_response_future
+
+    some_completed_response_futures_iterator = itertools.islice(
+        futures.as_completed(response_futures),
+        test_constants.THREAD_CONCURRENCY // 2)
+    for response_future in some_completed_response_futures_iterator:
+      self.assertEqual(expected_response, response_future.result())
+    with lock:
+      test_is_running_cell[0] = False
+
+  def testConsumingOneStreamResponseUnaryRequest(self):
+    request = b'\x57\x38'
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    response_iterator = multi_callable(
+        request,
+        metadata=(
+            ('test', 'ConsumingOneStreamResponseUnaryRequest'),))
+    next(response_iterator)
+
+  def testConsumingSomeButNotAllStreamResponsesUnaryRequest(self):
+    request = b'\x57\x38'
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    response_iterator = multi_callable(
+        request,
+        metadata=(
+            ('test', 'ConsumingSomeButNotAllStreamResponsesUnaryRequest'),))
+    for _ in range(test_constants.STREAM_LENGTH // 2):
+      next(response_iterator)
+
+  def testConsumingSomeButNotAllStreamResponsesStreamRequest(self):
+    requests = tuple(b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    response_iterator = multi_callable(
+        request_iterator,
+        metadata=(
+            ('test', 'ConsumingSomeButNotAllStreamResponsesStreamRequest'),))
+    for _ in range(test_constants.STREAM_LENGTH // 2):
+      next(response_iterator)
+
+  def testConsumingTooManyStreamResponsesStreamRequest(self):
+    requests = tuple(b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    response_iterator = multi_callable(
+        request_iterator,
+        metadata=(
+            ('test', 'ConsumingTooManyStreamResponsesStreamRequest'),))
+    for _ in range(test_constants.STREAM_LENGTH):
+      next(response_iterator)
+    for _ in range(test_constants.STREAM_LENGTH):
+      with self.assertRaises(StopIteration):
+        next(response_iterator)
+
+    self.assertIsNotNone(response_iterator.initial_metadata())
+    self.assertIs(grpc.StatusCode.OK, response_iterator.code())
+    self.assertIsNotNone(response_iterator.details())
+    self.assertIsNotNone(response_iterator.trailing_metadata())
+
+  def testCancelledUnaryRequestUnaryResponse(self):
+    request = b'\x07\x17'
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    with self._control.pause():
+      response_future = multi_callable.future(
+          request,
+          metadata=(('test', 'CancelledUnaryRequestUnaryResponse'),))
+      response_future.cancel()
+
+    self.assertTrue(response_future.cancelled())
+    with self.assertRaises(grpc.FutureCancelledError):
+      response_future.result()
+    self.assertIs(grpc.StatusCode.CANCELLED, response_future.code())
+
+  def testCancelledUnaryRequestStreamResponse(self):
+    request = b'\x07\x19'
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    with self._control.pause():
+      response_iterator = multi_callable(
+          request,
+          metadata=(('test', 'CancelledUnaryRequestStreamResponse'),))
+      self._control.block_until_paused()
+      response_iterator.cancel()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      next(response_iterator)
+    self.assertIs(grpc.StatusCode.CANCELLED, exception_context.exception.code())
+    self.assertIsNotNone(response_iterator.initial_metadata())
+    self.assertIs(grpc.StatusCode.CANCELLED, response_iterator.code())
+    self.assertIsNotNone(response_iterator.details())
+    self.assertIsNotNone(response_iterator.trailing_metadata())
+
+  def testCancelledStreamRequestUnaryResponse(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    with self._control.pause():
+      response_future = multi_callable.future(
+          request_iterator,
+          metadata=(('test', 'CancelledStreamRequestUnaryResponse'),))
+      self._control.block_until_paused()
+      response_future.cancel()
+
+    self.assertTrue(response_future.cancelled())
+    with self.assertRaises(grpc.FutureCancelledError):
+      response_future.result()
+    self.assertIsNotNone(response_future.initial_metadata())
+    self.assertIs(grpc.StatusCode.CANCELLED, response_future.code())
+    self.assertIsNotNone(response_future.details())
+    self.assertIsNotNone(response_future.trailing_metadata())
+
+  def testCancelledStreamRequestStreamResponse(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    with self._control.pause():
+      response_iterator = multi_callable(
+          request_iterator,
+          metadata=(('test', 'CancelledStreamRequestStreamResponse'),))
+      response_iterator.cancel()
+
+    with self.assertRaises(grpc.RpcError):
+      next(response_iterator)
+    self.assertIsNotNone(response_iterator.initial_metadata())
+    self.assertIs(grpc.StatusCode.CANCELLED, response_iterator.code())
+    self.assertIsNotNone(response_iterator.details())
+    self.assertIsNotNone(response_iterator.trailing_metadata())
+
+  def testExpiredUnaryRequestBlockingUnaryResponse(self):
+    request = b'\x07\x17'
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    with self._control.pause():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        multi_callable.with_call(
+            request, timeout=test_constants.SHORT_TIMEOUT,
+            metadata=(('test', 'ExpiredUnaryRequestBlockingUnaryResponse'),))
+
+    self.assertIsNotNone(exception_context.exception.initial_metadata())
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, exception_context.exception.code())
+    self.assertIsNotNone(exception_context.exception.details())
+    self.assertIsNotNone(exception_context.exception.trailing_metadata())
+
+  def testExpiredUnaryRequestFutureUnaryResponse(self):
+    request = b'\x07\x17'
+    callback = _Callback()
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    with self._control.pause():
+      response_future = multi_callable.future(
+          request, timeout=test_constants.SHORT_TIMEOUT,
+          metadata=(('test', 'ExpiredUnaryRequestFutureUnaryResponse'),))
+      response_future.add_done_callback(callback)
+      value_passed_to_callback = callback.value()
+
+    self.assertIs(response_future, value_passed_to_callback)
+    self.assertIsNotNone(response_future.initial_metadata())
+    self.assertIs(grpc.StatusCode.DEADLINE_EXCEEDED, response_future.code())
+    self.assertIsNotNone(response_future.details())
+    self.assertIsNotNone(response_future.trailing_metadata())
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      response_future.result()
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, exception_context.exception.code())
+    self.assertIsInstance(response_future.exception(), grpc.RpcError)
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, response_future.exception().code())
+
+  def testExpiredUnaryRequestStreamResponse(self):
+    request = b'\x07\x19'
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    with self._control.pause():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        response_iterator = multi_callable(
+            request, timeout=test_constants.SHORT_TIMEOUT,
+            metadata=(('test', 'ExpiredUnaryRequestStreamResponse'),))
+        next(response_iterator)
+
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, exception_context.exception.code())
+    self.assertIs(grpc.StatusCode.DEADLINE_EXCEEDED, response_iterator.code())
+
+  def testExpiredStreamRequestBlockingUnaryResponse(self):
+    requests = tuple(b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    with self._control.pause():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        multi_callable(
+            request_iterator, timeout=test_constants.SHORT_TIMEOUT,
+            metadata=(('test', 'ExpiredStreamRequestBlockingUnaryResponse'),))
+
+    self.assertIsNotNone(exception_context.exception.initial_metadata())
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, exception_context.exception.code())
+    self.assertIsNotNone(exception_context.exception.details())
+    self.assertIsNotNone(exception_context.exception.trailing_metadata())
+
+  def testExpiredStreamRequestFutureUnaryResponse(self):
+    requests = tuple(b'\x07\x18' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+    callback = _Callback()
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    with self._control.pause():
+      response_future = multi_callable.future(
+          request_iterator, timeout=test_constants.SHORT_TIMEOUT,
+          metadata=(('test', 'ExpiredStreamRequestFutureUnaryResponse'),))
+      response_future.add_done_callback(callback)
+      value_passed_to_callback = callback.value()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      response_future.result()
+    self.assertIs(grpc.StatusCode.DEADLINE_EXCEEDED, response_future.code())
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, exception_context.exception.code())
+    self.assertIsInstance(response_future.exception(), grpc.RpcError)
+    self.assertIs(response_future, value_passed_to_callback)
+    self.assertIsNotNone(response_future.initial_metadata())
+    self.assertIs(grpc.StatusCode.DEADLINE_EXCEEDED, response_future.code())
+    self.assertIsNotNone(response_future.details())
+    self.assertIsNotNone(response_future.trailing_metadata())
+
+  def testExpiredStreamRequestStreamResponse(self):
+    requests = tuple(b'\x67\x18' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    with self._control.pause():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        response_iterator = multi_callable(
+            request_iterator, timeout=test_constants.SHORT_TIMEOUT,
+            metadata=(('test', 'ExpiredStreamRequestStreamResponse'),))
+        next(response_iterator)
+
+    self.assertIs(
+        grpc.StatusCode.DEADLINE_EXCEEDED, exception_context.exception.code())
+    self.assertIs(grpc.StatusCode.DEADLINE_EXCEEDED, response_iterator.code())
+
+  def testFailedUnaryRequestBlockingUnaryResponse(self):
+    request = b'\x37\x17'
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    with self._control.fail():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        multi_callable.with_call(
+            request,
+            metadata=(('test', 'FailedUnaryRequestBlockingUnaryResponse'),))
+
+    self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code())
+
+  def testFailedUnaryRequestFutureUnaryResponse(self):
+    request = b'\x37\x17'
+    callback = _Callback()
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    with self._control.fail():
+      response_future = multi_callable.future(
+          request,
+          metadata=(('test', 'FailedUnaryRequestFutureUnaryResponse'),))
+      response_future.add_done_callback(callback)
+      value_passed_to_callback = callback.value()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      response_future.result()
+    self.assertIs(
+        grpc.StatusCode.UNKNOWN, exception_context.exception.code())
+    self.assertIsInstance(response_future.exception(), grpc.RpcError)
+    self.assertIs(grpc.StatusCode.UNKNOWN, response_future.exception().code())
+    self.assertIs(response_future, value_passed_to_callback)
+
+  def testFailedUnaryRequestStreamResponse(self):
+    request = b'\x37\x17'
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      with self._control.fail():
+        response_iterator = multi_callable(
+            request,
+            metadata=(('test', 'FailedUnaryRequestStreamResponse'),))
+        next(response_iterator)
+
+    self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code())
+
+  def testFailedStreamRequestBlockingUnaryResponse(self):
+    requests = tuple(b'\x47\x58' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    with self._control.fail():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        multi_callable(
+            request_iterator,
+            metadata=(('test', 'FailedStreamRequestBlockingUnaryResponse'),))
+
+    self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code())
+
+  def testFailedStreamRequestFutureUnaryResponse(self):
+    requests = tuple(b'\x07\x18' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+    callback = _Callback()
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    with self._control.fail():
+      response_future = multi_callable.future(
+          request_iterator,
+          metadata=(('test', 'FailedStreamRequestFutureUnaryResponse'),))
+      response_future.add_done_callback(callback)
+      value_passed_to_callback = callback.value()
+
+    with self.assertRaises(grpc.RpcError) as exception_context:
+      response_future.result()
+    self.assertIs(grpc.StatusCode.UNKNOWN, response_future.code())
+    self.assertIs(
+        grpc.StatusCode.UNKNOWN, exception_context.exception.code())
+    self.assertIsInstance(response_future.exception(), grpc.RpcError)
+    self.assertIs(response_future, value_passed_to_callback)
+
+  def testFailedStreamRequestStreamResponse(self):
+    requests = tuple(b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    with self._control.fail():
+      with self.assertRaises(grpc.RpcError) as exception_context:
+        response_iterator = multi_callable(
+            request_iterator,
+            metadata=(('test', 'FailedStreamRequestStreamResponse'),))
+        tuple(response_iterator)
+
+    self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code())
+    self.assertIs(grpc.StatusCode.UNKNOWN, response_iterator.code())
+
+  def testIgnoredUnaryRequestFutureUnaryResponse(self):
+    request = b'\x37\x17'
+
+    multi_callable = _unary_unary_multi_callable(self._channel)
+    multi_callable.future(
+        request,
+        metadata=(('test', 'IgnoredUnaryRequestFutureUnaryResponse'),))
+
+  def testIgnoredUnaryRequestStreamResponse(self):
+    request = b'\x37\x17'
+
+    multi_callable = _unary_stream_multi_callable(self._channel)
+    multi_callable(
+        request,
+        metadata=(('test', 'IgnoredUnaryRequestStreamResponse'),))
+
+  def testIgnoredStreamRequestFutureUnaryResponse(self):
+    requests = tuple(b'\x07\x18' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_unary_multi_callable(self._channel)
+    multi_callable.future(
+        request_iterator,
+        metadata=(('test', 'IgnoredStreamRequestFutureUnaryResponse'),))
+
+  def testIgnoredStreamRequestStreamResponse(self):
+    requests = tuple(b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH))
+    request_iterator = iter(requests)
+
+    multi_callable = _stream_stream_multi_callable(self._channel)
+    multi_callable(
+        request_iterator,
+        metadata=(('test', 'IgnoredStreamRequestStreamResponse'),))
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/_thread_cleanup_test.py b/src/python/grpcio/tests/unit/_thread_cleanup_test.py
new file mode 100644
index 0000000000..3e4f317edc
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_thread_cleanup_test.py
@@ -0,0 +1,117 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Tests for CleanupThread."""
+
+import threading
+import time
+import unittest
+
+from grpc import _common
+
+_SHORT_TIME = 0.5
+_LONG_TIME = 2.0
+_EPSILON = 0.1
+
+
+def cleanup(timeout):
+  if timeout is not None:
+    time.sleep(timeout)
+  else:
+    time.sleep(_LONG_TIME)
+
+
+def slow_cleanup(timeout):
+  # Don't respect timeout
+  time.sleep(_LONG_TIME)
+
+
+class CleanupThreadTest(unittest.TestCase):
+
+  def testTargetInvocation(self):
+    event = threading.Event()
+    def target(arg1, arg2, arg3=None):
+      self.assertEqual('arg1', arg1)
+      self.assertEqual('arg2', arg2)
+      self.assertEqual('arg3', arg3)
+      event.set()
+
+    cleanup_thread = _common.CleanupThread(behavior=lambda x: None,
+                              target=target, name='test-name',
+                              args=('arg1', 'arg2'), kwargs={'arg3': 'arg3'})
+    cleanup_thread.start()
+    cleanup_thread.join()
+    self.assertEqual(cleanup_thread.name, 'test-name')
+    self.assertTrue(event.is_set())
+
+  def testJoinNoTimeout(self):
+    cleanup_thread = _common.CleanupThread(behavior=cleanup)
+    cleanup_thread.start()
+    start_time = time.time()
+    cleanup_thread.join()
+    end_time = time.time()
+    self.assertAlmostEqual(_LONG_TIME, end_time - start_time, delta=_EPSILON)
+
+  def testJoinTimeout(self):
+    cleanup_thread = _common.CleanupThread(behavior=cleanup)
+    cleanup_thread.start()
+    start_time = time.time()
+    cleanup_thread.join(_SHORT_TIME)
+    end_time = time.time()
+    self.assertAlmostEqual(_SHORT_TIME, end_time - start_time, delta=_EPSILON)
+
+  def testJoinTimeoutSlowBehavior(self):
+    cleanup_thread = _common.CleanupThread(behavior=slow_cleanup)
+    cleanup_thread.start()
+    start_time = time.time()
+    cleanup_thread.join(_SHORT_TIME)
+    end_time = time.time()
+    self.assertAlmostEqual(_LONG_TIME, end_time - start_time, delta=_EPSILON)
+
+  def testJoinTimeoutSlowTarget(self):
+    event = threading.Event()
+    def target():
+      event.wait(_LONG_TIME)
+    cleanup_thread = _common.CleanupThread(behavior=cleanup, target=target)
+    cleanup_thread.start()
+    start_time = time.time()
+    cleanup_thread.join(_SHORT_TIME)
+    end_time = time.time()
+    self.assertAlmostEqual(_SHORT_TIME, end_time - start_time, delta=_EPSILON)
+    event.set()
+
+  def testJoinZeroTimeout(self):
+    cleanup_thread = _common.CleanupThread(behavior=cleanup)
+    cleanup_thread.start()
+    start_time = time.time()
+    cleanup_thread.join(0)
+    end_time = time.time()
+    self.assertAlmostEqual(0, end_time - start_time, delta=_EPSILON)
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/beta/_beta_features_test.py b/src/python/grpcio/tests/unit/beta/_beta_features_test.py
index bb2893a21b..3a9701b8eb 100644
--- a/src/python/grpcio/tests/unit/beta/_beta_features_test.py
+++ b/src/python/grpcio/tests/unit/beta/_beta_features_test.py
@@ -42,8 +42,8 @@ from tests.unit.framework.common import test_constants
 
 _SERVER_HOST_OVERRIDE = 'foo.test.google.fr'
 
-_PER_RPC_CREDENTIALS_METADATA_KEY = 'my-call-credentials-metadata-key'
-_PER_RPC_CREDENTIALS_METADATA_VALUE = 'my-call-credentials-metadata-value'
+_PER_RPC_CREDENTIALS_METADATA_KEY = b'my-call-credentials-metadata-key'
+_PER_RPC_CREDENTIALS_METADATA_VALUE = b'my-call-credentials-metadata-value'
 
 _GROUP = 'group'
 _UNARY_UNARY = 'unary-unary'
diff --git a/src/python/grpcio/tests/unit/beta/_connectivity_channel_test.py b/src/python/grpcio/tests/unit/beta/_connectivity_channel_test.py
index 5dc8720639..5d826a269d 100644
--- a/src/python/grpcio/tests/unit/beta/_connectivity_channel_test.py
+++ b/src/python/grpcio/tests/unit/beta/_connectivity_channel_test.py
@@ -29,162 +29,19 @@
 
 """Tests of grpc.beta._connectivity_channel."""
 
-import threading
-import time
 import unittest
 
-from grpc._adapter import _low
-from grpc._adapter import _types
-from grpc.beta import _connectivity_channel
 from grpc.beta import interfaces
-from tests.unit.framework.common import test_constants
 
 
-def _drive_completion_queue(completion_queue):
-  while True:
-    event = completion_queue.next(time.time() + 24 * 60 * 60)
-    if event.type == _types.EventType.QUEUE_SHUTDOWN:
-      break
+class ConnectivityStatesTest(unittest.TestCase):
 
-
-class _Callback(object):
-
-  def __init__(self):
-    self._condition = threading.Condition()
-    self._connectivities = []
-
-  def update(self, connectivity):
-    with self._condition:
-      self._connectivities.append(connectivity)
-      self._condition.notify()
-
-  def connectivities(self):
-    with self._condition:
-      return tuple(self._connectivities)
-
-  def block_until_connectivities_satisfy(self, predicate):
-    with self._condition:
-      while True:
-        connectivities = tuple(self._connectivities)
-        if predicate(connectivities):
-          return connectivities
-        else:
-          self._condition.wait()
-
-
-class ChannelConnectivityTest(unittest.TestCase):
-
-  def test_lonely_channel_connectivity(self):
-    low_channel = _low.Channel('localhost:12345', ())
-    callback = _Callback()
-
-    connectivity_channel = _connectivity_channel.ConnectivityChannel(
-        low_channel)
-    connectivity_channel.subscribe(callback.update, try_to_connect=False)
-    first_connectivities = callback.block_until_connectivities_satisfy(bool)
-    connectivity_channel.subscribe(callback.update, try_to_connect=True)
-    second_connectivities = callback.block_until_connectivities_satisfy(
-        lambda connectivities: 2 <= len(connectivities))
-    # Wait for a connection that will never happen.
-    time.sleep(test_constants.SHORT_TIMEOUT)
-    third_connectivities = callback.connectivities()
-    connectivity_channel.unsubscribe(callback.update)
-    fourth_connectivities = callback.connectivities()
-    connectivity_channel.unsubscribe(callback.update)
-    fifth_connectivities = callback.connectivities()
-
-    self.assertSequenceEqual(
-        (interfaces.ChannelConnectivity.IDLE,), first_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.READY, second_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.READY, third_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.READY, fourth_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.READY, fifth_connectivities)
-
-  def test_immediately_connectable_channel_connectivity(self):
-    server_completion_queue = _low.CompletionQueue()
-    server = _low.Server(server_completion_queue, [])
-    port = server.add_http2_port('[::]:0')
-    server.start()
-    server_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue, args=(server_completion_queue,))
-    server_completion_queue_thread.start()
-    low_channel = _low.Channel('localhost:%d' % port, ())
-    first_callback = _Callback()
-    second_callback = _Callback()
-
-    connectivity_channel = _connectivity_channel.ConnectivityChannel(
-        low_channel)
-    connectivity_channel.subscribe(first_callback.update, try_to_connect=False)
-    first_connectivities = first_callback.block_until_connectivities_satisfy(
-        bool)
-    # Wait for a connection that will never happen because try_to_connect=True
-    # has not yet been passed.
-    time.sleep(test_constants.SHORT_TIMEOUT)
-    second_connectivities = first_callback.connectivities()
-    connectivity_channel.subscribe(second_callback.update, try_to_connect=True)
-    third_connectivities = first_callback.block_until_connectivities_satisfy(
-        lambda connectivities: 2 <= len(connectivities))
-    fourth_connectivities = second_callback.block_until_connectivities_satisfy(
-        bool)
-    # Wait for a connection that will happen (or may already have happened).
-    first_callback.block_until_connectivities_satisfy(
-        lambda connectivities:
-        interfaces.ChannelConnectivity.READY in connectivities)
-    second_callback.block_until_connectivities_satisfy(
-        lambda connectivities:
-        interfaces.ChannelConnectivity.READY in connectivities)
-    connectivity_channel.unsubscribe(first_callback.update)
-    connectivity_channel.unsubscribe(second_callback.update)
-
-    server.shutdown()
-    server_completion_queue.shutdown()
-    server_completion_queue_thread.join()
-
-    self.assertSequenceEqual(
-        (interfaces.ChannelConnectivity.IDLE,), first_connectivities)
-    self.assertSequenceEqual(
-        (interfaces.ChannelConnectivity.IDLE,), second_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.TRANSIENT_FAILURE, third_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.FATAL_FAILURE, third_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.TRANSIENT_FAILURE,
-        fourth_connectivities)
-    self.assertNotIn(
-        interfaces.ChannelConnectivity.FATAL_FAILURE, fourth_connectivities)
-
-  def test_reachable_then_unreachable_channel_connectivity(self):
-    server_completion_queue = _low.CompletionQueue()
-    server = _low.Server(server_completion_queue, [])
-    port = server.add_http2_port('[::]:0')
-    server.start()
-    server_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue, args=(server_completion_queue,))
-    server_completion_queue_thread.start()
-    low_channel = _low.Channel('localhost:%d' % port, ())
-    callback = _Callback()
-
-    connectivity_channel = _connectivity_channel.ConnectivityChannel(
-        low_channel)
-    connectivity_channel.subscribe(callback.update, try_to_connect=True)
-    callback.block_until_connectivities_satisfy(
-        lambda connectivities:
-        interfaces.ChannelConnectivity.READY in connectivities)
-    # Now take down the server and confirm that channel readiness is repudiated.
-    server.shutdown()
-    callback.block_until_connectivities_satisfy(
-        lambda connectivities:
-        connectivities[-1] is not interfaces.ChannelConnectivity.READY)
-    connectivity_channel.unsubscribe(callback.update)
-
-    server.shutdown()
-    server_completion_queue.shutdown()
-    server_completion_queue_thread.join()
+  def testBetaConnectivityStates(self):
+    self.assertIsNotNone(interfaces.ChannelConnectivity.IDLE)
+    self.assertIsNotNone(interfaces.ChannelConnectivity.CONNECTING)
+    self.assertIsNotNone(interfaces.ChannelConnectivity.READY)
+    self.assertIsNotNone(interfaces.ChannelConnectivity.TRANSIENT_FAILURE)
+    self.assertIsNotNone(interfaces.ChannelConnectivity.FATAL_FAILURE)
 
 
 if __name__ == '__main__':
diff --git a/src/python/grpcio/tests/unit/beta/_implementations_test.py b/src/python/grpcio/tests/unit/beta/_implementations_test.py
index 26be670c45..127f93e9bb 100644
--- a/src/python/grpcio/tests/unit/beta/_implementations_test.py
+++ b/src/python/grpcio/tests/unit/beta/_implementations_test.py
@@ -29,8 +29,11 @@
 
 """Tests the implementations module of the gRPC Python Beta API."""
 
+import datetime
 import unittest
 
+from oauth2client import client as oauth2client_client
+
 from grpc.beta import implementations
 from tests.unit import resources
 
@@ -49,5 +52,19 @@ class ChannelCredentialsTest(unittest.TestCase):
         channel_credentials, implementations.ChannelCredentials)
 
 
+class CallCredentialsTest(unittest.TestCase):
+
+  def test_google_call_credentials(self):
+    creds = oauth2client_client.GoogleCredentials(
+        'token', 'client_id', 'secret', 'refresh_token',
+        datetime.datetime(2008, 6, 24), 'https://refresh.uri.com/',
+        'user_agent')
+    call_creds = implementations.google_call_credentials(creds)
+    self.assertIsInstance(call_creds, implementations.CallCredentials)
+
+  def test_access_token_call_credentials(self):
+    call_creds = implementations.access_token_call_credentials('token')
+    self.assertIsInstance(call_creds, implementations.CallCredentials)
+
 if __name__ == '__main__':
   unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/beta/_not_found_test.py b/src/python/grpcio/tests/unit/beta/_not_found_test.py
index 44fcd1e13c..37b8c49120 100644
--- a/src/python/grpcio/tests/unit/beta/_not_found_test.py
+++ b/src/python/grpcio/tests/unit/beta/_not_found_test.py
@@ -61,7 +61,7 @@ class NotFoundTest(unittest.TestCase):
 
   def test_future_stream_unary_not_found(self):
     rpc_future = self._generic_stub.future_stream_unary(
-        'grupe', 'mevvod', b'def', test_constants.LONG_TIMEOUT)
+        'grupe', 'mevvod', [b'def'], test_constants.LONG_TIMEOUT)
     with self.assertRaises(face.LocalError) as exception_assertion_context:
       rpc_future.result()
     self.assertIs(
diff --git a/src/python/grpcio/tests/unit/beta/_utilities_test.py b/src/python/grpcio/tests/unit/beta/_utilities_test.py
index 08ce98e751..90fe10c77c 100644
--- a/src/python/grpcio/tests/unit/beta/_utilities_test.py
+++ b/src/python/grpcio/tests/unit/beta/_utilities_test.py
@@ -33,21 +33,12 @@ import threading
 import time
 import unittest
 
-from grpc._adapter import _low
-from grpc._adapter import _types
 from grpc.beta import implementations
 from grpc.beta import utilities
 from grpc.framework.foundation import future
 from tests.unit.framework.common import test_constants
 
 
-def _drive_completion_queue(completion_queue):
-  while True:
-    event = completion_queue.next(time.time() + 24 * 60 * 60)
-    if event.type == _types.EventType.QUEUE_SHUTDOWN:
-      break
-
-
 class _Callback(object):
 
   def __init__(self):
@@ -87,13 +78,9 @@ class ChannelConnectivityTest(unittest.TestCase):
     self.assertFalse(ready_future.running())
 
   def test_immediately_connectable_channel_connectivity(self):
-    server_completion_queue = _low.CompletionQueue()
-    server = _low.Server(server_completion_queue, [])
-    port = server.add_http2_port('[::]:0')
+    server = implementations.server({})
+    port = server.add_insecure_port('[::]:0')
     server.start()
-    server_completion_queue_thread = threading.Thread(
-        target=_drive_completion_queue, args=(server_completion_queue,))
-    server_completion_queue_thread.start()
     channel = implementations.insecure_channel('localhost', port)
     callback = _Callback()
 
@@ -114,9 +101,7 @@ class ChannelConnectivityTest(unittest.TestCase):
       self.assertFalse(ready_future.running())
     finally:
       ready_future.cancel()
-      server.shutdown()
-      server_completion_queue.shutdown()
-      server_completion_queue_thread.join()
+      server.stop(0)
 
 
 if __name__ == '__main__':
diff --git a/src/python/grpcio/tests/unit/beta/test_utilities.py b/src/python/grpcio/tests/unit/beta/test_utilities.py
index 0313e06a93..692da9c97d 100644
--- a/src/python/grpcio/tests/unit/beta/test_utilities.py
+++ b/src/python/grpcio/tests/unit/beta/test_utilities.py
@@ -29,7 +29,7 @@
 
 """Test-appropriate entry points into the gRPC Python Beta API."""
 
-from grpc._adapter import _intermediary_low
+import grpc
 from grpc.beta import implementations
 
 
@@ -48,9 +48,8 @@ def not_really_secure_channel(
     An implementations.Channel to the remote host through which RPCs may be
       conducted.
   """
-  hostport = '%s:%d' % (host, port)
-  intermediary_low_channel = _intermediary_low.Channel(
-      hostport, channel_credentials._low_credentials,
-      server_host_override=server_host_override)
-  return implementations.Channel(
-      intermediary_low_channel._internal, intermediary_low_channel)
+  target = '%s:%d' % (host, port)
+  channel = grpc.secure_channel(
+      target, channel_credentials,
+      (('grpc.ssl_target_name_override', server_host_override,),))
+  return implementations.Channel(channel)
diff --git a/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py b/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py
deleted file mode 100644
index 43457be362..0000000000
--- a/src/python/grpcio/tests/unit/framework/_crust_over_core_face_interface_test.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests Face interface compliance of the crust-over-core stack."""
-
-import collections
-import unittest
-
-import six
-
-from grpc.framework.core import implementations as core_implementations
-from grpc.framework.crust import implementations as crust_implementations
-from grpc.framework.foundation import logging_pool
-from grpc.framework.interfaces.links import utilities
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.interfaces.face import test_cases
-from tests.unit.framework.interfaces.face import test_interfaces
-from tests.unit.framework.interfaces.links import test_utilities
-
-
-class _Implementation(test_interfaces.Implementation):
-
-  def instantiate(
-      self, methods, method_implementations, multi_method_implementation):
-    pool = logging_pool.pool(test_constants.POOL_SIZE)
-    servicer = crust_implementations.servicer(
-        method_implementations, multi_method_implementation, pool)
-
-    service_end_link = core_implementations.service_end_link(
-        servicer, test_constants.DEFAULT_TIMEOUT,
-        test_constants.MAXIMUM_TIMEOUT)
-    invocation_end_link = core_implementations.invocation_end_link()
-    invocation_end_link.join_link(service_end_link)
-    service_end_link.join_link(invocation_end_link)
-    service_end_link.start()
-    invocation_end_link.start()
-
-    generic_stub = crust_implementations.generic_stub(invocation_end_link, pool)
-    # TODO(nathaniel): Add a "groups" attribute to _digest.TestServiceDigest.
-    group = next(iter(methods))[0]
-    # TODO(nathaniel): Add a "cardinalities_by_group" attribute to
-    # _digest.TestServiceDigest.
-    cardinalities = {
-        method: method_object.cardinality()
-        for (group, method), method_object in six.iteritems(methods)}
-    dynamic_stub = crust_implementations.dynamic_stub(
-        invocation_end_link, group, cardinalities, pool)
-
-    return generic_stub, {group: dynamic_stub}, (
-        invocation_end_link, service_end_link, pool)
-
-  def destantiate(self, memo):
-    invocation_end_link, service_end_link, pool = memo
-    invocation_end_link.stop(0).wait()
-    service_end_link.stop(0).wait()
-    invocation_end_link.join_link(utilities.NULL_LINK)
-    service_end_link.join_link(utilities.NULL_LINK)
-    pool.shutdown(wait=True)
-
-  def invocation_metadata(self):
-    return object()
-
-  def initial_metadata(self):
-    return object()
-
-  def terminal_metadata(self):
-    return object()
-
-  def code(self):
-    return object()
-
-  def details(self):
-    return object()
-
-  def metadata_transmitted(self, original_metadata, transmitted_metadata):
-    return original_metadata is transmitted_metadata
-
-
-def load_tests(loader, tests, pattern):
-  return unittest.TestSuite(
-      tests=tuple(
-          loader.loadTestsFromTestCase(test_case_class)
-          for test_case_class in test_cases.test_cases(_Implementation())))
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/framework/common/test_constants.py b/src/python/grpcio/tests/unit/framework/common/test_constants.py
index 8d89101e09..b6682d396c 100644
--- a/src/python/grpcio/tests/unit/framework/common/test_constants.py
+++ b/src/python/grpcio/tests/unit/framework/common/test_constants.py
@@ -49,8 +49,13 @@ STREAM_LENGTH = 200
 # The size of payloads to transmit in tests.
 PAYLOAD_SIZE = 256 * 1024 + 17
 
-# The parallelism to use in tests of parallel RPCs.
-PARALLELISM = 200
+# The concurrency to use in tests of concurrent RPCs that will not create as
+# many threads as RPCs.
+RPC_CONCURRENCY = 200
+
+# The concurrency to use in tests of concurrent RPCs that will create as many
+# threads as RPCs.
+THREAD_CONCURRENCY = 25
 
 # The size of thread pools to use in tests.
 POOL_SIZE = 10
diff --git a/src/python/grpcio/tests/unit/framework/common/test_control.py b/src/python/grpcio/tests/unit/framework/common/test_control.py
index ca5ba3a854..088e2f8b88 100644
--- a/src/python/grpcio/tests/unit/framework/common/test_control.py
+++ b/src/python/grpcio/tests/unit/framework/common/test_control.py
@@ -60,10 +60,16 @@ class Control(six.with_metaclass(abc.ABCMeta)):
 
 
 class PauseFailControl(Control):
-  """A Control that can be used to pause or fail code under control."""
+  """A Control that can be used to pause or fail code under control.
+
+  This object is only safe for use from two threads: one of the system under
+  test calling control and the other from the test system calling pause,
+  block_until_paused, and fail.
+  """
 
   def __init__(self):
     self._condition = threading.Condition()
+    self._pause = False
     self._paused = False
     self._fail = False
 
@@ -72,19 +78,31 @@ class PauseFailControl(Control):
       if self._fail:
         raise Defect()
 
-      while self._paused:
+      while self._pause:
+        self._paused = True
+        self._condition.notify_all()
         self._condition.wait()
+      self._paused = False
 
   @contextlib.contextmanager
   def pause(self):
     """Pauses code under control while controlling code is in context."""
     with self._condition:
-      self._paused = True
+      self._pause = True
     yield
     with self._condition:
-      self._paused = False
+      self._pause = False
       self._condition.notify_all()
 
+  def block_until_paused(self):
+    """Blocks controlling code until code under control is paused.
+
+    May only be called within the context of a pause call.
+    """
+    with self._condition:
+      while not self._paused:
+        self._condition.wait()
+
   @contextlib.contextmanager
   def fail(self):
     """Fails code under control while controlling code is in context."""
diff --git a/src/python/grpcio/tests/unit/framework/core/_base_interface_test.py b/src/python/grpcio/tests/unit/framework/core/_base_interface_test.py
deleted file mode 100644
index 1310292306..0000000000
--- a/src/python/grpcio/tests/unit/framework/core/_base_interface_test.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests the RPC Framework Core's implementation of the Base interface."""
-
-import logging
-import random
-import time
-import unittest
-
-from grpc.framework.core import implementations
-from grpc.framework.interfaces.base import utilities
-from tests.unit.framework.common import test_constants
-from tests.unit.framework.interfaces.base import test_cases
-from tests.unit.framework.interfaces.base import test_interfaces
-
-
-class _Implementation(test_interfaces.Implementation):
-
-  def __init__(self):
-    self._invocation_initial_metadata = object()
-    self._service_initial_metadata = object()
-    self._invocation_terminal_metadata = object()
-    self._service_terminal_metadata = object()
-
-  def instantiate(self, serializations, servicer):
-    invocation = implementations.invocation_end_link()
-    service = implementations.service_end_link(
-        servicer, test_constants.DEFAULT_TIMEOUT,
-        test_constants.MAXIMUM_TIMEOUT)
-    invocation.join_link(service)
-    service.join_link(invocation)
-    return invocation, service, None
-
-  def destantiate(self, memo):
-    pass
-
-  def invocation_initial_metadata(self):
-    return self._invocation_initial_metadata
-
-  def service_initial_metadata(self):
-    return self._service_initial_metadata
-
-  def invocation_completion(self):
-    return utilities.completion(self._invocation_terminal_metadata, None, None)
-
-  def service_completion(self):
-    return utilities.completion(self._service_terminal_metadata, None, None)
-
-  def metadata_transmitted(self, original_metadata, transmitted_metadata):
-    return transmitted_metadata is original_metadata
-
-  def completion_transmitted(self, original_completion, transmitted_completion):
-    return (
-        (original_completion.terminal_metadata is
-         transmitted_completion.terminal_metadata) and
-        original_completion.code is transmitted_completion.code and
-        original_completion.message is transmitted_completion.message
-    )
-
-
-def load_tests(loader, tests, pattern):
-  return unittest.TestSuite(
-      tests=tuple(
-          loader.loadTestsFromTestCase(test_case_class)
-          for test_case_class in test_cases.test_cases(_Implementation())))
-
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/framework/foundation/_later_test.py b/src/python/grpcio/tests/unit/framework/foundation/_later_test.py
deleted file mode 100644
index 6c2459e185..0000000000
--- a/src/python/grpcio/tests/unit/framework/foundation/_later_test.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Tests of the later module."""
-
-import threading
-import time
-import unittest
-
-from grpc.framework.foundation import later
-
-TICK = 0.1
-
-
-class LaterTest(unittest.TestCase):
-
-  def test_simple_delay(self):
-    lock = threading.Lock()
-    cell = [0]
-    return_value = object()
-
-    def computation():
-      with lock:
-        cell[0] += 1
-      return return_value
-    computation_future = later.later(TICK * 2, computation)
-
-    self.assertFalse(computation_future.done())
-    self.assertFalse(computation_future.cancelled())
-    time.sleep(TICK)
-    self.assertFalse(computation_future.done())
-    self.assertFalse(computation_future.cancelled())
-    with lock:
-      self.assertEqual(0, cell[0])
-    time.sleep(TICK * 2)
-    self.assertTrue(computation_future.done())
-    self.assertFalse(computation_future.cancelled())
-    with lock:
-      self.assertEqual(1, cell[0])
-    self.assertEqual(return_value, computation_future.result())
-
-  def test_callback(self):
-    lock = threading.Lock()
-    cell = [0]
-    callback_called = [False]
-    future_passed_to_callback = [None]
-    def computation():
-      with lock:
-        cell[0] += 1
-    computation_future = later.later(TICK * 2, computation)
-    def callback(outcome):
-      with lock:
-        callback_called[0] = True
-        future_passed_to_callback[0] = outcome
-    computation_future.add_done_callback(callback)
-    time.sleep(TICK)
-    with lock:
-      self.assertFalse(callback_called[0])
-    time.sleep(TICK * 2)
-    with lock:
-      self.assertTrue(callback_called[0])
-      self.assertTrue(future_passed_to_callback[0].done())
-
-      callback_called[0] = False
-      future_passed_to_callback[0] = None
-
-    computation_future.add_done_callback(callback)
-    with lock:
-      self.assertTrue(callback_called[0])
-      self.assertTrue(future_passed_to_callback[0].done())
-
-  def test_cancel(self):
-    lock = threading.Lock()
-    cell = [0]
-    callback_called = [False]
-    future_passed_to_callback = [None]
-    def computation():
-      with lock:
-        cell[0] += 1
-    computation_future = later.later(TICK * 2, computation)
-    def callback(outcome):
-      with lock:
-        callback_called[0] = True
-        future_passed_to_callback[0] = outcome
-    computation_future.add_done_callback(callback)
-    time.sleep(TICK)
-    with lock:
-      self.assertFalse(callback_called[0])
-    computation_future.cancel()
-    self.assertTrue(computation_future.cancelled())
-    self.assertFalse(computation_future.running())
-    self.assertTrue(computation_future.done())
-    with lock:
-      self.assertTrue(callback_called[0])
-      self.assertTrue(future_passed_to_callback[0].cancelled())
-
-  def test_result(self):
-    lock = threading.Lock()
-    cell = [0]
-    callback_called = [False]
-    future_passed_to_callback_cell = [None]
-    return_value = object()
-
-    def computation():
-      with lock:
-        cell[0] += 1
-      return return_value
-    computation_future = later.later(TICK * 2, computation)
-
-    def callback(future_passed_to_callback):
-      with lock:
-        callback_called[0] = True
-        future_passed_to_callback_cell[0] = future_passed_to_callback
-    computation_future.add_done_callback(callback)
-    returned_value = computation_future.result()
-    self.assertEqual(return_value, returned_value)
-
-    # The callback may not yet have been called! Sleep a tick.
-    time.sleep(TICK)
-    with lock:
-      self.assertTrue(callback_called[0])
-      self.assertEqual(return_value, future_passed_to_callback_cell[0].result())
-
-if __name__ == '__main__':
-  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/base/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/base/test_cases.py
index 4f8e26c9a2..5d16bf98be 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/base/test_cases.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/base/test_cases.py
@@ -29,6 +29,8 @@
 
 """Tests of the base interface of RPC Framework."""
 
+from __future__ import division
+
 import logging
 import random
 import threading
@@ -54,13 +56,13 @@ class _Serialization(test_interfaces.Serialization):
     return request + request
 
   def deserialize_request(self, serialized_request):
-    return serialized_request[:len(serialized_request) / 2]
+    return serialized_request[:len(serialized_request) // 2]
 
   def serialize_response(self, response):
     return response * 3
 
   def deserialize_response(self, serialized_response):
-    return serialized_response[2 * len(serialized_response) / 3:]
+    return serialized_response[2 * len(serialized_response) // 3:]
 
 
 def _advance(quadruples, operator, controller):
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py
index 649892463a..e338aaa396 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py
@@ -146,13 +146,13 @@ class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.
         test_messages.verify(second_request, second_response, self)
 
   def testParallelInvocations(self):
-    pool = logging_pool.pool(test_constants.PARALLELISM)
+    pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
     for (group, method), test_messages_sequence in (
         six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures = []
-        for _ in range(test_constants.PARALLELISM):
+        for _ in range(test_constants.THREAD_CONCURRENCY):
           request = test_messages.request()
           response_future = pool.submit(
               self._invoker.blocking(group, method), request,
@@ -168,13 +168,13 @@ class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.
     pool.shutdown(wait=True)
 
   def testWaitingForSomeButNotAllParallelInvocations(self):
-    pool = logging_pool.pool(test_constants.PARALLELISM)
+    pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
     for (group, method), test_messages_sequence in (
         six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures_to_indices = {}
-        for index in range(test_constants.PARALLELISM):
+        for index in range(test_constants.THREAD_CONCURRENCY):
           request = test_messages.request()
           response_future = pool.submit(
               self._invoker.blocking(group, method), request,
@@ -184,7 +184,7 @@ class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.
 
         some_completed_response_futures_iterator = itertools.islice(
             futures.as_completed(response_futures_to_indices),
-            test_constants.PARALLELISM // 2)
+            test_constants.THREAD_CONCURRENCY // 2)
         for response_future in some_completed_response_futures_iterator:
           index = response_futures_to_indices[response_future]
           test_messages.verify(requests[index], response_future.result(), self)
diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py
index c3813d5f3a..791620307b 100644
--- a/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py
+++ b/src/python/grpcio/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py
@@ -249,7 +249,7 @@ class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.
       for test_messages in test_messages_sequence:
         requests = []
         response_futures = []
-        for _ in range(test_constants.PARALLELISM):
+        for _ in range(test_constants.THREAD_CONCURRENCY):
           request = test_messages.request()
           response_future = self._invoker.future(group, method)(
               request, test_constants.LONG_TIMEOUT)
@@ -263,13 +263,13 @@ class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.
           test_messages.verify(request, response, self)
 
   def testWaitingForSomeButNotAllParallelInvocations(self):
-    pool = logging_pool.pool(test_constants.PARALLELISM)
+    pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
     for (group, method), test_messages_sequence in (
         six.iteritems(self._digest.unary_unary_messages_sequences)):
       for test_messages in test_messages_sequence:
         requests = []
         response_futures_to_indices = {}
-        for index in range(test_constants.PARALLELISM):
+        for index in range(test_constants.THREAD_CONCURRENCY):
           request = test_messages.request()
           inner_response_future = self._invoker.future(group, method)(
               request, test_constants.LONG_TIMEOUT)
@@ -279,7 +279,7 @@ class TestCase(six.with_metaclass(abc.ABCMeta, test_coverage.Coverage, unittest.
 
         some_completed_response_futures_iterator = itertools.islice(
             futures.as_completed(response_futures_to_indices),
-            test_constants.PARALLELISM // 2)
+            test_constants.THREAD_CONCURRENCY // 2)
         for response_future in some_completed_response_futures_iterator:
           index = response_futures_to_indices[response_future]
           test_messages.verify(requests[index], response_future.result(), self)
diff --git a/src/python/grpcio/tests/unit/test_common.py b/src/python/grpcio/tests/unit/test_common.py
index 7b4d20dccb..c8886bf4ca 100644
--- a/src/python/grpcio/tests/unit/test_common.py
+++ b/src/python/grpcio/tests/unit/test_common.py
@@ -33,10 +33,10 @@ import collections
 
 import six
 
-INVOCATION_INITIAL_METADATA = ((b'0', b'abc'), (b'1', b'def'), (b'2', b'ghi'),)
-SERVICE_INITIAL_METADATA = ((b'3', b'jkl'), (b'4', b'mno'), (b'5', b'pqr'),)
-SERVICE_TERMINAL_METADATA = ((b'6', b'stu'), (b'7', b'vwx'), (b'8', b'yza'),)
-DETAILS = b'test details'
+INVOCATION_INITIAL_METADATA = (('0', 'abc'), ('1', 'def'), ('2', 'ghi'),)
+SERVICE_INITIAL_METADATA = (('3', 'jkl'), ('4', 'mno'), ('5', 'pqr'),)
+SERVICE_TERMINAL_METADATA = (('6', 'stu'), ('7', 'vwx'), ('8', 'yza'),)
+DETAILS = 'test details'
 
 
 def metadata_transmitted(original_metadata, transmitted_metadata):
@@ -59,12 +59,10 @@ def metadata_transmitted(original_metadata, transmitted_metadata):
       original_metadata after having been transmitted via gRPC.
   """
   original = collections.defaultdict(list)
-  for key_value_pair in original_metadata:
-    key, value = tuple(key_value_pair)
+  for key, value in original_metadata:
     original[key].append(value)
   transmitted = collections.defaultdict(list)
-  for key_value_pair in transmitted_metadata:
-    key, value = tuple(key_value_pair)
+  for key, value in transmitted_metadata:
     transmitted[key].append(value)
 
   for key, values in six.iteritems(original):
diff --git a/src/python/grpcio_health_checking/.gitignore b/src/python/grpcio_health_checking/.gitignore
new file mode 100644
index 0000000000..85af466886
--- /dev/null
+++ b/src/python/grpcio_health_checking/.gitignore
@@ -0,0 +1,5 @@
+*.proto
+*_pb2.py
+build/
+grpcio_health_checking.egg-info/
+dist/
diff --git a/src/python/grpcio_health_checking/MANIFEST.in b/src/python/grpcio_health_checking/MANIFEST.in
index 498b55f20a..7d26647697 100644
--- a/src/python/grpcio_health_checking/MANIFEST.in
+++ b/src/python/grpcio_health_checking/MANIFEST.in
@@ -1,2 +1,3 @@
-graft grpc
-include commands.py
+include health_commands.py
+graft grpc_health
+global-exclude *.pyc
diff --git a/src/python/grpcio_health_checking/grpc/health/v1/health.py b/src/python/grpcio_health_checking/grpc/health/v1/health.py
deleted file mode 100644
index 4b5af15aa6..0000000000
--- a/src/python/grpcio_health_checking/grpc/health/v1/health.py
+++ /dev/null
@@ -1,129 +0,0 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Reference implementation for health checking in gRPC Python."""
-
-import abc
-import enum
-import threading
-
-from grpc.health.v1 import health_pb2
-
-
-@enum.unique
-class HealthStatus(enum.Enum):
-  """Statuses for a service mirroring the reference health.proto's values."""
-  UNKNOWN = health_pb2.HealthCheckResponse.UNKNOWN
-  SERVING = health_pb2.HealthCheckResponse.SERVING
-  NOT_SERVING = health_pb2.HealthCheckResponse.NOT_SERVING
-
-
-class _HealthServicer(health_pb2.EarlyAdopterHealthServicer):
-  """Servicer handling RPCs for service statuses."""
-
-  def __init__(self):
-    self._server_status_lock = threading.Lock()
-    self._server_status = {}
-
-  def Check(self, request, context):
-    with self._server_status_lock:
-      if request.service not in self._server_status:
-        # TODO(atash): once the Python API has a way of setting the server
-        # status, bring us into conformance with the health check spec by
-        # returning the NOT_FOUND status here.
-        raise NotImplementedError()
-      else:
-        return health_pb2.HealthCheckResponse(
-            status=self._server_status[request.service].value)
-
-  def set(service, status):
-    if not isinstance(status, HealthStatus):
-      raise TypeError('expected grpc.health.v1.health.HealthStatus '
-                      'for argument `status` but got {}'.format(status))
-    with self._server_status_lock:
-      self._server_status[service] = status
-
-
-class HealthServer(health_pb2.EarlyAdopterHealthServer):
-  """Interface for the reference gRPC Python health server."""
-  __metaclass__ = abc.ABCMeta
-
-  @abc.abstractmethod
-  def start(self):
-    raise NotImplementedError()
-
-  @abc.abstractmethod
-  def stop(self):
-    raise NotImplementedError()
-
-  @abc.abstractmethod
-  def set(self, service, status):
-    """Set the status of the given service.
-
-    Args:
-      service (str): service name of the service to set the reported status of
-      status (HealthStatus): status to set for the specified service
-    """
-    raise NotImplementedError()
-
-
-class _HealthServerImplementation(HealthServer):
-  """Implementation for the reference gRPC Python health server."""
-
-  def __init__(self, server, servicer):
-    self._server = server
-    self._servicer = servicer
-
-  def start(self):
-    self._server.start()
-
-  def stop(self):
-    self._server.stop()
-
-  def set(self, service, status):
-    self._servicer.set(service, status)
-
-
-def create_Health_server(port, private_key=None, certificate_chain=None):
-  """Get a HealthServer instance.
-
-  Args:
-    port (int): port number passed through to health_pb2 server creation
-      routine.
-    private_key (str): to-be-created server's desired private key
-    certificate_chain (str): to-be-created server's desired certificate chain
-
-  Returns:
-    An instance of HealthServer (conforming thus to
-    EarlyAdopterHealthServer and providing a method to set server status)."""
-  servicer = _HealthServicer()
-  server = health_pb2.early_adopter_create_Health_server(
-      servicer, port=port, private_key=private_key,
-      certificate_chain=certificate_chain)
-  return _HealthServerImplementation(server, servicer)
diff --git a/src/python/grpcio_health_checking/grpc/__init__.py b/src/python/grpcio_health_checking/grpc_health/__init__.py
index 7086519106..7086519106 100644
--- a/src/python/grpcio_health_checking/grpc/__init__.py
+++ b/src/python/grpcio_health_checking/grpc_health/__init__.py
diff --git a/src/python/grpcio_health_checking/grpc/health/__init__.py b/src/python/grpcio_health_checking/grpc_health/health/__init__.py
index 7086519106..7086519106 100644
--- a/src/python/grpcio_health_checking/grpc/health/__init__.py
+++ b/src/python/grpcio_health_checking/grpc_health/health/__init__.py
diff --git a/src/python/grpcio_health_checking/grpc/health/v1/__init__.py b/src/python/grpcio_health_checking/grpc_health/health/v1/__init__.py
index 7086519106..7086519106 100644
--- a/src/python/grpcio_health_checking/grpc/health/v1/__init__.py
+++ b/src/python/grpcio_health_checking/grpc_health/health/v1/__init__.py
diff --git a/src/python/grpcio_health_checking/grpc_health/health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/health/v1/health.py
new file mode 100644
index 0000000000..8da60c70cb
--- /dev/null
+++ b/src/python/grpcio_health_checking/grpc_health/health/v1/health.py
@@ -0,0 +1,66 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Reference implementation for health checking in gRPC Python."""
+
+import threading
+
+from grpc_health.health.v1 import health_pb2
+
+
+class HealthServicer(health_pb2.BetaHealthServicer):
+  """Servicer handling RPCs for service statuses."""
+
+  def __init__(self):
+    self._server_status_lock = threading.Lock()
+    self._server_status = {}
+
+  def Check(self, request, context):
+    with self._server_status_lock:
+      if request.service not in self._server_status:
+        # TODO(atash): once the Python API has a way of setting the server
+        # status, bring us into conformance with the health check spec by
+        # returning the NOT_FOUND status here.
+        raise NotImplementedError()
+      else:
+        return health_pb2.HealthCheckResponse(
+            status=self._server_status[request.service])
+
+  def set(self, service, status):
+    """Sets the status of a service.
+
+    Args:
+        service: string, the name of the service.
+            NOTE, '' must be set.
+        status: HealthCheckResponse.status enum value indicating
+            the status of the service
+    """
+    with self._server_status_lock:
+      self._server_status[service] = status
+
diff --git a/src/python/grpcio_health_checking/commands.py b/src/python/grpcio_health_checking/health_commands.py
index 3f4ea6e22f..631066f331 100644
--- a/src/python/grpcio_health_checking/commands.py
+++ b/src/python/grpcio_health_checking/health_commands.py
@@ -33,11 +33,16 @@ import distutils
 import glob
 import os
 import os.path
+import shutil
 import subprocess
 import sys
 
 import setuptools
 from setuptools.command import build_py
+from setuptools.command import sdist
+
+ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
+HEALTH_PROTO = os.path.join(ROOT_DIR, '../../proto/grpc/health/v1/health.proto')
 
 
 class BuildProtoModules(setuptools.Command):
@@ -76,9 +81,34 @@ class BuildProtoModules(setuptools.Command):
       raise Exception('{}\nOutput:\n{}'.format(e.message, e.output))
 
 
+class CopyProtoModules(setuptools.Command):
+  """Command to copy proto modules from grpc/src/proto."""
+
+  def initialize_options(self):
+    pass
+
+  def finalize_options(self):
+    pass
+
+  def run(self):
+    if os.path.isfile(HEALTH_PROTO):
+      shutil.copyfile(
+          HEALTH_PROTO,
+          os.path.join(ROOT_DIR, 'grpc_health/health/v1/health.proto'))
+
+
 class BuildPy(build_py.build_py):
   """Custom project build command."""
 
   def run(self):
+    self.run_command('copy_proto_modules')
     self.run_command('build_proto_modules')
     build_py.build_py.run(self)
+
+
+class SDist(sdist.sdist):
+  """Custom project build command."""
+
+  def run(self):
+    self.run_command('copy_proto_modules')
+    sdist.sdist.run(self)
diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py
index 35253ba312..d68a7ced8e 100644
--- a/src/python/grpcio_health_checking/setup.py
+++ b/src/python/grpcio_health_checking/setup.py
@@ -40,7 +40,7 @@ import setuptools
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 # Break import-style to ensure we can actually find our commands module.
-import commands
+import health_commands
 
 _PACKAGES = (
     setuptools.find_packages('.')
@@ -51,22 +51,21 @@ _PACKAGE_DIRECTORIES = {
 }
 
 _INSTALL_REQUIRES = (
-    'grpcio>=0.11.0b0',
+    'grpcio>=0.13.1',
 )
 
-_SETUP_REQUIRES = _INSTALL_REQUIRES
-
 _COMMAND_CLASS = {
-    'build_proto_modules': commands.BuildProtoModules,
-    'build_py': commands.BuildPy,
+    'copy_proto_modules': health_commands.CopyProtoModules,
+    'build_proto_modules': health_commands.BuildProtoModules,
+    'build_py': health_commands.BuildPy,
+    'sdist': health_commands.SDist,
 }
 
 setuptools.setup(
     name='grpcio_health_checking',
-    version='0.11.0b0',
+    version='0.14.0b0',
     packages=list(_PACKAGES),
     package_dir=_PACKAGE_DIRECTORIES,
     install_requires=_INSTALL_REQUIRES,
-    setup_requires=_SETUP_REQUIRES,
     cmdclass=_COMMAND_CLASS
 )
diff --git a/src/ruby/bin/math_services.rb b/src/ruby/bin/math_services.rb
index 2d482129c2..34c36abdda 100755
--- a/src/ruby/bin/math_services.rb
+++ b/src/ruby/bin/math_services.rb
@@ -1,13 +1,41 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: math.proto for package 'math'
+# Original file comments:
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
 
 require 'grpc'
 require 'math'
 
 module Math
   module Math
-
-    # TODO: add proto service documentation here
     class Service
 
       include GRPC::GenericService
@@ -16,9 +44,20 @@ module Math
       self.unmarshal_class_method = :decode
       self.service_name = 'math.Math'
 
+      # Div divides args.dividend by args.divisor and returns the quotient and
+      # remainder.
       rpc :Div, DivArgs, DivReply
+      # DivMany accepts an arbitrary number of division args from the client stream
+      # and sends back the results in the reply stream.  The stream continues until
+      # the client closes its end; the server does the same after sending all the
+      # replies.  The stream ends immediately if either end aborts.
       rpc :DivMany, stream(DivArgs), stream(DivReply)
+      # Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+      # generates up to limit numbers; otherwise it continues until the call is
+      # canceled.  Unlike Fib above, Fib has no final FibReply.
       rpc :Fib, FibArgs, stream(Num)
+      # Sum sums a stream of numbers, returning the final result once the stream
+      # is closed.
       rpc :Sum, stream(Num), Num
     end
 
diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c
index 1b06273af9..b436057c16 100644
--- a/src/ruby/ext/grpc/rb_call.c
+++ b/src/ruby/ext/grpc/rb_call.c
@@ -101,30 +101,14 @@ static VALUE sym_message;
 static VALUE sym_status;
 static VALUE sym_cancelled;
 
-/* hash_all_calls is a hash of Call address -> reference count that is used to
- * track the creation and destruction of rb_call instances.
- */
-static VALUE hash_all_calls;
-
 /* Destroys a Call. */
 static void grpc_rb_call_destroy(void *p) {
-  grpc_call *call = NULL;
-  VALUE ref_count = Qnil;
+  grpc_call* call = NULL;
   if (p == NULL) {
     return;
-  };
-  call = (grpc_call *)p;
-
-  ref_count = rb_hash_aref(hash_all_calls, OFFT2NUM((VALUE)call));
-  if (ref_count == Qnil) {
-    return; /* No longer in the hash, so already deleted */
-  } else if (NUM2UINT(ref_count) == 1) {
-    rb_hash_delete(hash_all_calls, OFFT2NUM((VALUE)call));
-    grpc_call_destroy(call);
-  } else {
-    rb_hash_aset(hash_all_calls, OFFT2NUM((VALUE)call),
-                 UINT2NUM(NUM2UINT(ref_count) - 1));
   }
+  call = (grpc_call *)p;
+  grpc_call_destroy(call);
 }
 
 static size_t md_ary_datasize(const void *p) {
@@ -151,7 +135,7 @@ static const rb_data_type_t grpc_rb_md_ary_data_type = {
      * touches a hash object.
      * TODO(yugui) Directly use st_table and call the free function earlier?
      */
-    0,
+     0,
 #endif
 };
 
@@ -163,12 +147,7 @@ static const rb_data_type_t grpc_call_data_type = {
     NULL,
     NULL,
 #ifdef RUBY_TYPED_FREE_IMMEDIATELY
-    /* it is unsafe to specify RUBY_TYPED_FREE_IMMEDIATELY because
-     * grpc_rb_call_destroy
-     * touches a hash object.
-     * TODO(yugui) Directly use st_table and call the free function earlier?
-     */
-    0,
+    RUBY_TYPED_FREE_IMMEDIATELY
 #endif
 };
 
@@ -190,6 +169,11 @@ const char *grpc_call_error_detail_of(grpc_call_error err) {
 static VALUE grpc_rb_call_cancel(VALUE self) {
   grpc_call *call = NULL;
   grpc_call_error err;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    //This call has been closed
+    return Qnil;
+  }
+
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
   err = grpc_call_cancel(call, NULL);
   if (err != GRPC_CALL_OK) {
@@ -200,11 +184,29 @@ static VALUE grpc_rb_call_cancel(VALUE self) {
   return Qnil;
 }
 
+/* Releases the c-level resources associated with a call
+   Once a call has been closed, no further requests can be
+   processed.
+*/
+static VALUE grpc_rb_call_close(VALUE self) {
+  grpc_call *call = NULL;
+  TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
+  if(call != NULL) {
+    grpc_call_destroy(call);
+    RTYPEDDATA_DATA(self) = NULL;
+  }
+  return Qnil;
+}
+
 /* Called to obtain the peer that this call is connected to. */
 static VALUE grpc_rb_call_get_peer(VALUE self) {
   VALUE res = Qnil;
   grpc_call *call = NULL;
   char *peer = NULL;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot get peer value on closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
   peer = grpc_call_get_peer(call);
   res = rb_str_new2(peer);
@@ -218,6 +220,10 @@ static VALUE grpc_rb_call_get_peer_cert(VALUE self) {
   grpc_call *call = NULL;
   VALUE res = Qnil;
   grpc_auth_context *ctx = NULL;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot get peer cert on closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
 
   ctx = grpc_call_auth_context(call);
@@ -323,6 +329,10 @@ static VALUE grpc_rb_call_set_credentials(VALUE self, VALUE credentials) {
   grpc_call *call = NULL;
   grpc_call_credentials *creds;
   grpc_call_error err;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot set credentials of closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
   creds = grpc_rb_get_wrapped_call_credentials(credentials);
   err = grpc_call_set_credentials(call, creds);
@@ -731,7 +741,7 @@ static VALUE grpc_run_batch_stack_build_result(run_batch_stack *st) {
    }
    tag = Object.new
    timeout = 10
-   call.start_batch(cqueue, tag, timeout, ops)
+   call.start_batch(cq, tag, timeout, ops)
 
    Start a batch of operations defined in the array ops; when complete, post a
    completion of type 'tag' to the completion queue bound to the call.
@@ -749,6 +759,10 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE cqueue, VALUE tag,
   VALUE result = Qnil;
   VALUE rb_write_flag = rb_ivar_get(self, id_write_flag);
   unsigned write_flag = 0;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot run batch on closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
 
   /* Validate the ops args, adding them to a ruby array */
@@ -888,6 +902,7 @@ void Init_grpc_call() {
   /* Add ruby analogues of the Call methods. */
   rb_define_method(grpc_rb_cCall, "run_batch", grpc_rb_call_run_batch, 4);
   rb_define_method(grpc_rb_cCall, "cancel", grpc_rb_call_cancel, 0);
+  rb_define_method(grpc_rb_cCall, "close", grpc_rb_call_close, 0);
   rb_define_method(grpc_rb_cCall, "peer", grpc_rb_call_get_peer, 0);
   rb_define_method(grpc_rb_cCall, "peer_cert", grpc_rb_call_get_peer_cert, 0);
   rb_define_method(grpc_rb_cCall, "status", grpc_rb_call_get_status, 0);
@@ -925,11 +940,6 @@ void Init_grpc_call() {
       "BatchResult", "send_message", "send_metadata", "send_close",
       "send_status", "message", "metadata", "status", "cancelled", NULL);
 
-  /* The hash for reference counting calls, to ensure they can't be destroyed
-   * more than once */
-  hash_all_calls = rb_hash_new();
-  rb_define_const(grpc_rb_cCall, "INTERNAL_ALL_CALLs", hash_all_calls);
-
   Init_grpc_error_codes();
   Init_grpc_op_codes();
   Init_grpc_write_flags();
@@ -944,16 +954,8 @@ grpc_call *grpc_rb_get_wrapped_call(VALUE v) {
 
 /* Obtains the wrapped object for a given call */
 VALUE grpc_rb_wrap_call(grpc_call *c) {
-  VALUE obj = Qnil;
   if (c == NULL) {
     return Qnil;
   }
-  obj = rb_hash_aref(hash_all_calls, OFFT2NUM((VALUE)c));
-  if (obj == Qnil) { /* Not in the hash add it */
-    rb_hash_aset(hash_all_calls, OFFT2NUM((VALUE)c), UINT2NUM(1));
-  } else {
-    rb_hash_aset(hash_all_calls, OFFT2NUM((VALUE)c),
-                 UINT2NUM(NUM2UINT(obj) + 1));
-  }
   return TypedData_Wrap_Struct(grpc_rb_cCall, &grpc_call_data_type, c);
 }
diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c
index 013321ffc8..6943c93d4a 100644
--- a/src/ruby/ext/grpc/rb_channel.c
+++ b/src/ruby/ext/grpc/rb_channel.c
@@ -373,7 +373,7 @@ static void Init_grpc_connectivity_states() {
   rb_define_const(grpc_rb_mConnectivityStates, "TRANSIENT_FAILURE",
                   LONG2NUM(GRPC_CHANNEL_TRANSIENT_FAILURE));
   rb_define_const(grpc_rb_mConnectivityStates, "FATAL_FAILURE",
-                  LONG2NUM(GRPC_CHANNEL_FATAL_FAILURE));
+                  LONG2NUM(GRPC_CHANNEL_SHUTDOWN));
 }
 
 void Init_grpc_channel() {
diff --git a/src/ruby/ext/grpc/rb_completion_queue.c b/src/ruby/ext/grpc/rb_completion_queue.c
index 4bb615f8be..9466402db0 100644
--- a/src/ruby/ext/grpc/rb_completion_queue.c
+++ b/src/ruby/ext/grpc/rb_completion_queue.c
@@ -52,21 +52,41 @@ typedef struct next_call_stack {
   grpc_event event;
   gpr_timespec timeout;
   void *tag;
+  volatile int interrupted;
 } next_call_stack;
 
 /* Calls grpc_completion_queue_next without holding the ruby GIL */
 static void *grpc_rb_completion_queue_next_no_gil(void *param) {
   next_call_stack *const next_call = (next_call_stack*)param;
-  next_call->event =
-      grpc_completion_queue_next(next_call->cq, next_call->timeout, NULL);
+  gpr_timespec increment = gpr_time_from_millis(20, GPR_TIMESPAN);
+  gpr_timespec deadline;
+  do {
+    deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), increment);
+    next_call->event = grpc_completion_queue_next(next_call->cq,
+                                                  deadline, NULL);
+    if (next_call->event.type != GRPC_QUEUE_TIMEOUT ||
+        gpr_time_cmp(deadline, next_call->timeout) > 0) {
+      break;
+    }
+  } while (!next_call->interrupted);
   return NULL;
 }
 
 /* Calls grpc_completion_queue_pluck without holding the ruby GIL */
 static void *grpc_rb_completion_queue_pluck_no_gil(void *param) {
   next_call_stack *const next_call = (next_call_stack*)param;
-  next_call->event = grpc_completion_queue_pluck(next_call->cq, next_call->tag,
-                                                 next_call->timeout, NULL);
+  gpr_timespec increment = gpr_time_from_millis(20, GPR_TIMESPAN);
+  gpr_timespec deadline;
+  do {
+    deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), increment);
+    next_call->event = grpc_completion_queue_pluck(next_call->cq,
+                                                   next_call->tag,
+                                                   deadline, NULL);
+    if (next_call->event.type != GRPC_QUEUE_TIMEOUT ||
+        gpr_time_cmp(deadline, next_call->timeout) > 0) {
+      break;
+    }
+  } while (!next_call->interrupted);
   return NULL;
 }
 
@@ -130,6 +150,14 @@ static rb_data_type_t grpc_rb_completion_queue_data_type = {
 #endif
 };
 
+/* Releases the c-level resources associated with a completion queue */
+static VALUE grpc_rb_completion_queue_close(VALUE self) {
+  grpc_completion_queue* cq = grpc_rb_get_wrapped_completion_queue(self);
+  grpc_rb_completion_queue_destroy(cq);
+  RTYPEDDATA_DATA(self) = NULL;
+  return Qnil;
+}
+
 /* Allocates a completion queue. */
 static VALUE grpc_rb_completion_queue_alloc(VALUE cls) {
   grpc_completion_queue *cq = grpc_completion_queue_create(NULL);
@@ -139,6 +167,11 @@ static VALUE grpc_rb_completion_queue_alloc(VALUE cls) {
   return TypedData_Wrap_Struct(cls, &grpc_rb_completion_queue_data_type, cq);
 }
 
+static void unblock_func(void *param) {
+  next_call_stack *const next_call = (next_call_stack*)param;
+  next_call->interrupted = 1;
+}
+
 /* Blocks until the next event for given tag is available, and returns the
  * event. */
 grpc_event grpc_rb_completion_queue_pluck_event(VALUE self, VALUE tag,
@@ -158,8 +191,23 @@ grpc_event grpc_rb_completion_queue_pluck_event(VALUE self, VALUE tag,
     next_call.tag = ROBJECT(tag);
   }
   next_call.event.type = GRPC_QUEUE_TIMEOUT;
-  rb_thread_call_without_gvl(grpc_rb_completion_queue_pluck_no_gil,
-                             (void *)&next_call, NULL, NULL);
+  /* Loop until we finish a pluck without an interruption. The internal
+     pluck function runs either until it is interrupted or it gets an
+     event, or time runs out.
+
+     The basic reason we need this relatively complicated construction is that
+     we need to re-acquire the GVL when an interrupt comes in, so that the ruby
+     interpreter can do what it needs to do with the interrupt. But we also need
+     to get back to plucking when the interrupt has been handled. */
+  do {
+    next_call.interrupted = 0;
+    rb_thread_call_without_gvl(grpc_rb_completion_queue_pluck_no_gil,
+                               (void *)&next_call, unblock_func,
+                               (void *)&next_call);
+    /* If an interrupt prevented pluck from returning useful information, then
+       any plucks that did complete must have timed out */
+  } while (next_call.interrupted &&
+           next_call.event.type == GRPC_QUEUE_TIMEOUT);
   return next_call.event;
 }
 
@@ -172,6 +220,11 @@ void Init_grpc_completion_queue() {
      this func, so no separate initialization step is necessary. */
   rb_define_alloc_func(grpc_rb_cCompletionQueue,
                        grpc_rb_completion_queue_alloc);
+
+  /* close: Provides a way to close the underlying file descriptor without
+     waiting for ruby garbage collection. */
+  rb_define_method(grpc_rb_cCompletionQueue, "close",
+                   grpc_rb_completion_queue_close, 0);
 }
 
 /* Gets the wrapped completion queue from the ruby wrapper */
diff --git a/src/ruby/ext/grpc/rb_completion_queue.h b/src/ruby/ext/grpc/rb_completion_queue.h
index 6cc4e96589..42de43c3fb 100644
--- a/src/ruby/ext/grpc/rb_completion_queue.h
+++ b/src/ruby/ext/grpc/rb_completion_queue.h
@@ -46,7 +46,7 @@ grpc_completion_queue *grpc_rb_get_wrapped_completion_queue(VALUE v);
  *
  * This avoids having code that holds the GIL repeated at multiple sites.
  */
-grpc_event grpc_rb_completion_queue_pluck_event(VALUE cqueue, VALUE tag,
+grpc_event grpc_rb_completion_queue_pluck_event(VALUE self, VALUE tag,
                                                 VALUE timeout);
 
 /* Initializes the CompletionQueue class. */
diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c
index 5277148fc9..9246893f9f 100644
--- a/src/ruby/ext/grpc/rb_grpc.c
+++ b/src/ruby/ext/grpc/rb_grpc.c
@@ -50,7 +50,6 @@
 #include "rb_loader.h"
 #include "rb_server.h"
 #include "rb_server_credentials.h"
-#include "rb_signal.h"
 
 static VALUE grpc_rb_cTimeVal = Qnil;
 
@@ -319,7 +318,7 @@ void Init_grpc_c() {
   grpc_rb_mGrpcCore = rb_define_module_under(grpc_rb_mGRPC, "Core");
   grpc_rb_sNewServerRpc =
       rb_struct_define("NewServerRpc", "method", "host",
-                       "deadline", "metadata", "call", NULL);
+                       "deadline", "metadata", "call", "cq", NULL);
   grpc_rb_sStatus =
       rb_struct_define("Status", "code", "details", "metadata", NULL);
   sym_code = ID2SYM(rb_intern("code"));
@@ -333,7 +332,6 @@ void Init_grpc_c() {
   Init_grpc_channel_credentials();
   Init_grpc_server();
   Init_grpc_server_credentials();
-  Init_grpc_signals();
   Init_grpc_status_codes();
   Init_grpc_time_consts();
 }
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
index bc43f9d36b..9748cb576b 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include "rb_grpc_imports.generated.h"
 
@@ -115,6 +115,7 @@ grpc_server_register_method_type grpc_server_register_method_import;
 grpc_server_request_registered_call_type grpc_server_request_registered_call_import;
 grpc_server_create_type grpc_server_create_import;
 grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
+grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
 grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 grpc_server_start_type grpc_server_start_import;
 grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import;
@@ -125,6 +126,9 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import;
 grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import;
 grpc_is_binary_header_type grpc_is_binary_header_import;
 grpc_call_error_to_string_type grpc_call_error_to_string_import;
+grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import;
+grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import;
+grpc_use_signal_type grpc_use_signal_import;
 grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import;
 grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import;
 grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import;
@@ -257,6 +261,8 @@ gpr_avl_unref_type gpr_avl_unref_import;
 gpr_avl_add_type gpr_avl_add_import;
 gpr_avl_remove_type gpr_avl_remove_import;
 gpr_avl_get_type gpr_avl_get_import;
+gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
+gpr_avl_is_empty_type gpr_avl_is_empty_import;
 gpr_cmdline_create_type gpr_cmdline_create_import;
 gpr_cmdline_add_int_type gpr_cmdline_add_int_import;
 gpr_cmdline_add_flag_type gpr_cmdline_add_flag_import;
@@ -381,6 +387,7 @@ void grpc_rb_load_imports(HMODULE library) {
   grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call");
   grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create");
   grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue");
+  grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue");
   grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port");
   grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start");
   grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify");
@@ -391,6 +398,9 @@ void grpc_rb_load_imports(HMODULE library) {
   grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal");
   grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header");
   grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string");
+  grpc_insecure_channel_create_from_fd_import = (grpc_insecure_channel_create_from_fd_type) GetProcAddress(library, "grpc_insecure_channel_create_from_fd");
+  grpc_server_add_insecure_channel_from_fd_import = (grpc_server_add_insecure_channel_from_fd_type) GetProcAddress(library, "grpc_server_add_insecure_channel_from_fd");
+  grpc_use_signal_import = (grpc_use_signal_type) GetProcAddress(library, "grpc_use_signal");
   grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next");
   grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator");
   grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity");
@@ -523,6 +533,8 @@ void grpc_rb_load_imports(HMODULE library) {
   gpr_avl_add_import = (gpr_avl_add_type) GetProcAddress(library, "gpr_avl_add");
   gpr_avl_remove_import = (gpr_avl_remove_type) GetProcAddress(library, "gpr_avl_remove");
   gpr_avl_get_import = (gpr_avl_get_type) GetProcAddress(library, "gpr_avl_get");
+  gpr_avl_maybe_get_import = (gpr_avl_maybe_get_type) GetProcAddress(library, "gpr_avl_maybe_get");
+  gpr_avl_is_empty_import = (gpr_avl_is_empty_type) GetProcAddress(library, "gpr_avl_is_empty");
   gpr_cmdline_create_import = (gpr_cmdline_create_type) GetProcAddress(library, "gpr_cmdline_create");
   gpr_cmdline_add_int_import = (gpr_cmdline_add_int_type) GetProcAddress(library, "gpr_cmdline_add_int");
   gpr_cmdline_add_flag_import = (gpr_cmdline_add_flag_type) GetProcAddress(library, "gpr_cmdline_add_flag");
@@ -569,4 +581,4 @@ void grpc_rb_load_imports(HMODULE library) {
   gpr_thd_join_import = (gpr_thd_join_type) GetProcAddress(library, "gpr_thd_join");
 }
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
index b67361ca25..13f961495c 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -36,13 +36,14 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WIN32
+#ifdef GPR_WINDOWS
 
 #include <windows.h>
 
 #include <grpc/census.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
+#include <grpc/grpc_posix.h>
 #include <grpc/grpc_security.h>
 #include <grpc/impl/codegen/alloc.h>
 #include <grpc/impl/codegen/byte_buffer.h>
@@ -56,7 +57,7 @@
 #include <grpc/support/cpu.h>
 #include <grpc/support/histogram.h>
 #include <grpc/support/host_port.h>
-#include <grpc/support/log_win32.h>
+#include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
 #include <grpc/support/thd.h>
@@ -295,6 +296,9 @@ extern grpc_server_create_type grpc_server_create_import;
 typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved);
 extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
 #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import
+typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved);
+extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
+#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import
 typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr);
 extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import
@@ -325,6 +329,15 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import;
 typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error);
 extern grpc_call_error_to_string_type grpc_call_error_to_string_import;
 #define grpc_call_error_to_string grpc_call_error_to_string_import
+typedef grpc_channel *(*grpc_insecure_channel_create_from_fd_type)(const char *target, int fd, const grpc_channel_args *args);
+extern grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import;
+#define grpc_insecure_channel_create_from_fd grpc_insecure_channel_create_from_fd_import
+typedef void(*grpc_server_add_insecure_channel_from_fd_type)(grpc_server *server, grpc_completion_queue *cq, int fd);
+extern grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import;
+#define grpc_server_add_insecure_channel_from_fd grpc_server_add_insecure_channel_from_fd_import
+typedef void(*grpc_use_signal_type)(int signum);
+extern grpc_use_signal_type grpc_use_signal_import;
+#define grpc_use_signal grpc_use_signal_import
 typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it);
 extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import;
 #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import
@@ -472,7 +485,7 @@ extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_impo
 typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader);
 extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
 #define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import
-typedef void(*gpr_log_type)(const char *file, int line, gpr_log_severity severity, const char *format, ...);
+typedef void(*gpr_log_type)(const char *file, int line, gpr_log_severity severity, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(4, 5);
 extern gpr_log_type gpr_log_import;
 #define gpr_log gpr_log_import
 typedef void(*gpr_log_message_type)(const char *file, int line, gpr_log_severity severity, const char *message);
@@ -721,6 +734,12 @@ extern gpr_avl_remove_type gpr_avl_remove_import;
 typedef void *(*gpr_avl_get_type)(gpr_avl avl, void *key);
 extern gpr_avl_get_type gpr_avl_get_import;
 #define gpr_avl_get gpr_avl_get_import
+typedef int(*gpr_avl_maybe_get_type)(gpr_avl avl, void *key, void **value);
+extern gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
+#define gpr_avl_maybe_get gpr_avl_maybe_get_import
+typedef int(*gpr_avl_is_empty_type)(gpr_avl avl);
+extern gpr_avl_is_empty_type gpr_avl_is_empty_import;
+#define gpr_avl_is_empty gpr_avl_is_empty_import
 typedef gpr_cmdline *(*gpr_cmdline_create_type)(const char *description);
 extern gpr_cmdline_create_type gpr_cmdline_create_import;
 #define gpr_cmdline_create gpr_cmdline_create_import
@@ -811,7 +830,7 @@ extern gpr_format_message_type gpr_format_message_import;
 typedef char *(*gpr_strdup_type)(const char *src);
 extern gpr_strdup_type gpr_strdup_import;
 #define gpr_strdup gpr_strdup_import
-typedef int(*gpr_asprintf_type)(char **strp, const char *format, ...);
+typedef int(*gpr_asprintf_type)(char **strp, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(2, 3);
 extern gpr_asprintf_type gpr_asprintf_import;
 #define gpr_asprintf gpr_asprintf_import
 typedef const char *(*gpr_subprocess_binary_extension_type)();
@@ -856,6 +875,6 @@ extern gpr_thd_join_type gpr_thd_join_import;
 
 void grpc_rb_load_imports(HMODULE library);
 
-#endif /* GPR_WIN32 */
+#endif /* GPR_WINDOWS */
 
 #endif
diff --git a/src/ruby/ext/grpc/rb_loader.c b/src/ruby/ext/grpc/rb_loader.c
index 242535f164..19a6b33c29 100644
--- a/src/ruby/ext/grpc/rb_loader.c
+++ b/src/ruby/ext/grpc/rb_loader.c
@@ -33,7 +33,7 @@
 
 #include "rb_grpc_imports.generated.h"
 
-#if GPR_WIN32
+#if GPR_WINDOWS
 #include <tchar.h>
 
 int grpc_rb_load_core() {
diff --git a/src/ruby/ext/grpc/rb_server.c b/src/ruby/ext/grpc/rb_server.c
index 2b3acaaf59..f108b8acfc 100644
--- a/src/ruby/ext/grpc/rb_server.c
+++ b/src/ruby/ext/grpc/rb_server.c
@@ -60,6 +60,7 @@ typedef struct grpc_rb_server {
   VALUE mark;
   /* The actual server */
   grpc_server *wrapped;
+  grpc_completion_queue *queue;
 } grpc_rb_server;
 
 /* Destroys server instances. */
@@ -145,6 +146,7 @@ static VALUE grpc_rb_server_init(VALUE self, VALUE cqueue, VALUE channel_args) {
   }
   grpc_server_register_completion_queue(srv, cq, NULL);
   wrapper->wrapped = srv;
+  wrapper->queue = cq;
 
   /* Add the cq as the server's mark object. This ensures the ruby cq can't be
      GCed before the server */
@@ -232,7 +234,7 @@ static VALUE grpc_rb_server_request_call(VALUE self, VALUE cqueue,
     err = grpc_server_request_call(
         s->wrapped, &call, &st.details, &st.md_ary,
         grpc_rb_get_wrapped_completion_queue(cqueue),
-        grpc_rb_get_wrapped_completion_queue(cqueue),
+        grpc_rb_get_wrapped_completion_queue(s->mark),
         ROBJECT(tag_new));
     if (err != GRPC_CALL_OK) {
       grpc_request_call_stack_cleanup(&st);
@@ -242,7 +244,7 @@ static VALUE grpc_rb_server_request_call(VALUE self, VALUE cqueue,
       return Qnil;
     }
 
-    ev = grpc_rb_completion_queue_pluck_event(cqueue, tag_new, timeout);
+    ev = grpc_rb_completion_queue_pluck_event(s->mark, tag_new, timeout);
     if (ev.type == GRPC_QUEUE_TIMEOUT) {
       grpc_request_call_stack_cleanup(&st);
       return Qnil;
@@ -260,7 +262,7 @@ static VALUE grpc_rb_server_request_call(VALUE self, VALUE cqueue,
         rb_str_new2(st.details.host),
         rb_funcall(rb_cTime, id_at, 2, INT2NUM(deadline.tv_sec),
                    INT2NUM(deadline.tv_nsec)),
-        grpc_rb_md_ary_to_h(&st.md_ary), grpc_rb_wrap_call(call), NULL);
+        grpc_rb_md_ary_to_h(&st.md_ary), grpc_rb_wrap_call(call), cqueue, NULL);
     grpc_request_call_stack_cleanup(&st);
     return result;
   }
diff --git a/src/ruby/lib/grpc.rb b/src/ruby/lib/grpc.rb
index 7c9aae30e9..79fa705b1c 100644
--- a/src/ruby/lib/grpc.rb
+++ b/src/ruby/lib/grpc.rb
@@ -33,7 +33,6 @@ require_relative 'grpc/errors'
 require_relative 'grpc/grpc'
 require_relative 'grpc/logconfig'
 require_relative 'grpc/notifier'
-require_relative 'grpc/signals'
 require_relative 'grpc/version'
 require_relative 'grpc/core/time_consts'
 require_relative 'grpc/generic/active_call'
@@ -48,5 +47,3 @@ begin
 ensure
   file.close
 end
-
-GRPC::Signals.wait_for_signals
diff --git a/src/ruby/lib/grpc/errors.rb b/src/ruby/lib/grpc/errors.rb
index a1dd1e3e9d..23b2bb7e12 100644
--- a/src/ruby/lib/grpc/errors.rb
+++ b/src/ruby/lib/grpc/errors.rb
@@ -40,11 +40,12 @@ module GRPC
 
     # @param code [Numeric] the status code
     # @param details [String] the details of the exception
-    def initialize(code, details = 'unknown cause', **kw)
+    # @param metadata [Hash] the error's metadata
+    def initialize(code, details = 'unknown cause', metadata = {})
       super("#{code}:#{details}")
       @code = code
       @details = details
-      @metadata = kw
+      @metadata = metadata
     end
 
     # Converts the exception to a GRPC::Status for use in the networking
diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb
index fd20a86144..b03ddbc193 100644
--- a/src/ruby/lib/grpc/generic/active_call.rb
+++ b/src/ruby/lib/grpc/generic/active_call.rb
@@ -30,7 +30,6 @@
 require 'forwardable'
 require 'weakref'
 require_relative 'bidi_call'
-require_relative '../signals'
 
 class Struct
   # BatchResult is the struct returned by calls to call#start_batch.
@@ -45,7 +44,7 @@ class Struct
         # raise BadStatus, propagating the metadata if present.
         md = status.metadata
         with_sym_keys = Hash[md.each_pair.collect { |x, y| [x.to_sym, y] }]
-        fail GRPC::BadStatus.new(status.code, status.details, **with_sym_keys)
+        fail GRPC::BadStatus.new(status.code, status.details, with_sym_keys)
       end
       status
     end
@@ -77,14 +76,15 @@ module GRPC
     #
     # @param call [Call] a call on which to start and invocation
     # @param q [CompletionQueue] the completion queue
-    def self.client_invoke(call, q, **kw)
+    # @param metadata [Hash] the metadata
+    def self.client_invoke(call, q, metadata = {})
       fail(TypeError, '!Core::Call') unless call.is_a? Core::Call
       unless q.is_a? Core::CompletionQueue
         fail(TypeError, '!Core::CompletionQueue')
       end
       metadata_tag = Object.new
       call.run_batch(q, metadata_tag, INFINITE_FUTURE,
-                     SEND_INITIAL_METADATA => kw)
+                     SEND_INITIAL_METADATA => metadata)
       metadata_tag
     end
 
@@ -103,7 +103,7 @@ module GRPC
     #
     # @param call [Call] the call used by the ActiveCall
     # @param q [CompletionQueue] the completion queue used to accept
-    #          the call
+    #          the call.  This queue will be closed on call completion.
     # @param marshal [Function] f(obj)->string that marshal requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
     # @param deadline [Fixnum] the deadline for the call to complete
@@ -123,10 +123,6 @@ module GRPC
       @unmarshal = unmarshal
       @metadata_tag = metadata_tag
       @op_notifier = nil
-      weak_self = WeakRef.new(self)
-      remove_handler = GRPC::Signals.register_handler(&weak_self
-                                                        .method(:cancel))
-      ObjectSpace.define_finalizer(self, remove_handler)
     end
 
     # output_metadata are provides access to hash that can be used to
@@ -195,6 +191,8 @@ module GRPC
       @call.status = batch_result.status
       op_is_done
       batch_result.check_status
+      @call.close
+      @cq.close
     end
 
     # remote_send sends a request to the remote endpoint.
@@ -216,13 +214,12 @@ module GRPC
     # @param details [String] details
     # @param assert_finished [true, false] when true(default), waits for
     # FINISHED.
-    #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    def send_status(code = OK, details = '', assert_finished = false, **kw)
+    # @param metadata [Hash] metadata to send to the server. If a value is a
+    # list, mulitple metadata for its key are sent
+    def send_status(code = OK, details = '', assert_finished = false,
+                    metadata: {})
       ops = {
-        SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, kw)
+        SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, metadata)
       }
       ops[RECV_CLOSE_ON_SERVER] = nil if assert_finished
       @call.run_batch(@cq, self, INFINITE_FUTURE, ops)
@@ -316,14 +313,12 @@ module GRPC
     # request_response sends a request to a GRPC server, and returns the
     # response.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param req [Object] the request sent to the server
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Object] the response received from the server
-    def request_response(req, **kw)
-      start_call(**kw) unless @started
+    def request_response(req, metadata: {})
+      start_call(metadata) unless @started
       remote_send(req)
       writes_done(false)
       response = remote_read
@@ -342,14 +337,12 @@ module GRPC
     # array of marshallable objects; in typical case it will be an Enumerable
     # that allows dynamic construction of the marshallable objects.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param requests [Object] an Enumerable of requests to send
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Object] the response received from the server
-    def client_streamer(requests, **kw)
-      start_call(**kw) unless @started
+    def client_streamer(requests, metadata: {})
+      start_call(metadata) unless @started
       requests.each { |r| remote_send(r) }
       writes_done(false)
       response = remote_read
@@ -370,15 +363,12 @@ module GRPC
     # it is executed with each response as the argument and no result is
     # returned.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    # any keyword arguments are treated as metadata to be sent to the server.
-    #
     # @param req [Object] the request sent to the server
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Enumerator|nil] a response Enumerator
-    def server_streamer(req, **kw)
-      start_call(**kw) unless @started
+    def server_streamer(req, metadata: {})
+      start_call(metadata) unless @started
       remote_send(req)
       writes_done(false)
       replies = enum_for(:each_remote_read_then_finish)
@@ -412,14 +402,12 @@ module GRPC
     # the_call#writes_done has been called, otherwise the block will loop
     # forever.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param requests [Object] an Enumerable of requests to send
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Enumerator, nil] a response Enumerator
-    def bidi_streamer(requests, **kw, &blk)
-      start_call(**kw) unless @started
+    def bidi_streamer(requests, metadata: {}, &blk)
+      start_call(metadata) unless @started
       bd = BidiCall.new(@call, @cq, @marshal, @unmarshal,
                         metadata_tag: @metadata_tag)
       @metadata_tag = nil  # run_on_client ensures metadata is read
@@ -458,9 +446,11 @@ module GRPC
     private
 
     # Starts the call if not already started
-    def start_call(**kw)
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
+    def start_call(metadata = {})
       return if @started
-      @metadata_tag = ActiveCall.client_invoke(@call, @cq, **kw)
+      @metadata_tag = ActiveCall.client_invoke(@call, @cq, metadata)
       @started = true
     end
 
diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb
index 1f6d5f365d..238f409a1d 100644
--- a/src/ruby/lib/grpc/generic/bidi_call.rb
+++ b/src/ruby/lib/grpc/generic/bidi_call.rb
@@ -69,6 +69,10 @@ module GRPC
       @readq = Queue.new
       @unmarshal = unmarshal
       @metadata_tag = metadata_tag
+      @reads_complete = false
+      @writes_complete = false
+      @complete = false
+      @done_mutex = Mutex.new
     end
 
     # Begins orchestration of the Bidi stream for a client sending requests.
@@ -115,6 +119,16 @@ module GRPC
       @op_notifier.notify(self)
     end
 
+    # signals that a bidi operation is complete (read + write)
+    def finished
+      @done_mutex.synchronize do
+        return unless @reads_complete && @writes_complete && !@complete
+        @call.close
+        @cq.close
+        @complete = true
+      end
+    end
+
     # performs a read using @call.run_batch, ensures metadata is set up
     def read_using_run_batch
       ops = { RECV_MESSAGE => nil }
@@ -163,12 +177,16 @@ module GRPC
                         SEND_CLOSE_FROM_CLIENT => nil)
         GRPC.logger.debug('bidi-write-loop: done')
         notify_done
+        @writes_complete = true
+        finished
       end
       GRPC.logger.debug('bidi-write-loop: finished')
     rescue StandardError => e
       GRPC.logger.warn('bidi-write-loop: failed')
       GRPC.logger.warn(e)
       notify_done
+      @writes_complete = true
+      finished
       raise e
     end
 
@@ -212,6 +230,8 @@ module GRPC
           @readq.push(e)  # let each_queued_msg terminate with this error
         end
         GRPC.logger.debug('bidi-read-loop: finished')
+        @reads_complete = true
+        finished
       end
     end
   end
diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb
index 12946fe819..cddca13d17 100644
--- a/src/ruby/lib/grpc/generic/client_stub.rb
+++ b/src/ruby/lib/grpc/generic/client_stub.rb
@@ -44,21 +44,21 @@ module GRPC
 
     # setup_channel is used by #initialize to constuct a channel from its
     # arguments.
-    def self.setup_channel(alt_chan, host, creds, **kw)
+    def self.setup_channel(alt_chan, host, creds, channel_args = {})
       unless alt_chan.nil?
         fail(TypeError, '!Channel') unless alt_chan.is_a?(Core::Channel)
         return alt_chan
       end
-      if kw['grpc.primary_user_agent'].nil?
-        kw['grpc.primary_user_agent'] = ''
+      if channel_args['grpc.primary_user_agent'].nil?
+        channel_args['grpc.primary_user_agent'] = ''
       else
-        kw['grpc.primary_user_agent'] += ' '
+        channel_args['grpc.primary_user_agent'] += ' '
       end
-      kw['grpc.primary_user_agent'] += "grpc-ruby/#{VERSION}"
+      channel_args['grpc.primary_user_agent'] += "grpc-ruby/#{VERSION}"
       unless creds.is_a?(Core::ChannelCredentials) || creds.is_a?(Symbol)
         fail(TypeError, '!ChannelCredentials or Symbol')
       end
-      Core::Channel.new(host, kw, creds)
+      Core::Channel.new(host, channel_args, creds)
     end
 
     # Allows users of the stub to modify the propagate mask.
@@ -96,15 +96,16 @@ module GRPC
     #     :this_channel_is_insecure
     # @param channel_override [Core::Channel] a pre-created channel
     # @param timeout [Number] the default timeout to use in requests
-    # @param kw [KeywordArgs]the channel arguments
+    # @param channel_args [Hash] the channel arguments
     def initialize(host, q, creds,
                    channel_override: nil,
                    timeout: nil,
                    propagate_mask: nil,
-                   **kw)
+                   channel_args: {})
       fail(TypeError, '!CompletionQueue') unless q.is_a?(Core::CompletionQueue)
-      @ch = ClientStub.setup_channel(channel_override, host, creds, **kw)
-      alt_host = kw[Core::Channel::SSL_TARGET]
+      @ch = ClientStub.setup_channel(channel_override, host, creds,
+                                     channel_args)
+      alt_host = channel_args[Core::Channel::SSL_TARGET]
       @host = alt_host.nil? ? host : alt_host
       @propagate_mask = propagate_mask
       @timeout = timeout.nil? ? DEFAULT_TIMEOUT : timeout
@@ -135,42 +136,35 @@ module GRPC
     # If return_op is true, the call returns an Operation, calling execute
     # on the Operation returns the response.
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # @param method [String] the RPC method to call on the GRPC server
     # @param req [Object] the request sent to the server
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
+    # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
-    # @param return_op [true|false] return an Operation if true
+    # @param metadata [Hash] metadata to be sent to the server
     # @return [Object] the response received from the server
     def request_response(method, req, marshal, unmarshal,
                          deadline: nil,
-                         timeout: nil,
                          return_op: false,
                          parent: nil,
                          credentials: nil,
-                         **kw)
+                         metadata: {})
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.request_response(req, **kw) unless return_op
+      return c.request_response(req, metadata: metadata) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #request_response.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.request_response(req, **kw)
+        c.request_response(req, metadata: metadata)
       end
       op
     end
@@ -205,42 +199,35 @@ module GRPC
     #
     # If return_op is true, the call returns the response.
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # @param method [String] the RPC method to call on the GRPC server
     # @param requests [Object] an Enumerable of requests to send
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
+    # @param metadata [Hash] metadata to be sent to the server
     # @return [Object|Operation] the response received from the server
     def client_streamer(method, requests, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         return_op: false,
                         parent: nil,
                         credentials: nil,
-                        **kw)
+                        metadata: {})
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.client_streamer(requests, **kw) unless return_op
+      return c.client_streamer(requests, metadata: metadata) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #client_streamer.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.client_streamer(requests, **kw)
+        c.client_streamer(requests, metadata: metadata)
       end
       op
     end
@@ -292,35 +279,33 @@ module GRPC
     # @param req [Object] the request sent to the server
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param return_op [true|false]return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
+    # @param metadata [Hash] metadata to be sent to the server
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|Operation|nil] as discussed above
     def server_streamer(method, req, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         return_op: false,
                         parent: nil,
                         credentials: nil,
-                        **kw,
+                        metadata: {},
                         &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.server_streamer(req, **kw, &blk) unless return_op
+      return c.server_streamer(req, metadata: metadata, &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #server_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.server_streamer(req, **kw, &blk)
+        c.server_streamer(req, metadata: metadata, &blk)
       end
       op
     end
@@ -391,11 +376,6 @@ module GRPC
     # * the deadline is exceeded
     #
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # == Return Value ==
     #
     # if the return_op is false, the return value is an Enumerator of the
@@ -411,36 +391,35 @@ module GRPC
     # @param requests [Object] an Enumerable of requests to send
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
     # @param return_op [true|false] return an Operation if true
+    # @param metadata [Hash] metadata to be sent to the server
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|nil|Operation] as discussed above
     def bidi_streamer(method, requests, marshal, unmarshal,
                       deadline: nil,
-                      timeout: nil,
                       return_op: false,
                       parent: nil,
                       credentials: nil,
-                      **kw,
+                      metadata: {},
                       &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
 
-      return c.bidi_streamer(requests, **kw, &blk) unless return_op
+      return c.bidi_streamer(requests, metadata: metadata,
+                             &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #bidi_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.bidi_streamer(requests, **kw, &blk)
+        c.bidi_streamer(requests, metadata: metadata, &blk)
       end
       op
     end
@@ -457,12 +436,10 @@ module GRPC
     # @param timeout [TimeConst]
     def new_active_call(method, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         parent: nil,
                         credentials: nil)
-      if deadline.nil?
-        deadline = from_relative_time(timeout.nil? ? @timeout : timeout)
-      end
+
+      deadline = from_relative_time(@timeout) if deadline.nil?
       # Provide each new client call with its own completion queue
       call_queue = Core::CompletionQueue.new
       call = @ch.create_call(call_queue,
diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb
index cc21ffd3c5..913f55d0d3 100644
--- a/src/ruby/lib/grpc/generic/rpc_desc.rb
+++ b/src/ruby/lib/grpc/generic/rpc_desc.rb
@@ -80,12 +80,12 @@ module GRPC
       else  # is a bidi_stream
         active_call.run_server_bidi(mth)
       end
-      send_status(active_call, OK, 'OK', **active_call.output_metadata)
+      send_status(active_call, OK, 'OK', active_call.output_metadata)
     rescue BadStatus => e
       # this is raised by handlers that want GRPC to send an application error
       # code and detail message and some additional app-specific metadata.
       GRPC.logger.debug("app err:#{active_call}, status:#{e.code}:#{e.details}")
-      send_status(active_call, e.code, e.details, **e.metadata)
+      send_status(active_call, e.code, e.details, e.metadata)
     rescue Core::CallError => e
       # This is raised by GRPC internals but should rarely, if ever happen.
       # Log it, but don't notify the other endpoint..
@@ -135,10 +135,10 @@ module GRPC
       "##{mth.name}: bad arg count; got:#{mth.arity}, want:#{want}, #{msg}"
     end
 
-    def send_status(active_client, code, details, **kw)
+    def send_status(active_client, code, details, metadata = {})
       details = 'Not sure why' if details.nil?
       GRPC.logger.debug("Sending status  #{code}:#{details}")
-      active_client.send_status(code, details, code == OK, **kw)
+      active_client.send_status(code, details, code == OK, metadata: metadata)
     rescue StandardError => e
       GRPC.logger.warn("Could not send status #{code}:#{details}")
       GRPC.logger.warn(e)
diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb
index 238aaa9656..ab7333d133 100644
--- a/src/ruby/lib/grpc/generic/rpc_server.rb
+++ b/src/ruby/lib/grpc/generic/rpc_server.rb
@@ -28,7 +28,6 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 require_relative '../grpc'
-require_relative '../signals'
 require_relative 'active_call'
 require_relative 'service'
 require 'thread'
@@ -170,14 +169,6 @@ module GRPC
       alt_cq
     end
 
-    # setup_srv is used by #initialize to constuct a Core::Server from its
-    # arguments.
-    def self.setup_srv(alt_srv, cq, **kw)
-      return Core::Server.new(cq, kw) if alt_srv.nil?
-      fail(TypeError, '!Server') unless alt_srv.is_a? Core::Server
-      alt_srv
-    end
-
     # setup_connect_md_proc is used by #initialize to validate the
     # connect_md_proc.
     def self.setup_connect_md_proc(a_proc)
@@ -194,9 +185,6 @@ module GRPC
     # instance, however other arbitrary are allowed and when present are used
     # to configure the listeninng connection set up by the RpcServer.
     #
-    # * server_override: which if passed must be a [GRPC::Core::Server].  When
-    # present.
-    #
     # * poll_period: when present, the server polls for new events with this
     # period
     #
@@ -218,13 +206,15 @@ module GRPC
     # when non-nil is a proc for determining metadata to to send back the client
     # on receiving an invocation req.  The proc signature is:
     # {key: val, ..} func(method_name, {key: val, ...})
+    #
+    # * server_args:
+    # A server arguments hash to be passed down to the underlying core server
     def initialize(pool_size:DEFAULT_POOL_SIZE,
                    max_waiting_requests:DEFAULT_MAX_WAITING_REQUESTS,
                    poll_period:DEFAULT_POLL_PERIOD,
                    completion_queue_override:nil,
-                   server_override:nil,
                    connect_md_proc:nil,
-                   **kw)
+                   server_args:{})
       @connect_md_proc = RpcServer.setup_connect_md_proc(connect_md_proc)
       @cq = RpcServer.setup_cq(completion_queue_override)
       @max_waiting_requests = max_waiting_requests
@@ -236,7 +226,7 @@ module GRPC
       # running_state can take 4 values: :not_started, :running, :stopping, and
       # :stopped. State transitions can only proceed in that order.
       @running_state = :not_started
-      @server = RpcServer.setup_srv(server_override, @cq, **kw)
+      @server = Core::Server.new(@cq, server_args)
     end
 
     # stops a running server
@@ -353,10 +343,7 @@ module GRPC
         transition_running_state(:running)
         @run_cond.broadcast
       end
-      remove_signal_handler = GRPC::Signals.register_handler { stop }
       loop_handle_server_calls
-      # Remove signal handler when server stops
-      remove_signal_handler.call
     end
 
     alias_method :run_till_terminated, :run
@@ -368,7 +355,7 @@ module GRPC
       return an_rpc if @pool.jobs_waiting <= @max_waiting_requests
       GRPC.logger.warn("NOT AVAILABLE: too many jobs_waiting: #{an_rpc}")
       noop = proc { |x| x }
-      c = ActiveCall.new(an_rpc.call, @cq, noop, noop, an_rpc.deadline)
+      c = ActiveCall.new(an_rpc.call, an_rpc.cq, noop, noop, an_rpc.deadline)
       c.send_status(GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED, '')
       nil
     end
@@ -379,7 +366,7 @@ module GRPC
       return an_rpc if rpc_descs.key?(mth)
       GRPC.logger.warn("UNIMPLEMENTED: #{an_rpc}")
       noop = proc { |x| x }
-      c = ActiveCall.new(an_rpc.call, @cq, noop, noop, an_rpc.deadline)
+      c = ActiveCall.new(an_rpc.call, an_rpc.cq, noop, noop, an_rpc.deadline)
       c.send_status(GRPC::Core::StatusCodes::UNIMPLEMENTED, '')
       nil
     end
@@ -390,7 +377,8 @@ module GRPC
       loop_tag = Object.new
       while running_state == :running
         begin
-          an_rpc = @server.request_call(@cq, loop_tag, INFINITE_FUTURE)
+          comp_queue = Core::CompletionQueue.new
+          an_rpc = @server.request_call(comp_queue, loop_tag, INFINITE_FUTURE)
           break if (!an_rpc.nil?) && an_rpc.call.nil?
           active_call = new_active_server_call(an_rpc)
           unless active_call.nil?
@@ -429,15 +417,16 @@ module GRPC
       unless @connect_md_proc.nil?
         connect_md = @connect_md_proc.call(an_rpc.method, an_rpc.metadata)
       end
-      an_rpc.call.run_batch(@cq, handle_call_tag, INFINITE_FUTURE,
+      an_rpc.call.run_batch(an_rpc.cq, handle_call_tag, INFINITE_FUTURE,
                             SEND_INITIAL_METADATA => connect_md)
+
       return nil unless available?(an_rpc)
       return nil unless implemented?(an_rpc)
 
       # Create the ActiveCall
       GRPC.logger.info("deadline is #{an_rpc.deadline}; (now=#{Time.now})")
       rpc_desc = rpc_descs[an_rpc.method.to_sym]
-      c = ActiveCall.new(an_rpc.call, @cq,
+      c = ActiveCall.new(an_rpc.call, an_rpc.cq,
                          rpc_desc.marshal_proc, rpc_desc.unmarshal_proc(:input),
                          an_rpc.deadline)
       mth = an_rpc.method.to_sym
diff --git a/src/ruby/lib/grpc/generic/service.rb b/src/ruby/lib/grpc/generic/service.rb
index 0a166e823e..f30242ee80 100644
--- a/src/ruby/lib/grpc/generic/service.rb
+++ b/src/ruby/lib/grpc/generic/service.rb
@@ -179,24 +179,24 @@ module GRPC
             unmarshal = desc.unmarshal_proc(:output)
             route = "/#{route_prefix}/#{name}"
             if desc.request_response?
-              define_method(mth_name) do |req, **kw|
+              define_method(mth_name) do |req, metadata = {}|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                request_response(route, req, marshal, unmarshal, **kw)
+                request_response(route, req, marshal, unmarshal, metadata)
               end
             elsif desc.client_streamer?
-              define_method(mth_name) do |reqs, **kw|
+              define_method(mth_name) do |reqs, metadata = {}|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                client_streamer(route, reqs, marshal, unmarshal, **kw)
+                client_streamer(route, reqs, marshal, unmarshal, metadata)
               end
             elsif desc.server_streamer?
-              define_method(mth_name) do |req, **kw, &blk|
+              define_method(mth_name) do |req, metadata = {}, &blk|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                server_streamer(route, req, marshal, unmarshal, **kw, &blk)
+                server_streamer(route, req, marshal, unmarshal, metadata, &blk)
               end
             else  # is a bidi_stream
-              define_method(mth_name) do |reqs, **kw, &blk|
+              define_method(mth_name) do |reqs, metadata = {}, &blk|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                bidi_streamer(route, reqs, marshal, unmarshal, **kw, &blk)
+                bidi_streamer(route, reqs, marshal, unmarshal, metadata, &blk)
               end
             end
           end
diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb
index e0f1dee596..5e6aaef2eb 100644
--- a/src/ruby/lib/grpc/version.rb
+++ b/src/ruby/lib/grpc/version.rb
@@ -29,5 +29,5 @@
 
 # GRPC contains the General RPC module.
 module GRPC
-  VERSION = '0.14.2.pre1'
+  VERSION = '0.16.0.dev'
 end
diff --git a/src/ruby/pb/grpc/health/v1/health_services.rb b/src/ruby/pb/grpc/health/v1/health_services.rb
index cb79b20437..68a3956f54 100644
--- a/src/ruby/pb/grpc/health/v1/health_services.rb
+++ b/src/ruby/pb/grpc/health/v1/health_services.rb
@@ -1,5 +1,35 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: grpc/health/v1/health.proto for package 'grpc.health.v1'
+# Original file comments:
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
 
 require 'grpc'
 require 'grpc/health/v1/health'
@@ -8,8 +38,6 @@ module Grpc
   module Health
     module V1
       module Health
-
-        # TODO: add proto service documentation here
         class Service
 
           include GRPC::GenericService
diff --git a/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services.rb b/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services.rb
index 9f6e7e0e42..eb523ffa6f 100644
--- a/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services.rb
+++ b/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services.rb
@@ -1,15 +1,45 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
-# Source: src/proto/grpc/testing/duplicate/echo_duplicate.proto for package 'grpc.testing.duplicate'
+# Source: grpc/testing/duplicate/echo_duplicate.proto for package 'grpc.testing.duplicate'
+# Original file comments:
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This is a partial copy of echo.proto with a different package name.
+#
 
 require 'grpc'
-require 'src/proto/grpc/testing/duplicate/echo_duplicate'
+require 'grpc/testing/duplicate/echo_duplicate'
 
 module Grpc
   module Testing
     module Duplicate
       module EchoTestService
-
-        # TODO: add proto service documentation here
         class Service
 
           include GRPC::GenericService
diff --git a/src/ruby/pb/grpc/testing/metrics_services.rb b/src/ruby/pb/grpc/testing/metrics_services.rb
index f5778bbbb1..467b7b3ee5 100644
--- a/src/ruby/pb/grpc/testing/metrics_services.rb
+++ b/src/ruby/pb/grpc/testing/metrics_services.rb
@@ -1,5 +1,41 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: grpc/testing/metrics.proto for package 'grpc.testing'
+# Original file comments:
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Contains the definitions for a metrics service and the type of metrics
+# exposed by the service.
+#
+# Currently, 'Gauge' (i.e a metric that represents the measured value of
+# something at an instant of time) is the only metric type supported by the
+# service.
 
 require 'grpc'
 require 'grpc/testing/metrics'
@@ -7,8 +43,6 @@ require 'grpc/testing/metrics'
 module Grpc
   module Testing
     module MetricsService
-
-      # TODO: add proto service documentation here
       class Service
 
         include GRPC::GenericService
@@ -17,7 +51,10 @@ module Grpc
         self.unmarshal_class_method = :decode
         self.service_name = 'grpc.testing.MetricsService'
 
+        # Returns the values of all the gauges that are currently being maintained by
+        # the service
         rpc :GetAllGauges, EmptyMessage, stream(GaugeResponse)
+        # Returns the value of one gauge
         rpc :GetGauge, GaugeRequest, GaugeResponse
       end
 
diff --git a/src/ruby/pb/src/proto/grpc/testing/empty.rb b/src/ruby/pb/src/proto/grpc/testing/empty.rb
new file mode 100644
index 0000000000..9c2568d605
--- /dev/null
+++ b/src/ruby/pb/src/proto/grpc/testing/empty.rb
@@ -0,0 +1,15 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: src/proto/grpc/testing/empty.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "grpc.testing.Empty" do
+  end
+end
+
+module Grpc
+  module Testing
+    Empty = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.Empty").msgclass
+  end
+end
diff --git a/src/ruby/pb/src/proto/grpc/testing/messages.rb b/src/ruby/pb/src/proto/grpc/testing/messages.rb
new file mode 100644
index 0000000000..2bdfe0eade
--- /dev/null
+++ b/src/ruby/pb/src/proto/grpc/testing/messages.rb
@@ -0,0 +1,84 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: src/proto/grpc/testing/messages.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "grpc.testing.Payload" do
+    optional :type, :enum, 1, "grpc.testing.PayloadType"
+    optional :body, :bytes, 2
+  end
+  add_message "grpc.testing.EchoStatus" do
+    optional :code, :int32, 1
+    optional :message, :string, 2
+  end
+  add_message "grpc.testing.SimpleRequest" do
+    optional :response_type, :enum, 1, "grpc.testing.PayloadType"
+    optional :response_size, :int32, 2
+    optional :payload, :message, 3, "grpc.testing.Payload"
+    optional :fill_username, :bool, 4
+    optional :fill_oauth_scope, :bool, 5
+    optional :response_compression, :enum, 6, "grpc.testing.CompressionType"
+    optional :response_status, :message, 7, "grpc.testing.EchoStatus"
+  end
+  add_message "grpc.testing.SimpleResponse" do
+    optional :payload, :message, 1, "grpc.testing.Payload"
+    optional :username, :string, 2
+    optional :oauth_scope, :string, 3
+  end
+  add_message "grpc.testing.StreamingInputCallRequest" do
+    optional :payload, :message, 1, "grpc.testing.Payload"
+  end
+  add_message "grpc.testing.StreamingInputCallResponse" do
+    optional :aggregated_payload_size, :int32, 1
+  end
+  add_message "grpc.testing.ResponseParameters" do
+    optional :size, :int32, 1
+    optional :interval_us, :int32, 2
+  end
+  add_message "grpc.testing.StreamingOutputCallRequest" do
+    optional :response_type, :enum, 1, "grpc.testing.PayloadType"
+    repeated :response_parameters, :message, 2, "grpc.testing.ResponseParameters"
+    optional :payload, :message, 3, "grpc.testing.Payload"
+    optional :response_compression, :enum, 6, "grpc.testing.CompressionType"
+    optional :response_status, :message, 7, "grpc.testing.EchoStatus"
+  end
+  add_message "grpc.testing.StreamingOutputCallResponse" do
+    optional :payload, :message, 1, "grpc.testing.Payload"
+  end
+  add_message "grpc.testing.ReconnectParams" do
+    optional :max_reconnect_backoff_ms, :int32, 1
+  end
+  add_message "grpc.testing.ReconnectInfo" do
+    optional :passed, :bool, 1
+    repeated :backoff_ms, :int32, 2
+  end
+  add_enum "grpc.testing.PayloadType" do
+    value :COMPRESSABLE, 0
+    value :UNCOMPRESSABLE, 1
+    value :RANDOM, 2
+  end
+  add_enum "grpc.testing.CompressionType" do
+    value :NONE, 0
+    value :GZIP, 1
+    value :DEFLATE, 2
+  end
+end
+
+module Grpc
+  module Testing
+    Payload = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.Payload").msgclass
+    EchoStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.EchoStatus").msgclass
+    SimpleRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.SimpleRequest").msgclass
+    SimpleResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.SimpleResponse").msgclass
+    StreamingInputCallRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.StreamingInputCallRequest").msgclass
+    StreamingInputCallResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.StreamingInputCallResponse").msgclass
+    ResponseParameters = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.ResponseParameters").msgclass
+    StreamingOutputCallRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.StreamingOutputCallRequest").msgclass
+    StreamingOutputCallResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.StreamingOutputCallResponse").msgclass
+    ReconnectParams = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.ReconnectParams").msgclass
+    ReconnectInfo = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.ReconnectInfo").msgclass
+    PayloadType = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.PayloadType").enummodule
+    CompressionType = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.CompressionType").enummodule
+  end
+end
diff --git a/src/ruby/pb/src/proto/grpc/testing/test.rb b/src/ruby/pb/src/proto/grpc/testing/test.rb
new file mode 100644
index 0000000000..245b5ce00c
--- /dev/null
+++ b/src/ruby/pb/src/proto/grpc/testing/test.rb
@@ -0,0 +1,14 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: src/proto/grpc/testing/test.proto
+
+require 'google/protobuf'
+
+require 'src/proto/grpc/testing/empty'
+require 'src/proto/grpc/testing/messages'
+Google::Protobuf::DescriptorPool.generated_pool.build do
+end
+
+module Grpc
+  module Testing
+  end
+end
diff --git a/src/ruby/pb/src/proto/grpc/testing/test_services.rb b/src/ruby/pb/src/proto/grpc/testing/test_services.rb
new file mode 100644
index 0000000000..2652de5e6d
--- /dev/null
+++ b/src/ruby/pb/src/proto/grpc/testing/test_services.rb
@@ -0,0 +1,110 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: src/proto/grpc/testing/test.proto for package 'grpc.testing'
+# Original file comments:
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# An integration test service that covers all the method signature permutations
+# of unary/streaming requests/responses.
+#
+
+require 'grpc'
+require 'src/proto/grpc/testing/test'
+
+module Grpc
+  module Testing
+    module TestService
+      # A simple service to test the various types of RPCs and experiment with
+      # performance with various types of payload.
+      class Service
+
+        include GRPC::GenericService
+
+        self.marshal_class_method = :encode
+        self.unmarshal_class_method = :decode
+        self.service_name = 'grpc.testing.TestService'
+
+        # One empty request followed by one empty response.
+        rpc :EmptyCall, Empty, Empty
+        # One request followed by one response.
+        rpc :UnaryCall, SimpleRequest, SimpleResponse
+        # One request followed by a sequence of responses (streamed download).
+        # The server returns the payload with client desired type and sizes.
+        rpc :StreamingOutputCall, StreamingOutputCallRequest, stream(StreamingOutputCallResponse)
+        # A sequence of requests followed by one response (streamed upload).
+        # The server returns the aggregated size of client payload as the result.
+        rpc :StreamingInputCall, stream(StreamingInputCallRequest), StreamingInputCallResponse
+        # A sequence of requests with each request served by the server immediately.
+        # As one request could lead to multiple responses, this interface
+        # demonstrates the idea of full duplexing.
+        rpc :FullDuplexCall, stream(StreamingOutputCallRequest), stream(StreamingOutputCallResponse)
+        # A sequence of requests followed by a sequence of responses.
+        # The server buffers all the client requests and then serves them in order. A
+        # stream of responses are returned to the client when the server starts with
+        # first request.
+        rpc :HalfDuplexCall, stream(StreamingOutputCallRequest), stream(StreamingOutputCallResponse)
+      end
+
+      Stub = Service.rpc_stub_class
+    end
+    module UnimplementedService
+      # A simple service NOT implemented at servers so clients can test for
+      # that case.
+      class Service
+
+        include GRPC::GenericService
+
+        self.marshal_class_method = :encode
+        self.unmarshal_class_method = :decode
+        self.service_name = 'grpc.testing.UnimplementedService'
+
+        # A call that no server should implement
+        rpc :UnimplementedCall, Empty, Empty
+      end
+
+      Stub = Service.rpc_stub_class
+    end
+    module ReconnectService
+      # A service used to control reconnect server.
+      class Service
+
+        include GRPC::GenericService
+
+        self.marshal_class_method = :encode
+        self.unmarshal_class_method = :decode
+        self.service_name = 'grpc.testing.ReconnectService'
+
+        rpc :Start, ReconnectParams, Empty
+        rpc :Stop, Empty, ReconnectInfo
+      end
+
+      Stub = Service.rpc_stub_class
+    end
+  end
+end
diff --git a/src/ruby/pb/test/client.rb b/src/ruby/pb/test/client.rb
index 95b059a18e..b6695482a2 100755
--- a/src/ruby/pb/test/client.rb
+++ b/src/ruby/pb/test/client.rb
@@ -114,7 +114,9 @@ def create_stub(opts)
   if opts.secure
     creds = ssl_creds(opts.use_test_ca)
     stub_opts = {
-      GRPC::Core::Channel::SSL_TARGET => opts.host_override
+      channel_args: {
+        GRPC::Core::Channel::SSL_TARGET => opts.host_override
+      }
     }
 
     # Add service account creds if specified
@@ -315,7 +317,8 @@ class NamedTests
   def timeout_on_sleeping_server
     msg_sizes = [[27_182, 31_415]]
     ppp = PingPongPlayer.new(msg_sizes)
-    resps = @stub.full_duplex_call(ppp.each_item, timeout: 0.001)
+    deadline = GRPC::Core::TimeConsts::from_relative_time(0.001)
+    resps = @stub.full_duplex_call(ppp.each_item, deadline: deadline)
     resps.each { |r| ppp.queue.push(r) }
     fail 'Should have raised GRPC::BadStatus(DEADLINE_EXCEEDED)'
   rescue GRPC::BadStatus => e
diff --git a/src/ruby/qps/client.rb b/src/ruby/qps/client.rb
index d04f707479..917b01271e 100644
--- a/src/ruby/qps/client.rb
+++ b/src/ruby/qps/client.rb
@@ -66,8 +66,10 @@ class BenchmarkClient
         cred = GRPC::Core::ChannelCredentials.new()
       end
       if config.security_params.server_host_override
-        opts[GRPC::Core::Channel::SSL_TARGET] =
+        channel_args = {}
+        channel_args[GRPC::Core::Channel::SSL_TARGET] =
           config.security_params.server_host_override
+        opts[:channel_args] = channel_args
       end
     else
       cred = :this_channel_is_insecure
diff --git a/src/ruby/qps/server.rb b/src/ruby/qps/server.rb
index f05fbbdaaf..52a89ce847 100644
--- a/src/ruby/qps/server.rb
+++ b/src/ruby/qps/server.rb
@@ -75,13 +75,14 @@ class BenchmarkServer
     @port = @server.add_http2_port("0.0.0.0:" + port.to_s, cred)
     @server.handle(BenchmarkServiceImpl.new)
     @start_time = Time.now
-    Thread.new {
+    t = Thread.new {
       @server.run
     }
+    t.abort_on_exception
   end
   def mark(reset)
     s = Grpc::Testing::ServerStats.new(time_elapsed:
-                                         (Time.now-@start_time).to_f)
+                                       (Time.now-@start_time).to_f)
     @start_time = Time.now if reset
     s
   end
diff --git a/src/ruby/qps/src/proto/grpc/testing/services_services.rb b/src/ruby/qps/src/proto/grpc/testing/services_services.rb
index 3fd9f20f47..94b9a1e164 100644
--- a/src/ruby/qps/src/proto/grpc/testing/services_services.rb
+++ b/src/ruby/qps/src/proto/grpc/testing/services_services.rb
@@ -1,5 +1,37 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: src/proto/grpc/testing/services.proto for package 'grpc.testing'
+# Original file comments:
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# An integration test service that covers all the method signature permutations
+# of unary/streaming requests/responses.
 
 require 'grpc'
 require 'src/proto/grpc/testing/services'
@@ -7,8 +39,6 @@ require 'src/proto/grpc/testing/services'
 module Grpc
   module Testing
     module BenchmarkService
-
-      # TODO: add proto service documentation here
       class Service
 
         include GRPC::GenericService
@@ -17,15 +47,17 @@ module Grpc
         self.unmarshal_class_method = :decode
         self.service_name = 'grpc.testing.BenchmarkService'
 
+        # One request followed by one response.
+        # The server returns the client payload as-is.
         rpc :UnaryCall, SimpleRequest, SimpleResponse
+        # One request followed by one response.
+        # The server returns the client payload as-is.
         rpc :StreamingCall, stream(SimpleRequest), stream(SimpleResponse)
       end
 
       Stub = Service.rpc_stub_class
     end
     module WorkerService
-
-      # TODO: add proto service documentation here
       class Service
 
         include GRPC::GenericService
@@ -34,9 +66,23 @@ module Grpc
         self.unmarshal_class_method = :decode
         self.service_name = 'grpc.testing.WorkerService'
 
+        # Start server with specified workload.
+        # First request sent specifies the ServerConfig followed by ServerStatus
+        # response. After that, a "Mark" can be sent anytime to request the latest
+        # stats. Closing the stream will initiate shutdown of the test server
+        # and once the shutdown has finished, the OK status is sent to terminate
+        # this RPC.
         rpc :RunServer, stream(ServerArgs), stream(ServerStatus)
+        # Start client with specified workload.
+        # First request sent specifies the ClientConfig followed by ClientStatus
+        # response. After that, a "Mark" can be sent anytime to request the latest
+        # stats. Closing the stream will initiate shutdown of the test client
+        # and once the shutdown has finished, the OK status is sent to terminate
+        # this RPC.
         rpc :RunClient, stream(ClientArgs), stream(ClientStatus)
+        # Just return the core count - unary call
         rpc :CoreCount, CoreRequest, CoreResponse
+        # Quit this worker
         rpc :QuitWorker, Void, Void
       end
 
diff --git a/src/ruby/spec/client_server_spec.rb b/src/ruby/spec/client_server_spec.rb
index aedeca272d..d60d84996f 100644
--- a/src/ruby/spec/client_server_spec.rb
+++ b/src/ruby/spec/client_server_spec.rb
@@ -43,11 +43,11 @@ shared_context 'setup: tags' do
     Time.now + 5
   end
 
-  def server_allows_client_to_proceed
+  def server_allows_client_to_proceed(metadata = {})
     recvd_rpc = @server.request_call(@server_queue, @server_tag, deadline)
     expect(recvd_rpc).to_not eq nil
     server_call = recvd_rpc.call
-    ops = { CallOps::SEND_INITIAL_METADATA => {} }
+    ops = { CallOps::SEND_INITIAL_METADATA => metadata }
     svr_batch = server_call.run_batch(@server_queue, @server_tag, deadline, ops)
     expect(svr_batch.send_metadata).to be true
     server_call
@@ -135,6 +135,48 @@ shared_examples 'basic GRPC message delivery is OK' do
     expect(svr_batch.send_message).to be true
   end
 
+  it 'compressed messages can be sent and received' do
+    call = new_client_call
+    server_call = nil
+    long_request_str = '0' * 2000
+    long_response_str = '1' * 2000
+    md = { 'grpc-internal-encoding-request' => 'gzip' }
+
+    server_thread = Thread.new do
+      server_call = server_allows_client_to_proceed(md)
+    end
+
+    client_ops = {
+      CallOps::SEND_INITIAL_METADATA => md,
+      CallOps::SEND_MESSAGE => long_request_str
+    }
+    batch_result = call.run_batch(@client_queue, @client_tag, deadline,
+                                  client_ops)
+    expect(batch_result.send_metadata).to be true
+    expect(batch_result.send_message).to be true
+
+    # confirm the server can read the inbound message
+    server_thread.join
+    server_ops = {
+      CallOps::RECV_MESSAGE => nil,
+      CallOps::SEND_MESSAGE => long_response_str
+    }
+    svr_batch = server_call.run_batch(@server_queue, @server_tag, deadline,
+                                      server_ops)
+    expect(svr_batch.message).to eq(long_request_str)
+    expect(svr_batch.send_message).to be true
+
+    client_ops = {
+      CallOps::SEND_CLOSE_FROM_CLIENT => nil,
+      CallOps::RECV_INITIAL_METADATA => nil,
+      CallOps::RECV_MESSAGE => nil
+    }
+    batch_result = call.run_batch(@client_queue, @client_tag, deadline,
+                                  client_ops)
+    expect(batch_result.send_close).to be true
+    expect(batch_result.message).to eq long_response_str
+  end
+
   it 'servers can ignore a client write and send a status' do
     call = new_client_call
     server_call = nil
diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb
index c0181e21fb..d9c9780c93 100644
--- a/src/ruby/spec/generic/active_call_spec.rb
+++ b/src/ruby/spec/generic/active_call_spec.rb
@@ -159,9 +159,10 @@ describe GRPC::ActiveCall do
   end
 
   describe '#client_invoke' do
-    it 'sends keywords as metadata to the server when the are present' do
+    it 'sends metadata to the server when present' do
       call = make_test_call
-      ActiveCall.client_invoke(call, @client_queue, k1: 'v1', k2: 'v2')
+      metadata = { k1: 'v1', k2: 'v2' }
+      ActiveCall.client_invoke(call, @client_queue, metadata)
       recvd_rpc =  @server.request_call(@server_queue, @server_tag, deadline)
       recvd_call = recvd_rpc.call
       expect(recvd_call).to_not be_nil
diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb
index dd8e2e9f7a..168e7fb791 100644
--- a/src/ruby/spec/generic/client_stub_spec.rb
+++ b/src/ruby/spec/generic/client_stub_spec.rb
@@ -68,15 +68,7 @@ describe 'ClientStub' do
   describe '#new' do
     let(:fake_host) { 'localhost:0' }
     it 'can be created from a host and args' do
-      opts = { a_channel_arg: 'an_arg' }
-      blk = proc do
-        GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'can be created with a default deadline' do
-      opts = { a_channel_arg: 'an_arg', deadline: 5 }
+      opts = { channel_args: { a_channel_arg: 'an_arg' } }
       blk = proc do
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
@@ -84,7 +76,10 @@ describe 'ClientStub' do
     end
 
     it 'can be created with an channel override' do
-      opts = { a_channel_arg: 'an_arg', channel_override: @ch }
+      opts = {
+        channel_args: { a_channel_arg: 'an_arg' },
+        channel_override: @ch
+      }
       blk = proc do
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
@@ -93,7 +88,10 @@ describe 'ClientStub' do
 
     it 'cannot be created with a bad channel override' do
       blk = proc do
-        opts = { a_channel_arg: 'an_arg', channel_override: Object.new }
+        opts = {
+          channel_args: { a_channel_arg: 'an_arg' },
+          channel_override: Object.new
+        }
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
       expect(&blk).to raise_error
@@ -101,7 +99,7 @@ describe 'ClientStub' do
 
     it 'cannot be created with bad credentials' do
       blk = proc do
-        opts = { a_channel_arg: 'an_arg' }
+        opts = { channel_args: { a_channel_arg: 'an_arg' } }
         GRPC::ClientStub.new(fake_host, @cq, Object.new, **opts)
       end
       expect(&blk).to raise_error
@@ -111,8 +109,10 @@ describe 'ClientStub' do
       certs = load_test_certs
       blk = proc do
         opts = {
-          GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr',
-          a_channel_arg: 'an_arg'
+          channel_args: {
+            GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr',
+            a_channel_arg: 'an_arg'
+          }
         }
         creds = GRPC::Core::ChannelCredentials.new(certs[0], nil, nil)
         GRPC::ClientStub.new(fake_host, @cq, creds,  **opts)
@@ -172,7 +172,7 @@ describe 'ClientStub' do
     describe 'without a call operation' do
       def get_response(stub)
         stub.request_response(@method, @sent_msg, noop, noop,
-                              k1: 'v1', k2: 'v2')
+                              metadata: { k1: 'v1', k2: 'v2' })
       end
 
       it_behaves_like 'request response'
@@ -181,7 +181,8 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_response(stub)
         op = stub.request_response(@method, @sent_msg, noop, noop,
-                                   return_op: true, k1: 'v1', k2: 'v2')
+                                   return_op: true,
+                                   metadata: { k1: 'v1', k2: 'v2' })
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         op.execute
       end
@@ -196,7 +197,7 @@ describe 'ClientStub' do
         server_port = create_test_server
         host = "localhost:#{server_port}"
         @stub = GRPC::ClientStub.new(host, @cq, :this_channel_is_insecure)
-        @options = { k1: 'v1', k2: 'v2' }
+        @metadata = { k1: 'v1', k2: 'v2' }
         @sent_msgs = Array.new(3) { |i| 'msg_' + (i + 1).to_s }
         @resp = 'a_reply'
       end
@@ -208,7 +209,7 @@ describe 'ClientStub' do
       end
 
       it 'should send metadata to the server ok' do
-        th = run_client_streamer(@sent_msgs, @resp, @pass, @options)
+        th = run_client_streamer(@sent_msgs, @resp, @pass, **@metadata)
         expect(get_response(@stub)).to eq(@resp)
         th.join
       end
@@ -221,7 +222,7 @@ describe 'ClientStub' do
       end
 
       it 'should raise ArgumentError if metadata contains invalid values' do
-        @options.merge!(k3: 3)
+        @metadata.merge!(k3: 3)
         expect do
           get_response(@stub)
         end.to raise_error(ArgumentError,
@@ -231,7 +232,8 @@ describe 'ClientStub' do
 
     describe 'without a call operation' do
       def get_response(stub)
-        stub.client_streamer(@method, @sent_msgs, noop, noop, @options)
+        stub.client_streamer(@method, @sent_msgs, noop, noop,
+                             metadata: @metadata)
       end
 
       it_behaves_like 'client streaming'
@@ -240,7 +242,7 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_response(stub)
         op = stub.client_streamer(@method, @sent_msgs, noop, noop,
-                                  @options.merge(return_op: true))
+                                  return_op: true, metadata: @metadata)
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         op.execute
       end
@@ -290,7 +292,7 @@ describe 'ClientStub' do
     describe 'without a call operation' do
       def get_responses(stub)
         e = stub.server_streamer(@method, @sent_msg, noop, noop,
-                                 k1: 'v1', k2: 'v2')
+                                 metadata: { k1: 'v1', k2: 'v2' })
         expect(e).to be_a(Enumerator)
         e
       end
@@ -301,7 +303,8 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_responses(stub)
         op = stub.server_streamer(@method, @sent_msg, noop, noop,
-                                  return_op: true, k1: 'v1', k2: 'v2')
+                                  return_op: true,
+                                  metadata: { k1: 'v1', k2: 'v2' })
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         e = op.execute
         expect(e).to be_a(Enumerator)
@@ -383,7 +386,7 @@ describe 'ClientStub' do
         stub = GRPC::ClientStub.new(@host, @cq, :this_channel_is_insecure)
         blk = proc do
           e = stub.bidi_streamer(@method, @sent_msgs, noop, noop,
-                                 timeout: 0.001)
+                                 deadline: from_relative_time(0.001))
           e.collect { |r| r }
         end
         expect(&blk).to raise_error GRPC::BadStatus, /Deadline Exceeded/
diff --git a/src/ruby/spec/generic/rpc_desc_spec.rb b/src/ruby/spec/generic/rpc_desc_spec.rb
index 083632a080..d2080b7ca2 100644
--- a/src/ruby/spec/generic/rpc_desc_spec.rb
+++ b/src/ruby/spec/generic/rpc_desc_spec.rb
@@ -56,14 +56,14 @@ describe GRPC::RpcDesc do
     it 'sends the specified status if BadStatus is raised' do
       expect(@call).to receive(:remote_read).once.and_return(Object.new)
       expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                       {})
+                                                       metadata: {})
       this_desc.run_server_method(@call, method(:bad_status))
     end
 
     it 'sends status UNKNOWN if other StandardErrors are raised' do
       expect(@call).to receive(:remote_read).once.and_return(Object.new)
       expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason,
-                                                        false, {})
+                                                        false, metadata: {})
       this_desc.run_server_method(@call, method(:other_error))
     end
 
@@ -93,7 +93,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).once.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         this_desc.run_server_method(@call, method(:fake_reqresp))
       end
     end
@@ -106,13 +106,13 @@ describe GRPC::RpcDesc do
 
       it 'sends the specified status if BadStatus is raised' do
         expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                         {})
+                                                         metadata: {})
         @client_streamer.run_server_method(@call, method(:bad_status_alt))
       end
 
       it 'sends status UNKNOWN if other StandardErrors are raised' do
-        expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason,
-                                                          false, {})
+        expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason,
+                                                         false, metadata: {})
         @client_streamer.run_server_method(@call, method(:other_error_alt))
       end
 
@@ -128,7 +128,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).once.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @client_streamer.run_server_method(@call, method(:fake_clstream))
       end
     end
@@ -148,7 +148,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).twice.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @server_streamer.run_server_method(@call, method(:fake_svstream))
       end
     end
@@ -165,14 +165,14 @@ describe GRPC::RpcDesc do
         e = GRPC::BadStatus.new(@bs_code, 'NOK')
         expect(@call).to receive(:run_server_bidi).and_raise(e)
         expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                         {})
+                                                         metadata: {})
         @bidi_streamer.run_server_method(@call, method(:bad_status_alt))
       end
 
       it 'sends status UNKNOWN if other StandardErrors are raised' do
         expect(@call).to receive(:run_server_bidi).and_raise(StandardError)
         expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason,
-                                                         false, {})
+                                                         false, metadata: {})
         @bidi_streamer.run_server_method(@call, method(:other_error_alt))
       end
 
@@ -180,7 +180,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:run_server_bidi)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @bidi_streamer.run_server_method(@call, method(:fake_bidistream))
       end
     end
diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb
index 2a42736237..943502cea2 100644
--- a/src/ruby/spec/generic/rpc_server_spec.rb
+++ b/src/ruby/spec/generic/rpc_server_spec.rb
@@ -99,7 +99,7 @@ class FailingService
   end
 
   def an_rpc(_req, _call)
-    fail GRPC::BadStatus.new(@code, @details, **@md)
+    fail GRPC::BadStatus.new(@code, @details, @md)
   end
 end
 
@@ -137,24 +137,11 @@ describe GRPC::RpcServer do
     @noop = proc { |x| x }
 
     @server_queue = GRPC::Core::CompletionQueue.new
-    server_host = '0.0.0.0:0'
-    @server = GRPC::Core::Server.new(@server_queue, nil)
-    server_port = @server.add_http2_port(server_host, :this_port_is_insecure)
-    @host = "localhost:#{server_port}"
-    @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
   end
 
   describe '#new' do
     it 'can be created with just some args' do
-      opts = { a_channel_arg: 'an_arg' }
-      blk = proc do
-        RpcServer.new(**opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'can be created with a default deadline' do
-      opts = { a_channel_arg: 'an_arg', deadline: 5 }
+      opts = { server_args: { a_channel_arg: 'an_arg' } }
       blk = proc do
         RpcServer.new(**opts)
       end
@@ -163,7 +150,7 @@ describe GRPC::RpcServer do
 
     it 'can be created with a completion queue override' do
       opts = {
-        a_channel_arg: 'an_arg',
+        server_args: { a_channel_arg: 'an_arg' },
         completion_queue_override: @server_queue
       }
       blk = proc do
@@ -175,7 +162,7 @@ describe GRPC::RpcServer do
     it 'cannot be created with a bad completion queue override' do
       blk = proc do
         opts = {
-          a_channel_arg: 'an_arg',
+          server_args: { a_channel_arg: 'an_arg' },
           completion_queue_override: Object.new
         }
         RpcServer.new(**opts)
@@ -186,38 +173,20 @@ describe GRPC::RpcServer do
     it 'cannot be created with invalid ServerCredentials' do
       blk = proc do
         opts = {
-          a_channel_arg: 'an_arg',
+          server_args: { a_channel_arg: 'an_arg' },
           creds: Object.new
         }
         RpcServer.new(**opts)
       end
       expect(&blk).to raise_error
     end
-
-    it 'can be created with a server override' do
-      opts = { a_channel_arg: 'an_arg', server_override: @server }
-      blk = proc do
-        RpcServer.new(**opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'cannot be created with a bad server override' do
-      blk = proc do
-        opts = {
-          a_channel_arg: 'an_arg',
-          server_override: Object.new
-        }
-        RpcServer.new(**opts)
-      end
-      expect(&blk).to raise_error
-    end
   end
 
   describe '#stopped?' do
     before(:each) do
-      opts = { a_channel_arg: 'an_arg', poll_period: 1.5 }
+      opts = { server_args: { a_channel_arg: 'an_arg' }, poll_period: 1.5 }
       @srv = RpcServer.new(**opts)
+      @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
     end
 
     it 'starts out false' do
@@ -245,28 +214,30 @@ describe GRPC::RpcServer do
 
   describe '#running?' do
     it 'starts out false' do
-      opts = { a_channel_arg: 'an_arg', server_override: @server }
+      opts = {
+        server_args: { a_channel_arg: 'an_arg' }
+      }
       r = RpcServer.new(**opts)
       expect(r.running?).to be(false)
     end
 
     it 'is false if run is called with no services registered', server: true do
       opts = {
-        a_channel_arg: 'an_arg',
-        poll_period: 2,
-        server_override: @server
+        server_args: { a_channel_arg: 'an_arg' },
+        poll_period: 2
       }
       r = RpcServer.new(**opts)
+      r.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
       expect { r.run }.to raise_error(RuntimeError)
     end
 
     it 'is true after run is called with a registered service' do
       opts = {
-        a_channel_arg: 'an_arg',
-        poll_period: 2.5,
-        server_override: @server
+        server_args: { a_channel_arg: 'an_arg' },
+        poll_period: 2.5
       }
       r = RpcServer.new(**opts)
+      r.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
       r.handle(EchoService)
       t = Thread.new { r.run }
       r.wait_till_running
@@ -278,8 +249,9 @@ describe GRPC::RpcServer do
 
   describe '#handle' do
     before(:each) do
-      @opts = { a_channel_arg: 'an_arg', poll_period: 1 }
+      @opts = { server_args: { a_channel_arg: 'an_arg' }, poll_period: 1 }
       @srv = RpcServer.new(**@opts)
+      @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
     end
 
     it 'raises if #run has already been called' do
@@ -322,11 +294,13 @@ describe GRPC::RpcServer do
     context 'with no connect_metadata' do
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1
         }
         @srv = RpcServer.new(**server_opts)
+        server_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @host = "localhost:#{server_port}"
+        @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
       end
 
       it 'should return NOT_FOUND status on unknown methods', server: true do
@@ -383,7 +357,8 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        expect(stub.an_rpc(req, k1: 'v1', k2: 'v2')).to be_a(EchoMsg)
+        expect(stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }))
+          .to be_a(EchoMsg)
         wanted_md = [{ 'k1' => 'v1', 'k2' => 'v2' }]
         check_md(wanted_md, service.received_md)
         @srv.stop
@@ -397,8 +372,11 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
-        timeout = service.delay + 1.0 # wait for long enough
-        resp = stub.an_rpc(req, timeout: timeout, k1: 'v1', k2: 'v2')
+        timeout = service.delay + 1.0
+        deadline = GRPC::Core::TimeConsts.from_relative_time(timeout)
+        resp = stub.an_rpc(req,
+                           deadline: deadline,
+                           metadata: { k1: 'v1', k2: 'v2' })
         expect(resp).to be_a(EchoMsg)
         wanted_md = [{ 'k1' => 'v1', 'k2' => 'v2' }]
         check_md(wanted_md, service.received_md)
@@ -413,7 +391,7 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        op = stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }, return_op: true)
         Thread.new do  # cancel the call
           sleep 0.1
           op.cancel
@@ -443,8 +421,7 @@ describe GRPC::RpcServer do
 
       it 'should return RESOURCE_EXHAUSTED on too many jobs', server: true do
         opts = {
-          a_channel_arg: 'an_arg',
-          server_override: @server,
+          server_args: { a_channel_arg: 'an_arg' },
           completion_queue_override: @server_queue,
           pool_size: 1,
           poll_period: 1,
@@ -452,6 +429,8 @@ describe GRPC::RpcServer do
         }
         alt_srv = RpcServer.new(**opts)
         alt_srv.handle(SlowService)
+        alt_port = alt_srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        alt_host = "0.0.0.0:#{alt_port}"
         t = Thread.new { alt_srv.run }
         alt_srv.wait_till_running
         req = EchoMsg.new
@@ -460,7 +439,7 @@ describe GRPC::RpcServer do
         one_failed_as_unavailable = false
         n.times do
           threads << Thread.new do
-            stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
+            stub = SlowStub.new(alt_host, :this_channel_is_insecure)
             begin
               stub.an_rpc(req)
             rescue GRPC::BadStatus => e
@@ -487,12 +466,13 @@ describe GRPC::RpcServer do
       end
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1,
           connect_md_proc: test_md_proc
         }
         @srv = RpcServer.new(**server_opts)
+        alt_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @alt_host = "0.0.0.0:#{alt_port}"
       end
 
       it 'should send connect metadata to the client', server: true do
@@ -501,8 +481,8 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        stub = EchoStub.new(@alt_host, :this_channel_is_insecure)
+        op = stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }, return_op: true)
         expect(op.metadata).to be nil
         expect(op.execute).to be_a(EchoMsg)
         wanted_md = {
@@ -522,11 +502,12 @@ describe GRPC::RpcServer do
     context 'with trailing metadata' do
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1
         }
         @srv = RpcServer.new(**server_opts)
+        alt_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @alt_host = "0.0.0.0:#{alt_port}"
       end
 
       it 'should be added to BadStatus when requests fail', server: true do
@@ -535,7 +516,7 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = FailingStub.new(@host, :this_channel_is_insecure, **client_opts)
+        stub = FailingStub.new(@alt_host, :this_channel_is_insecure)
         blk = proc { stub.an_rpc(req) }
 
         # confirm it raise the expected error
@@ -560,8 +541,8 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        stub = EchoStub.new(@alt_host, :this_channel_is_insecure)
+        op = stub.an_rpc(req, return_op: true, metadata: { k1: 'v1', k2: 'v2' })
         expect(op.metadata).to be nil
         expect(op.execute).to be_a(EchoMsg)
         expect(op.metadata).to eq(wanted_trailers)
diff --git a/src/ruby/spec/pb/health/checker_spec.rb b/src/ruby/spec/pb/health/checker_spec.rb
index 5523347f45..f3d121a31e 100644
--- a/src/ruby/spec/pb/health/checker_spec.rb
+++ b/src/ruby/spec/pb/health/checker_spec.rb
@@ -170,17 +170,15 @@ describe Grpc::Health::Checker do
     before(:each) do
       @server_queue = GRPC::Core::CompletionQueue.new
       server_host = '0.0.0.0:0'
-      @server = GRPC::Core::Server.new(@server_queue, nil)
-      server_port = @server.add_http2_port(server_host, :this_port_is_insecure)
-      @host = "localhost:#{server_port}"
-      @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
       @client_opts = { channel_override: @ch }
       server_opts = {
-        server_override: @server,
         completion_queue_override: @server_queue,
         poll_period: 1
       }
       @srv = RpcServer.new(**server_opts)
+      server_port = @srv.add_http2_port(server_host, :this_port_is_insecure)
+      @host = "localhost:#{server_port}"
+      @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
     end
 
     after(:each) do
diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb
index e19d7338ad..68c1bf369d 100644
--- a/src/ruby/tools/version.rb
+++ b/src/ruby/tools/version.rb
@@ -29,6 +29,6 @@
 
 module GRPC
   module Tools
-    VERSION = '0.14.2.pre1'
+    VERSION = '0.16.0.dev'
   end
 end