Merge github.com:grpc/grpc into channel_max_reconnect_backoff

51 files changed, 1224 insertions, 231 deletions

diff --git a/include/grpc++/alarm.h b/include/grpc++/alarm.h
index 3b8104d135..764837a958 100644
--- a/include/grpc++/alarm.h
+++ b/include/grpc++/alarm.h
@@ -50,7 +50,7 @@ namespace grpc {
 class CompletionQueue;
 
 /// A thin wrapper around \a grpc_alarm (see / \a / src/core/surface/alarm.h).
-class Alarm : private GrpcLibrary {
+class Alarm : private GrpcLibraryCodegen {
  public:
   /// Create a completion queue alarm instance associated to \a cq.
   ///
diff --git a/include/grpc++/channel.h b/include/grpc++/channel.h
index 80547f7ab8..679d473204 100644
--- a/include/grpc++/channel.h
+++ b/include/grpc++/channel.h
@@ -49,7 +49,7 @@ namespace grpc {
 class Channel GRPC_FINAL : public ChannelInterface,
                            public CallHook,
                            public std::enable_shared_from_this<Channel>,
-                           private GrpcLibrary {
+                           private GrpcLibraryCodegen {
  public:
   ~Channel();
 
diff --git a/include/grpc++/generic/async_generic_service.h b/include/grpc++/generic/async_generic_service.h
index 9ae8391dc4..b87b17ee0d 100644
--- a/include/grpc++/generic/async_generic_service.h
+++ b/include/grpc++/generic/async_generic_service.h
@@ -34,8 +34,8 @@
 #ifndef GRPCXX_GENERIC_ASYNC_GENERIC_SERVICE_H
 #define GRPCXX_GENERIC_ASYNC_GENERIC_SERVICE_H
 
-#include <grpc++/support/byte_buffer.h>
 #include <grpc++/support/async_stream.h>
+#include <grpc++/support/byte_buffer.h>
 
 struct grpc_server;
 
diff --git a/include/grpc++/impl/codegen/async_stream.h b/include/grpc++/impl/codegen/async_stream.h
index b0410485f8..cac345e0dc 100644
--- a/include/grpc++/impl/codegen/async_stream.h
+++ b/include/grpc++/impl/codegen/async_stream.h
@@ -34,10 +34,11 @@
 #ifndef GRPCXX_IMPL_CODEGEN_ASYNC_STREAM_H
 #define GRPCXX_IMPL_CODEGEN_ASYNC_STREAM_H
 
-#include <grpc++/impl/codegen/channel_interface.h>
 #include <grpc++/impl/codegen/call.h>
-#include <grpc++/impl/codegen/service_type.h>
+#include <grpc++/impl/codegen/channel_interface.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/server_context.h>
+#include <grpc++/impl/codegen/service_type.h>
 #include <grpc++/impl/codegen/status.h>
 
 namespace grpc {
@@ -109,13 +110,13 @@ class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
     init_ops_.set_output_tag(tag);
     init_ops_.SendInitialMetadata(context->send_initial_metadata_);
     // TODO(ctiller): don't assert
-    GPR_ASSERT(init_ops_.SendMessage(request).ok());
+    GPR_CODEGEN_ASSERT(init_ops_.SendMessage(request).ok());
     init_ops_.ClientSendClose();
     call_.PerformOps(&init_ops_);
   }
 
   void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     meta_ops_.set_output_tag(tag);
     meta_ops_.RecvInitialMetadata(context_);
@@ -177,7 +178,7 @@ class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
   }
 
   void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     meta_ops_.set_output_tag(tag);
     meta_ops_.RecvInitialMetadata(context_);
@@ -187,7 +188,7 @@ class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
   void Write(const W& msg, void* tag) GRPC_OVERRIDE {
     write_ops_.set_output_tag(tag);
     // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
     call_.PerformOps(&write_ops_);
   }
 
@@ -214,7 +215,8 @@ class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
   CallOpSet<CallOpSendMessage> write_ops_;
   CallOpSet<CallOpClientSendClose> writes_done_ops_;
   CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
-            CallOpClientRecvStatus> finish_ops_;
+            CallOpClientRecvStatus>
+      finish_ops_;
 };
 
 /// Client-side interface for asynchronous bi-directional streaming.
@@ -243,7 +245,7 @@ class ClientAsyncReaderWriter GRPC_FINAL
   }
 
   void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     meta_ops_.set_output_tag(tag);
     meta_ops_.RecvInitialMetadata(context_);
@@ -262,7 +264,7 @@ class ClientAsyncReaderWriter GRPC_FINAL
   void Write(const W& msg, void* tag) GRPC_OVERRIDE {
     write_ops_.set_output_tag(tag);
     // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
     call_.PerformOps(&write_ops_);
   }
 
@@ -300,7 +302,7 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
       : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
 
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     meta_ops_.set_output_tag(tag);
     meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
@@ -331,7 +333,7 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
   }
 
   void FinishWithError(const Status& status, void* tag) {
-    GPR_ASSERT(!status.ok());
+    GPR_CODEGEN_ASSERT(!status.ok());
     finish_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
       finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
@@ -349,7 +351,8 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
   CallOpSet<CallOpSendInitialMetadata> meta_ops_;
   CallOpSet<CallOpRecvMessage<R>> read_ops_;
   CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-            CallOpServerSendStatus> finish_ops_;
+            CallOpServerSendStatus>
+      finish_ops_;
 };
 
 template <class W>
@@ -360,7 +363,7 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
       : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
 
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     meta_ops_.set_output_tag(tag);
     meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
@@ -375,7 +378,7 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
       ctx_->sent_initial_metadata_ = true;
     }
     // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
     call_.PerformOps(&write_ops_);
   }
 
@@ -409,7 +412,7 @@ class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
       : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
 
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     meta_ops_.set_output_tag(tag);
     meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
@@ -430,7 +433,7 @@ class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
       ctx_->sent_initial_metadata_ = true;
     }
     // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
     call_.PerformOps(&write_ops_);
   }
 
diff --git a/include/grpc++/impl/codegen/async_unary_call.h b/include/grpc++/impl/codegen/async_unary_call.h
index f3c75dc3b1..1526debf54 100644
--- a/include/grpc++/impl/codegen/async_unary_call.h
+++ b/include/grpc++/impl/codegen/async_unary_call.h
@@ -45,6 +45,7 @@
 namespace grpc {
 
 class CompletionQueue;
+extern CoreCodegenInterface* g_core_codegen_interface;
 
 template <class R>
 class ClientAsyncResponseReaderInterface {
@@ -68,13 +69,13 @@ class ClientAsyncResponseReader GRPC_FINAL
     collection_->init_buf_.SetCollection(collection_);
     collection_->init_buf_.SendInitialMetadata(context->send_initial_metadata_);
     // TODO(ctiller): don't assert
-    GPR_ASSERT(collection_->init_buf_.SendMessage(request).ok());
+    GPR_CODEGEN_ASSERT(collection_->init_buf_.SendMessage(request).ok());
     collection_->init_buf_.ClientSendClose();
     call_.PerformOps(&collection_->init_buf_);
   }
 
   void ReadInitialMetadata(void* tag) {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     collection_->meta_buf_.SetCollection(collection_);
     collection_->meta_buf_.set_output_tag(tag);
@@ -100,10 +101,12 @@ class ClientAsyncResponseReader GRPC_FINAL
   class CallOpSetCollection : public CallOpSetCollectionInterface {
    public:
     SneakyCallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-                    CallOpClientSendClose> init_buf_;
+                    CallOpClientSendClose>
+        init_buf_;
     CallOpSet<CallOpRecvInitialMetadata> meta_buf_;
     CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>,
-              CallOpClientRecvStatus> finish_buf_;
+              CallOpClientRecvStatus>
+        finish_buf_;
   };
   std::shared_ptr<CallOpSetCollection> collection_;
 };
@@ -116,7 +119,7 @@ class ServerAsyncResponseWriter GRPC_FINAL
       : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
 
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     meta_buf_.set_output_tag(tag);
     meta_buf_.SendInitialMetadata(ctx_->initial_metadata_);
@@ -141,7 +144,7 @@ class ServerAsyncResponseWriter GRPC_FINAL
   }
 
   void FinishWithError(const Status& status, void* tag) {
-    GPR_ASSERT(!status.ok());
+    GPR_CODEGEN_ASSERT(!status.ok());
     finish_buf_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
       finish_buf_.SendInitialMetadata(ctx_->initial_metadata_);
@@ -158,7 +161,8 @@ class ServerAsyncResponseWriter GRPC_FINAL
   ServerContext* ctx_;
   CallOpSet<CallOpSendInitialMetadata> meta_buf_;
   CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-            CallOpServerSendStatus> finish_buf_;
+            CallOpServerSendStatus>
+      finish_buf_;
 };
 
 }  // namespace grpc
diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h
index 5e450b0d24..50f5a75191 100644
--- a/include/grpc++/impl/codegen/call.h
+++ b/include/grpc++/impl/codegen/call.h
@@ -34,19 +34,21 @@
 #ifndef GRPCXX_IMPL_CODEGEN_CALL_H
 #define GRPCXX_IMPL_CODEGEN_CALL_H
 
+#include <cstring>
 #include <functional>
-#include <memory>
 #include <map>
-#include <cstring>
+#include <memory>
 
-#include <grpc/impl/codegen/alloc.h>
-#include <grpc/impl/codegen/grpc_types.h>
-#include <grpc++/impl/codegen/client_context.h>
 #include <grpc++/impl/codegen/call_hook.h>
+#include <grpc++/impl/codegen/client_context.h>
 #include <grpc++/impl/codegen/completion_queue_tag.h>
-#include <grpc++/impl/codegen/serialization_traits.h>
 #include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
+#include <grpc++/impl/codegen/serialization_traits.h>
 #include <grpc++/impl/codegen/status.h>
+#include <grpc++/impl/codegen/string_ref.h>
+#include <grpc/impl/codegen/alloc.h>
+#include <grpc/impl/codegen/grpc_types.h>
 
 struct grpc_byte_buffer;
 
@@ -56,12 +58,39 @@ class ByteBuffer;
 class Call;
 class CallHook;
 class CompletionQueue;
+extern CoreCodegenInterface* g_core_codegen_interface;
 
-void FillMetadataMap(
+inline void FillMetadataMap(
     grpc_metadata_array* arr,
-    std::multimap<grpc::string_ref, grpc::string_ref>* metadata);
-grpc_metadata* FillMetadataArray(
-    const std::multimap<grpc::string, grpc::string>& metadata);
+    std::multimap<grpc::string_ref, grpc::string_ref>* metadata) {
+  for (size_t i = 0; i < arr->count; i++) {
+    // TODO(yangg) handle duplicates?
+    metadata->insert(std::pair<grpc::string_ref, grpc::string_ref>(
+        arr->metadata[i].key, grpc::string_ref(arr->metadata[i].value,
+                                               arr->metadata[i].value_length)));
+  }
+  g_core_codegen_interface->grpc_metadata_array_destroy(arr);
+  g_core_codegen_interface->grpc_metadata_array_init(arr);
+}
+
+// TODO(yangg) if the map is changed before we send, the pointers will be a
+// mess. Make sure it does not happen.
+inline grpc_metadata* FillMetadataArray(
+    const std::multimap<grpc::string, grpc::string>& metadata) {
+  if (metadata.empty()) {
+    return nullptr;
+  }
+  grpc_metadata* metadata_array =
+      (grpc_metadata*)(g_core_codegen_interface->gpr_malloc(
+          metadata.size() * sizeof(grpc_metadata)));
+  size_t i = 0;
+  for (auto iter = metadata.cbegin(); iter != metadata.cend(); ++iter, ++i) {
+    metadata_array[i].key = iter->first.c_str();
+    metadata_array[i].value = iter->second.c_str();
+    metadata_array[i].value_length = iter->second.size();
+  }
+  return metadata_array;
+}
 
 /// Per-message write options.
 class WriteOptions {
@@ -170,7 +199,7 @@ class CallOpSendInitialMetadata {
   }
   void FinishOp(bool* status, int max_message_size) {
     if (!send_) return;
-    gpr_free(initial_metadata_);
+    g_core_codegen_interface->gpr_free(initial_metadata_);
     send_ = false;
   }
 
@@ -204,7 +233,7 @@ class CallOpSendMessage {
     write_options_.Clear();
   }
   void FinishOp(bool* status, int max_message_size) {
-    if (own_buf_) grpc_byte_buffer_destroy(send_buf_);
+    if (own_buf_) g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_);
     send_buf_ = nullptr;
   }
 
@@ -251,10 +280,11 @@ class CallOpRecvMessage {
       if (*status) {
         got_message = true;
         *status = SerializationTraits<R>::Deserialize(recv_buf_, message_,
-                                                      max_message_size).ok();
+                                                      max_message_size)
+                      .ok();
       } else {
         got_message = false;
-        grpc_byte_buffer_destroy(recv_buf_);
+        g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_);
       }
     } else {
       got_message = false;
@@ -321,7 +351,7 @@ class CallOpGenericRecvMessage {
         *status = deserialize_->Deserialize(recv_buf_, max_message_size).ok();
       } else {
         got_message = false;
-        grpc_byte_buffer_destroy(recv_buf_);
+        g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_);
       }
     } else {
       got_message = false;
@@ -386,7 +416,7 @@ class CallOpServerSendStatus {
 
   void FinishOp(bool* status, int max_message_size) {
     if (!send_status_available_) return;
-    gpr_free(trailing_metadata_);
+    g_core_codegen_interface->gpr_free(trailing_metadata_);
     send_status_available_ = false;
   }
 
@@ -462,7 +492,7 @@ class CallOpClientRecvStatus {
     *recv_status_ = Status(
         static_cast<StatusCode>(status_code_),
         status_details_ ? grpc::string(status_details_) : grpc::string());
-    gpr_free(status_details_);
+    g_core_codegen_interface->gpr_free(status_details_);
     recv_status_ = nullptr;
   }
 
@@ -576,11 +606,22 @@ class SneakyCallOpSet : public CallOpSet<Op1, Op2, Op3, Op4, Op5, Op6> {
 class Call GRPC_FINAL {
  public:
   /* call is owned by the caller */
-  Call(grpc_call* call, CallHook* call_hook_, CompletionQueue* cq);
-  Call(grpc_call* call, CallHook* call_hook_, CompletionQueue* cq,
-       int max_message_size);
-
-  void PerformOps(CallOpSetInterface* ops);
+  Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq)
+      : call_hook_(call_hook), cq_(cq), call_(call), max_message_size_(-1) {}
+
+  Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq,
+       int max_message_size)
+      : call_hook_(call_hook),
+        cq_(cq),
+        call_(call),
+        max_message_size_(max_message_size) {}
+
+  void PerformOps(CallOpSetInterface* ops) {
+    if (max_message_size_ > 0) {
+      ops->set_max_message_size(max_message_size_);
+    }
+    call_hook_->PerformOpsOnCall(ops, this);
+  }
 
   grpc_call* call() { return call_; }
   CompletionQueue* cq() { return cq_; }
diff --git a/include/grpc++/impl/codegen/client_context.h b/include/grpc++/impl/codegen/client_context.h
index db2afe930c..271d464583 100644
--- a/include/grpc++/impl/codegen/client_context.h
+++ b/include/grpc++/impl/codegen/client_context.h
@@ -54,6 +54,7 @@
 #include <string>
 
 #include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/security/auth_context.h>
 #include <grpc++/impl/codegen/status.h>
 #include <grpc++/impl/codegen/string_ref.h>
@@ -192,7 +193,7 @@ class ClientContext {
   /// \return A multimap of initial metadata key-value pairs from the server.
   const std::multimap<grpc::string_ref, grpc::string_ref>&
   GetServerInitialMetadata() {
-    GPR_ASSERT(initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(initial_metadata_received_);
     return recv_initial_metadata_;
   }
 
diff --git a/include/grpc++/impl/codegen/client_unary_call.h b/include/grpc++/impl/codegen/client_unary_call.h
index 0134dec800..6c35a95765 100644
--- a/include/grpc++/impl/codegen/client_unary_call.h
+++ b/include/grpc++/impl/codegen/client_unary_call.h
@@ -37,6 +37,7 @@
 #include <grpc++/impl/codegen/call.h>
 #include <grpc++/impl/codegen/channel_interface.h>
 #include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/status.h>
 
 namespace grpc {
@@ -55,7 +56,8 @@ Status BlockingUnaryCall(ChannelInterface* channel, const RpcMethod& method,
   Call call(channel->CreateCall(method, context, &cq));
   CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
             CallOpRecvInitialMetadata, CallOpRecvMessage<OutputMessage>,
-            CallOpClientSendClose, CallOpClientRecvStatus> ops;
+            CallOpClientSendClose, CallOpClientRecvStatus>
+      ops;
   Status status = ops.SendMessage(request);
   if (!status.ok()) {
     return status;
@@ -66,7 +68,7 @@ Status BlockingUnaryCall(ChannelInterface* channel, const RpcMethod& method,
   ops.ClientSendClose();
   ops.ClientRecvStatus(context, &status);
   call.PerformOps(&ops);
-  GPR_ASSERT((cq.Pluck(&ops) && ops.got_message) || !status.ok());
+  GPR_CODEGEN_ASSERT((cq.Pluck(&ops) && ops.got_message) || !status.ok());
   return status;
 }
 
diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h
index 102831e1c9..42ec287e65 100644
--- a/include/grpc++/impl/codegen/completion_queue.h
+++ b/include/grpc++/impl/codegen/completion_queue.h
@@ -36,9 +36,12 @@
 #ifndef GRPCXX_IMPL_CODEGEN_COMPLETION_QUEUE_H
 #define GRPCXX_IMPL_CODEGEN_COMPLETION_QUEUE_H
 
+#include <grpc++/impl/codegen/completion_queue_tag.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/grpc_library.h>
 #include <grpc++/impl/codegen/status.h>
 #include <grpc++/impl/codegen/time.h>
+#include <grpc/impl/codegen/time.h>
 
 struct grpc_completion_queue;
 
@@ -76,13 +79,17 @@ class Server;
 class ServerBuilder;
 class ServerContext;
 
+extern CoreCodegenInterface* g_core_codegen_interface;
+
 /// A thin wrapper around \a grpc_completion_queue (see / \a
 /// src/core/surface/completion_queue.h).
-class CompletionQueue : private GrpcLibrary {
+class CompletionQueue : private GrpcLibraryCodegen {
  public:
   /// Default constructor. Implicitly creates a \a grpc_completion_queue
   /// instance.
-  CompletionQueue();
+  CompletionQueue() {
+    cq_ = g_core_codegen_interface->grpc_completion_queue_create(nullptr);
+  }
 
   /// Wrap \a take, taking ownership of the instance.
   ///
@@ -90,7 +97,9 @@ class CompletionQueue : private GrpcLibrary {
   explicit CompletionQueue(grpc_completion_queue* take);
 
   /// Destructor. Destroys the owned wrapped completion queue / instance.
-  ~CompletionQueue();
+  ~CompletionQueue() {
+    g_core_codegen_interface->grpc_completion_queue_destroy(cq_);
+  }
 
   /// Tri-state return for AsyncNext: SHUTDOWN, GOT_EVENT, TIMEOUT.
   enum NextStatus {
@@ -124,8 +133,8 @@ class CompletionQueue : private GrpcLibrary {
   ///
   /// \return true if read a regular event, false if the queue is shutting down.
   bool Next(void** tag, bool* ok) {
-    return (AsyncNextInternal(tag, ok, gpr_inf_future(GPR_CLOCK_REALTIME)) !=
-            SHUTDOWN);
+    return (AsyncNextInternal(tag, ok, g_core_codegen_interface->gpr_inf_future(
+                                           GPR_CLOCK_REALTIME)) != SHUTDOWN);
   }
 
   /// Request the shutdown of the queue.
@@ -181,10 +190,31 @@ class CompletionQueue : private GrpcLibrary {
 
   /// Wraps \a grpc_completion_queue_pluck.
   /// \warning Must not be mixed with calls to \a Next.
-  bool Pluck(CompletionQueueTag* tag);
+  bool Pluck(CompletionQueueTag* tag) {
+    auto deadline =
+        g_core_codegen_interface->gpr_inf_future(GPR_CLOCK_REALTIME);
+    auto ev = g_core_codegen_interface->grpc_completion_queue_pluck(
+        cq_, tag, deadline, nullptr);
+    bool ok = ev.success != 0;
+    void* ignored = tag;
+    GPR_CODEGEN_ASSERT(tag->FinalizeResult(&ignored, &ok));
+    GPR_CODEGEN_ASSERT(ignored == tag);
+    // Ignore mutations by FinalizeResult: Pluck returns the C API status
+    return ev.success != 0;
+  }
 
   /// Performs a single polling pluck on \a tag.
-  void TryPluck(CompletionQueueTag* tag);
+  /// \warning Must not be mixed with calls to \a Next.
+  void TryPluck(CompletionQueueTag* tag) {
+    auto deadline = gpr_time_0(GPR_CLOCK_REALTIME);
+    auto ev = g_core_codegen_interface->grpc_completion_queue_pluck(
+        cq_, tag, deadline, nullptr);
+    if (ev.type == GRPC_QUEUE_TIMEOUT) return;
+    bool ok = ev.success != 0;
+    void* ignored = tag;
+    // the tag must be swallowed if using TryPluck
+    GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok));
+  }
 
   grpc_completion_queue* cq_;  // owned
 };
diff --git a/include/grpc++/impl/codegen/completion_queue_tag.h b/include/grpc++/impl/codegen/completion_queue_tag.h
index 8be2ac36d6..e904f73e96 100644
--- a/include/grpc++/impl/codegen/completion_queue_tag.h
+++ b/include/grpc++/impl/codegen/completion_queue_tag.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_COMPLETION_QUEUE_TAG_H
-#define GRPCXX_COMPLETION_QUEUE_TAG_H
+#ifndef GRPCXX_IMPL_CODEGEN_COMPLETION_QUEUE_TAG_H
+#define GRPCXX_IMPL_CODEGEN_COMPLETION_QUEUE_TAG_H
 
 namespace grpc {
 
@@ -49,4 +49,4 @@ class CompletionQueueTag {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_COMPLETION_QUEUE_TAG_H
+#endif  // GRPCXX_IMPL_CODEGEN_COMPLETION_QUEUE_TAG_H
diff --git a/include/grpc++/impl/codegen/core_codegen_interface.h b/include/grpc++/impl/codegen/core_codegen_interface.h
new file mode 100644
index 0000000000..a23031fe65
--- /dev/null
+++ b/include/grpc++/impl/codegen/core_codegen_interface.h
@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_CORE_CODEGEN_INTERFACE_H
+#define GRPCXX_IMPL_CODEGEN_CORE_CODEGEN_INTERFACE_H
+
+#include <grpc++/impl/codegen/config_protobuf.h>
+#include <grpc++/impl/codegen/status.h>
+#include <grpc/impl/codegen/grpc_types.h>
+
+namespace grpc {
+
+/// Interface between the codegen library and the minimal subset of core
+/// features required by the generated code.
+///
+/// All undocumented methods are simply forwarding the call to their namesakes.
+/// Please refer to their corresponding documentation for details.
+///
+/// \warning This interface should be considered internal and private.
+class CoreCodegenInterface {
+ public:
+  // Serialize the msg into a buffer created inside the function. The caller
+  // should destroy the returned buffer when done with it. If serialization
+  // fails,
+  // false is returned and buffer is left unchanged.
+  virtual Status SerializeProto(const grpc::protobuf::Message& msg,
+                                grpc_byte_buffer** buffer) = 0;
+
+  // The caller keeps ownership of buffer and msg.
+  virtual Status DeserializeProto(grpc_byte_buffer* buffer,
+                                  grpc::protobuf::Message* msg,
+                                  int max_message_size) = 0;
+
+  /// Upon a failed assertion, log the error.
+  virtual void assert_fail(const char* failed_assertion) = 0;
+
+  virtual grpc_completion_queue* grpc_completion_queue_create(
+      void* reserved) = 0;
+  virtual void grpc_completion_queue_destroy(grpc_completion_queue* cq) = 0;
+  virtual grpc_event grpc_completion_queue_pluck(grpc_completion_queue* cq,
+                                                 void* tag,
+                                                 gpr_timespec deadline,
+                                                 void* reserved) = 0;
+
+  virtual void* gpr_malloc(size_t size) = 0;
+  virtual void gpr_free(void* p) = 0;
+
+  virtual void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) = 0;
+  virtual void grpc_metadata_array_init(grpc_metadata_array* array) = 0;
+  virtual void grpc_metadata_array_destroy(grpc_metadata_array* array) = 0;
+
+  virtual gpr_timespec gpr_inf_future(gpr_clock_type type) = 0;
+};
+
+extern CoreCodegenInterface* g_core_codegen_interface;
+
+/// Codegen specific version of \a GPR_ASSERT.
+#define GPR_CODEGEN_ASSERT(x)                          \
+  do {                                                 \
+    if (!(x)) {                                        \
+      grpc::g_core_codegen_interface->assert_fail(#x); \
+    }                                                  \
+  } while (0)
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_CORE_CODEGEN_INTERFACE_H
diff --git a/include/grpc++/impl/codegen/grpc_library.h b/include/grpc++/impl/codegen/grpc_library.h
index eb7152a2c6..3cdc6f3f7c 100644
--- a/include/grpc++/impl/codegen/grpc_library.h
+++ b/include/grpc++/impl/codegen/grpc_library.h
@@ -34,6 +34,7 @@
 #ifndef GRPCXX_IMPL_CODEGEN_GRPC_LIBRARY_H
 #define GRPCXX_IMPL_CODEGEN_GRPC_LIBRARY_H
 
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc/impl/codegen/log.h>
 
 namespace grpc {
@@ -44,24 +45,27 @@ class GrpcLibraryInterface {
   virtual void shutdown() = 0;
 };
 
+/// Initialized by \a grpc::GrpcLibraryInitializer from
+/// <grpc++/impl/grpc_library.h>
 extern GrpcLibraryInterface* g_glip;
 
-class GrpcLibrary {
+/// Classes that require gRPC to be initialized should inherit from this class.
+class GrpcLibraryCodegen {
  public:
-  GrpcLibrary() {
-    GPR_ASSERT(g_glip &&
-               "gRPC library not initialized. See "
-               "grpc::internal::GrpcLibraryInitializer.");
+  GrpcLibraryCodegen() {
+    GPR_CODEGEN_ASSERT(g_glip &&
+                       "gRPC library not initialized. See "
+                       "grpc::internal::GrpcLibraryInitializer.");
     g_glip->init();
   }
-  virtual ~GrpcLibrary() {
-    GPR_ASSERT(g_glip &&
-               "gRPC library not initialized. See "
-               "grpc::internal::GrpcLibraryInitializer.");
+  virtual ~GrpcLibraryCodegen() {
+    GPR_CODEGEN_ASSERT(g_glip &&
+                       "gRPC library not initialized. See "
+                       "grpc::internal::GrpcLibraryInitializer.");
     g_glip->shutdown();
   }
 };
 
 }  // namespace grpc
 
-#endif  // GRPCXX_IMPL_GRPC_LIBRARY_H
+#endif  // GRPCXX_IMPL_CODEGEN_GRPC_LIBRARY_H
diff --git a/include/grpc++/impl/codegen/impl/async_stream.h b/include/grpc++/impl/codegen/impl/async_stream.h
new file mode 100644
index 0000000000..95c844723a
--- /dev/null
+++ b/include/grpc++/impl/codegen/impl/async_stream.h
@@ -0,0 +1,465 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_IMPL_ASYNC_STREAM_H
+#define GRPCXX_IMPL_CODEGEN_IMPL_ASYNC_STREAM_H
+
+#include <grpc++/impl/codegen/call.h>
+#include <grpc++/impl/codegen/channel_interface.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
+#include <grpc++/impl/codegen/server_context.h>
+#include <grpc++/impl/codegen/service_type.h>
+#include <grpc++/impl/codegen/status.h>
+
+namespace grpc {
+
+class CompletionQueue;
+
+/// Common interface for all client side asynchronous streaming.
+class ClientAsyncStreamingInterface {
+ public:
+  virtual ~ClientAsyncStreamingInterface() {}
+
+  /// Request notification of the reading of the initial metadata. Completion
+  /// will be notified by \a tag on the associated completion queue.
+  ///
+  /// \param[in] tag Tag identifying this request.
+  virtual void ReadInitialMetadata(void* tag) = 0;
+
+  /// Request notification completion.
+  ///
+  /// \param[out] status To be updated with the operation status.
+  /// \param[in] tag Tag identifying this request.
+  virtual void Finish(Status* status, void* tag) = 0;
+};
+
+/// An interface that yields a sequence of messages of type \a R.
+template <class R>
+class AsyncReaderInterface {
+ public:
+  virtual ~AsyncReaderInterface() {}
+
+  /// Read a message of type \a R into \a msg. Completion will be notified by \a
+  /// tag on the associated completion queue.
+  ///
+  /// \param[out] msg Where to eventually store the read message.
+  /// \param[in] tag The tag identifying the operation.
+  virtual void Read(R* msg, void* tag) = 0;
+};
+
+/// An interface that can be fed a sequence of messages of type \a W.
+template <class W>
+class AsyncWriterInterface {
+ public:
+  virtual ~AsyncWriterInterface() {}
+
+  /// Request the writing of \a msg with identifying tag \a tag.
+  ///
+  /// Only one write may be outstanding at any given time. This means that
+  /// after calling Write, one must wait to receive \a tag from the completion
+  /// queue BEFORE calling Write again.
+  ///
+  /// \param[in] msg The message to be written.
+  /// \param[in] tag The tag identifying the operation.
+  virtual void Write(const W& msg, void* tag) = 0;
+};
+
+template <class R>
+class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface,
+                                   public AsyncReaderInterface<R> {};
+
+template <class R>
+class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
+ public:
+  /// Create a stream and write the first request out.
+  template <class W>
+  ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
+                    const RpcMethod& method, ClientContext* context,
+                    const W& request, void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    // TODO(ctiller): don't assert
+    GPR_CODEGEN_ASSERT(init_ops_.SendMessage(request).ok());
+    init_ops_.ClientSendClose();
+    call_.PerformOps(&init_ops_);
+  }
+
+  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      read_ops_.RecvInitialMetadata(context_);
+    }
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
+    finish_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_ops_.RecvInitialMetadata(context_);
+    }
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  ClientContext* context_;
+  Call call_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose>
+      init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
+};
+
+/// Common interface for client side asynchronous writing.
+template <class W>
+class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface,
+                                   public AsyncWriterInterface<W> {
+ public:
+  /// Signal the client is done with the writes.
+  ///
+  /// \param[in] tag The tag identifying the operation.
+  virtual void WritesDone(void* tag) = 0;
+};
+
+template <class W>
+class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
+ public:
+  template <class R>
+  ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
+                    const RpcMethod& method, ClientContext* context,
+                    R* response, void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    finish_ops_.RecvMessage(response);
+
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&init_ops_);
+  }
+
+  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    // TODO(ctiller): don't assert
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void WritesDone(void* tag) GRPC_OVERRIDE {
+    writes_done_ops_.set_output_tag(tag);
+    writes_done_ops_.ClientSendClose();
+    call_.PerformOps(&writes_done_ops_);
+  }
+
+  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
+    finish_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_ops_.RecvInitialMetadata(context_);
+    }
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  ClientContext* context_;
+  Call call_;
+  CallOpSet<CallOpSendInitialMetadata> init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpClientSendClose> writes_done_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
+            CallOpClientRecvStatus>
+      finish_ops_;
+};
+
+/// Client-side interface for asynchronous bi-directional streaming.
+template <class W, class R>
+class ClientAsyncReaderWriterInterface : public ClientAsyncStreamingInterface,
+                                         public AsyncWriterInterface<W>,
+                                         public AsyncReaderInterface<R> {
+ public:
+  /// Signal the client is done with the writes.
+  ///
+  /// \param[in] tag The tag identifying the operation.
+  virtual void WritesDone(void* tag) = 0;
+};
+
+template <class W, class R>
+class ClientAsyncReaderWriter GRPC_FINAL
+    : public ClientAsyncReaderWriterInterface<W, R> {
+ public:
+  ClientAsyncReaderWriter(ChannelInterface* channel, CompletionQueue* cq,
+                          const RpcMethod& method, ClientContext* context,
+                          void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&init_ops_);
+  }
+
+  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      read_ops_.RecvInitialMetadata(context_);
+    }
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    // TODO(ctiller): don't assert
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void WritesDone(void* tag) GRPC_OVERRIDE {
+    writes_done_ops_.set_output_tag(tag);
+    writes_done_ops_.ClientSendClose();
+    call_.PerformOps(&writes_done_ops_);
+  }
+
+  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
+    finish_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_ops_.RecvInitialMetadata(context_);
+    }
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  ClientContext* context_;
+  Call call_;
+  CallOpSet<CallOpSendInitialMetadata> init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpClientSendClose> writes_done_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
+};
+
+template <class W, class R>
+class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
+                                     public AsyncReaderInterface<R> {
+ public:
+  explicit ServerAsyncReader(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Finish(const W& msg, const Status& status, void* tag) {
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // The response is dropped if the status is not OK.
+    if (status.ok()) {
+      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_,
+                                   finish_ops_.SendMessage(msg));
+    } else {
+      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    }
+    call_.PerformOps(&finish_ops_);
+  }
+
+  void FinishWithError(const Status& status, void* tag) {
+    GPR_CODEGEN_ASSERT(!status.ok());
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+            CallOpServerSendStatus>
+      finish_ops_;
+};
+
+template <class W>
+class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
+                                     public AsyncWriterInterface<W> {
+ public:
+  explicit ServerAsyncWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // TODO(ctiller): don't assert
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void Finish(const Status& status, void* tag) {
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
+};
+
+/// Server-side interface for asynchronous bi-directional streaming.
+template <class W, class R>
+class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
+                                           public AsyncWriterInterface<W>,
+                                           public AsyncReaderInterface<R> {
+ public:
+  explicit ServerAsyncReaderWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // TODO(ctiller): don't assert
+    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void Finish(const Status& status, void* tag) {
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  friend class ::grpc::Server;
+
+  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_IMPL_ASYNC_STREAM_H
diff --git a/include/grpc++/impl/codegen/impl/status_code_enum.h b/include/grpc++/impl/codegen/impl/status_code_enum.h
new file mode 100644
index 0000000000..f8caec0c11
--- /dev/null
+++ b/include/grpc++/impl/codegen/impl/status_code_enum.h
@@ -0,0 +1,152 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_IMPL_STATUS_CODE_ENUM_H
+#define GRPCXX_IMPL_CODEGEN_IMPL_STATUS_CODE_ENUM_H
+
+namespace grpc {
+
+enum StatusCode {
+  /// Not an error; returned on success.
+  OK = 0,
+
+  /// The operation was cancelled (typically by the caller).
+  CANCELLED = 1,
+
+  /// Unknown error. An example of where this error may be returned is if a
+  /// Status value received from another address space belongs to an error-space
+  /// that is not known in this address space. Also errors raised by APIs that
+  /// do not return enough error information may be converted to this error.
+  UNKNOWN = 2,
+
+  /// Client specified an invalid argument. Note that this differs from
+  /// FAILED_PRECONDITION. INVALID_ARGUMENT indicates arguments that are
+  /// problematic regardless of the state of the system (e.g., a malformed file
+  /// name).
+  INVALID_ARGUMENT = 3,
+
+  /// Deadline expired before operation could complete. For operations that
+  /// change the state of the system, this error may be returned even if the
+  /// operation has completed successfully. For example, a successful response
+  /// from a server could have been delayed long enough for the deadline to
+  /// expire.
+  DEADLINE_EXCEEDED = 4,
+
+  /// Some requested entity (e.g., file or directory) was not found.
+  NOT_FOUND = 5,
+
+  /// Some entity that we attempted to create (e.g., file or directory) already
+  /// exists.
+  ALREADY_EXISTS = 6,
+
+  /// The caller does not have permission to execute the specified operation.
+  /// PERMISSION_DENIED must not be used for rejections caused by exhausting
+  /// some resource (use RESOURCE_EXHAUSTED instead for those errors).
+  /// PERMISSION_DENIED must not be used if the caller can not be identified
+  /// (use UNAUTHENTICATED instead for those errors).
+  PERMISSION_DENIED = 7,
+
+  /// The request does not have valid authentication credentials for the
+  /// operation.
+  UNAUTHENTICATED = 16,
+
+  /// Some resource has been exhausted, perhaps a per-user quota, or perhaps the
+  /// entire file system is out of space.
+  RESOURCE_EXHAUSTED = 8,
+
+  /// Operation was rejected because the system is not in a state required for
+  /// the operation's execution. For example, directory to be deleted may be
+  /// non-empty, an rmdir operation is applied to a non-directory, etc.
+  ///
+  /// A litmus test that may help a service implementor in deciding
+  /// between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE:
+  ///  (a) Use UNAVAILABLE if the client can retry just the failing call.
+  ///  (b) Use ABORTED if the client should retry at a higher-level
+  ///      (e.g., restarting a read-modify-write sequence).
+  ///  (c) Use FAILED_PRECONDITION if the client should not retry until
+  ///      the system state has been explicitly fixed. E.g., if an "rmdir"
+  ///      fails because the directory is non-empty, FAILED_PRECONDITION
+  ///      should be returned since the client should not retry unless
+  ///      they have first fixed up the directory by deleting files from it.
+  ///  (d) Use FAILED_PRECONDITION if the client performs conditional
+  ///      REST Get/Update/Delete on a resource and the resource on the
+  ///      server does not match the condition. E.g., conflicting
+  ///      read-modify-write on the same resource.
+  FAILED_PRECONDITION = 9,
+
+  /// The operation was aborted, typically due to a concurrency issue like
+  /// sequencer check failures, transaction aborts, etc.
+  ///
+  /// See litmus test above for deciding between FAILED_PRECONDITION, ABORTED,
+  /// and UNAVAILABLE.
+  ABORTED = 10,
+
+  /// Operation was attempted past the valid range. E.g., seeking or reading
+  /// past end of file.
+  ///
+  /// Unlike INVALID_ARGUMENT, this error indicates a problem that may be fixed
+  /// if the system state changes. For example, a 32-bit file system will
+  /// generate INVALID_ARGUMENT if asked to read at an offset that is not in the
+  /// range [0,2^32-1], but it will generate OUT_OF_RANGE if asked to read from
+  /// an offset past the current file size.
+  ///
+  /// There is a fair bit of overlap between FAILED_PRECONDITION and
+  /// OUT_OF_RANGE. We recommend using OUT_OF_RANGE (the more specific error)
+  /// when it applies so that callers who are iterating through a space can
+  /// easily look for an OUT_OF_RANGE error to detect when they are done.
+  OUT_OF_RANGE = 11,
+
+  /// Operation is not implemented or not supported/enabled in this service.
+  UNIMPLEMENTED = 12,
+
+  /// Internal errors. Means some invariants expected by underlying System has
+  /// been broken. If you see one of these errors, Something is very broken.
+  INTERNAL = 13,
+
+  /// The service is currently unavailable. This is a most likely a transient
+  /// condition and may be corrected by retrying with a backoff.
+  ///
+  /// See litmus test above for deciding between FAILED_PRECONDITION, ABORTED,
+  /// and UNAVAILABLE.
+  UNAVAILABLE = 14,
+
+  /// Unrecoverable data loss or corruption.
+  DATA_LOSS = 15,
+
+  /// Force users to include a default branch:
+  DO_NOT_USE = -1
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_IMPL_STATUS_CODE_ENUM_H
diff --git a/include/grpc++/impl/codegen/impl/sync.h b/include/grpc++/impl/codegen/impl/sync.h
new file mode 100644
index 0000000000..68fd0c4f2d
--- /dev/null
+++ b/include/grpc++/impl/codegen/impl/sync.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_IMPL_SYNC_H
+#define GRPCXX_IMPL_CODEGEN_IMPL_SYNC_H
+
+#include <grpc++/impl/codegen/config.h>
+
+#ifdef GRPC_CXX0X_NO_THREAD
+#include <grpc++/impl/codegen/sync_no_cxx11.h>
+#else
+#include <grpc++/impl/codegen/sync_cxx11.h>
+#endif
+
+#endif  // GRPCXX_IMPL_CODEGEN_IMPL_SYNC_H
diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index 1bf9bf0549..0ab2ae466a 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -34,6 +34,7 @@
 #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H
 #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H
 
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/rpc_service_method.h>
 #include <grpc++/impl/codegen/sync_stream.h>
 
@@ -58,9 +59,10 @@ class RpcMethodHandler : public MethodHandler {
       status = func_(service_, param.server_context, &req, &rsp);
     }
 
-    GPR_ASSERT(!param.server_context->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!param.server_context->sent_initial_metadata_);
     CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-              CallOpServerSendStatus> ops;
+              CallOpServerSendStatus>
+        ops;
     ops.SendInitialMetadata(param.server_context->initial_metadata_);
     if (status.ok()) {
       status = ops.SendMessage(rsp);
@@ -73,7 +75,8 @@ class RpcMethodHandler : public MethodHandler {
  private:
   // Application provided rpc handler function.
   std::function<Status(ServiceType*, ServerContext*, const RequestType*,
-                       ResponseType*)> func_;
+                       ResponseType*)>
+      func_;
   // The class the above handler function lives in.
   ServiceType* service_;
 };
@@ -93,9 +96,10 @@ class ClientStreamingHandler : public MethodHandler {
     ResponseType rsp;
     Status status = func_(service_, param.server_context, &reader, &rsp);
 
-    GPR_ASSERT(!param.server_context->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!param.server_context->sent_initial_metadata_);
     CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-              CallOpServerSendStatus> ops;
+              CallOpServerSendStatus>
+        ops;
     ops.SendInitialMetadata(param.server_context->initial_metadata_);
     if (status.ok()) {
       status = ops.SendMessage(rsp);
@@ -107,7 +111,8 @@ class ClientStreamingHandler : public MethodHandler {
 
  private:
   std::function<Status(ServiceType*, ServerContext*, ServerReader<RequestType>*,
-                       ResponseType*)> func_;
+                       ResponseType*)>
+      func_;
   ServiceType* service_;
 };
 
@@ -142,7 +147,8 @@ class ServerStreamingHandler : public MethodHandler {
 
  private:
   std::function<Status(ServiceType*, ServerContext*, const RequestType*,
-                       ServerWriter<ResponseType>*)> func_;
+                       ServerWriter<ResponseType>*)>
+      func_;
   ServiceType* service_;
 };
 
@@ -173,7 +179,8 @@ class BidiStreamingHandler : public MethodHandler {
 
  private:
   std::function<Status(ServiceType*, ServerContext*,
-                       ServerReaderWriter<ResponseType, RequestType>*)> func_;
+                       ServerReaderWriter<ResponseType, RequestType>*)>
+      func_;
   ServiceType* service_;
 };
 
diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h
index ce177104e0..f77f8eebb3 100644
--- a/include/grpc++/impl/codegen/proto_utils.h
+++ b/include/grpc++/impl/codegen/proto_utils.h
@@ -36,22 +36,16 @@
 
 #include <type_traits>
 
-#include <grpc/impl/codegen/byte_buffer.h>
-#include <grpc++/impl/codegen/serialization_traits.h>
 #include <grpc++/impl/codegen/config_protobuf.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
+#include <grpc++/impl/codegen/serialization_traits.h>
 #include <grpc++/impl/codegen/status.h>
+#include <grpc/impl/codegen/byte_buffer.h>
+#include <grpc/impl/codegen/log.h>
 
 namespace grpc {
 
-// Serialize the msg into a buffer created inside the function. The caller
-// should destroy the returned buffer when done with it. If serialization fails,
-// false is returned and buffer is left unchanged.
-Status SerializeProto(const grpc::protobuf::Message& msg,
-                      grpc_byte_buffer** buffer);
-
-// The caller keeps ownership of buffer and msg.
-Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
-                        int max_message_size);
+extern CoreCodegenInterface* g_core_codegen_interface;
 
 template <class T>
 class SerializationTraits<T, typename std::enable_if<std::is_base_of<
@@ -60,14 +54,13 @@ class SerializationTraits<T, typename std::enable_if<std::is_base_of<
   static Status Serialize(const grpc::protobuf::Message& msg,
                           grpc_byte_buffer** buffer, bool* own_buffer) {
     *own_buffer = true;
-    return SerializeProto(msg, buffer);
+    return g_core_codegen_interface->SerializeProto(msg, buffer);
   }
   static Status Deserialize(grpc_byte_buffer* buffer,
                             grpc::protobuf::Message* msg,
                             int max_message_size) {
-    auto status = DeserializeProto(buffer, msg, max_message_size);
-    grpc_byte_buffer_destroy(buffer);
-    return status;
+    return g_core_codegen_interface->DeserializeProto(buffer, msg,
+                                                      max_message_size);
   }
 };
 
diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h
index 6256301677..8b1f026c91 100644
--- a/include/grpc++/impl/codegen/rpc_service_method.h
+++ b/include/grpc++/impl/codegen/rpc_service_method.h
@@ -40,10 +40,10 @@
 #include <memory>
 #include <vector>
 
-#include <grpc/impl/codegen/byte_buffer.h>
 #include <grpc++/impl/codegen/config.h>
 #include <grpc++/impl/codegen/rpc_method.h>
 #include <grpc++/impl/codegen/status.h>
+#include <grpc/impl/codegen/byte_buffer.h>
 
 namespace grpc {
 class ServerContext;
diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h
index ad08b8210d..f8326bc44b 100644
--- a/include/grpc++/impl/codegen/server_context.h
+++ b/include/grpc++/impl/codegen/server_context.h
@@ -37,12 +37,12 @@
 #include <map>
 #include <memory>
 
-#include <grpc/impl/codegen/compression_types.h>
-#include <grpc/impl/codegen/time.h>
-#include <grpc++/impl/codegen/security/auth_context.h>
 #include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/security/auth_context.h>
 #include <grpc++/impl/codegen/string_ref.h>
 #include <grpc++/impl/codegen/time.h>
+#include <grpc/impl/codegen/compression_types.h>
+#include <grpc/impl/codegen/time.h>
 
 struct gpr_timespec;
 struct grpc_metadata;
@@ -103,6 +103,9 @@ class ServerContext {
   void AddInitialMetadata(const grpc::string& key, const grpc::string& value);
   void AddTrailingMetadata(const grpc::string& key, const grpc::string& value);
 
+  // IsCancelled is always safe to call when using sync API
+  // When using async API, it is only safe to call IsCancelled after
+  // the AsyncNotifyWhenDone tag has been delivered
   bool IsCancelled() const;
 
   // Cancel the Call from the server. This is a best-effort API and depending on
diff --git a/include/grpc++/impl/codegen/server_interface.h b/include/grpc++/impl/codegen/server_interface.h
index f934619c20..908d124df1 100644
--- a/include/grpc++/impl/codegen/server_interface.h
+++ b/include/grpc++/impl/codegen/server_interface.h
@@ -34,10 +34,11 @@
 #ifndef GRPCXX_IMPL_CODEGEN_SERVER_INTERFACE_H
 #define GRPCXX_IMPL_CODEGEN_SERVER_INTERFACE_H
 
-#include <grpc/impl/codegen/grpc_types.h>
 #include <grpc++/impl/codegen/call_hook.h>
 #include <grpc++/impl/codegen/completion_queue_tag.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/rpc_service_method.h>
+#include <grpc/impl/codegen/grpc_types.h>
 
 namespace grpc {
 
@@ -191,10 +192,11 @@ class ServerInterface : public CallHook {
     bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE {
       bool serialization_status =
           *status && payload_ &&
-          SerializationTraits<Message>::Deserialize(
-              payload_, request_, server_->max_message_size()).ok();
+          SerializationTraits<Message>::Deserialize(payload_, request_,
+                                                    server_->max_message_size())
+              .ok();
       bool ret = RegisteredAsyncRequest::FinalizeResult(tag, status);
-      *status = serialization_status&&* status;
+      *status = serialization_status && *status;
       return ret;
     }
 
@@ -223,7 +225,7 @@ class ServerInterface : public CallHook {
                         CompletionQueue* call_cq,
                         ServerCompletionQueue* notification_cq, void* tag,
                         Message* message) {
-    GPR_ASSERT(method);
+    GPR_CODEGEN_ASSERT(method);
     new PayloadAsyncRequest<Message>(method->server_tag(), this, context,
                                      stream, call_cq, notification_cq, tag,
                                      message);
@@ -233,7 +235,7 @@ class ServerInterface : public CallHook {
                         ServerAsyncStreamingInterface* stream,
                         CompletionQueue* call_cq,
                         ServerCompletionQueue* notification_cq, void* tag) {
-    GPR_ASSERT(method);
+    GPR_CODEGEN_ASSERT(method);
     new NoPayloadAsyncRequest(method->server_tag(), this, context, stream,
                               call_cq, notification_cq, tag);
   }
diff --git a/include/grpc++/impl/codegen/service_type.h b/include/grpc++/impl/codegen/service_type.h
index deb91a41d9..faf189314c 100644
--- a/include/grpc++/impl/codegen/service_type.h
+++ b/include/grpc++/impl/codegen/service_type.h
@@ -35,6 +35,7 @@
 #define GRPCXX_IMPL_CODEGEN_SERVICE_TYPE_H
 
 #include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/rpc_service_method.h>
 #include <grpc++/impl/codegen/serialization_traits.h>
 #include <grpc++/impl/codegen/server_interface.h>
@@ -131,21 +132,18 @@ class Service {
   void AddMethod(RpcServiceMethod* method) { methods_.emplace_back(method); }
 
   void MarkMethodAsync(int index) {
-    if (methods_[index].get() == nullptr) {
-      gpr_log(GPR_ERROR,
-              "Cannot mark the method as 'async' because it has already been "
-              "marked as 'generic'.");
-      return;
-    }
+    GPR_CODEGEN_ASSERT(
+        methods_[index].get() != nullptr &&
+        "Cannot mark the method as 'async' because it has already been "
+        "marked as 'generic'.");
     methods_[index]->ResetHandler();
   }
 
   void MarkMethodGeneric(int index) {
-    if (methods_[index]->handler() == nullptr) {
-      gpr_log(GPR_ERROR,
-              "Cannot mark the method as 'generic' because it has already been "
-              "marked as 'async'.");
-    }
+    GPR_CODEGEN_ASSERT(
+        methods_[index]->handler() != nullptr &&
+        "Cannot mark the method as 'generic' because it has already been "
+        "marked as 'async'.");
     methods_[index].reset();
   }
 
diff --git a/include/grpc++/impl/codegen/string_ref.h b/include/grpc++/impl/codegen/string_ref.h
index e3af37e0c2..4af2000ffd 100644
--- a/include/grpc++/impl/codegen/string_ref.h
+++ b/include/grpc++/impl/codegen/string_ref.h
@@ -34,8 +34,12 @@
 #ifndef GRPCXX_IMPL_CODEGEN_STRING_REF_H
 #define GRPCXX_IMPL_CODEGEN_STRING_REF_H
 
-#include <iterator>
+#include <string.h>
+
+#include <algorithm>
 #include <iosfwd>
+#include <iostream>
+#include <iterator>
 
 #include <grpc++/impl/codegen/config.h>
 
@@ -62,8 +66,13 @@ class string_ref {
   string_ref() : data_(nullptr), length_(0) {}
   string_ref(const string_ref& other)
       : data_(other.data_), length_(other.length_) {}
-  string_ref& operator=(const string_ref& rhs);
-  string_ref(const char* s);
+  string_ref& operator=(const string_ref& rhs) {
+    data_ = rhs.data_;
+    length_ = rhs.length_;
+    return *this;
+  }
+
+  string_ref(const char* s) : data_(s), length_(strlen(s)) {}
   string_ref(const char* s, size_t l) : data_(s), length_(l) {}
   string_ref(const grpc::string& s) : data_(s.data()), length_(s.length()) {}
 
@@ -95,13 +104,40 @@ class string_ref {
   const char* data() const { return data_; }
 
   // string operations
-  int compare(string_ref x) const;
-  bool starts_with(string_ref x) const;
-  bool ends_with(string_ref x) const;
-  size_t find(string_ref s) const;
-  size_t find(char c) const;
+  int compare(string_ref x) const {
+    size_t min_size = length_ < x.length_ ? length_ : x.length_;
+    int r = memcmp(data_, x.data_, min_size);
+    if (r < 0) return -1;
+    if (r > 0) return 1;
+    if (length_ < x.length_) return -1;
+    if (length_ > x.length_) return 1;
+    return 0;
+  }
+
+  bool starts_with(string_ref x) const {
+    return length_ >= x.length_ && (memcmp(data_, x.data_, x.length_) == 0);
+  }
 
-  string_ref substr(size_t pos, size_t n = npos) const;
+  bool ends_with(string_ref x) const {
+    return length_ >= x.length_ &&
+           (memcmp(data_ + (length_ - x.length_), x.data_, x.length_) == 0);
+  }
+
+  size_t find(string_ref s) const {
+    auto it = std::search(cbegin(), cend(), s.cbegin(), s.cend());
+    return it == cend() ? npos : std::distance(cbegin(), it);
+  }
+
+  size_t find(char c) const {
+    auto it = std::find(cbegin(), cend(), c);
+    return it == cend() ? npos : std::distance(cbegin(), it);
+  }
+
+  string_ref substr(size_t pos, size_t n = npos) const {
+    if (pos > length_) pos = length_;
+    if (n > (length_ - pos)) n = length_ - pos;
+    return string_ref(data_ + pos, n);
+  }
 
  private:
   const char* data_;
@@ -109,14 +145,16 @@ class string_ref {
 };
 
 // Comparison operators
-bool operator==(string_ref x, string_ref y);
-bool operator!=(string_ref x, string_ref y);
-bool operator<(string_ref x, string_ref y);
-bool operator>(string_ref x, string_ref y);
-bool operator<=(string_ref x, string_ref y);
-bool operator>=(string_ref x, string_ref y);
-
-std::ostream& operator<<(std::ostream& stream, const string_ref& string);
+inline bool operator==(string_ref x, string_ref y) { return x.compare(y) == 0; }
+inline bool operator!=(string_ref x, string_ref y) { return x.compare(y) != 0; }
+inline bool operator<(string_ref x, string_ref y) { return x.compare(y) < 0; }
+inline bool operator<=(string_ref x, string_ref y) { return x.compare(y) <= 0; }
+inline bool operator>(string_ref x, string_ref y) { return x.compare(y) > 0; }
+inline bool operator>=(string_ref x, string_ref y) { return x.compare(y) >= 0; }
+
+inline std::ostream& operator<<(std::ostream& out, const string_ref& string) {
+  return out << grpc::string(string.begin(), string.end());
+}
 
 }  // namespace grpc
 
diff --git a/include/grpc++/impl/codegen/sync_stream.h b/include/grpc++/impl/codegen/sync_stream.h
index 9ae48bd23d..0eabc5fc0f 100644
--- a/include/grpc++/impl/codegen/sync_stream.h
+++ b/include/grpc++/impl/codegen/sync_stream.h
@@ -38,6 +38,7 @@
 #include <grpc++/impl/codegen/channel_interface.h>
 #include <grpc++/impl/codegen/client_context.h>
 #include <grpc++/impl/codegen/completion_queue.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/server_context.h>
 #include <grpc++/impl/codegen/service_type.h>
 #include <grpc++/impl/codegen/status.h>
@@ -122,17 +123,18 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
                ClientContext* context, const W& request)
       : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
     CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-              CallOpClientSendClose> ops;
+              CallOpClientSendClose>
+        ops;
     ops.SendInitialMetadata(context->send_initial_metadata_);
     // TODO(ctiller): don't assert
-    GPR_ASSERT(ops.SendMessage(request).ok());
+    GPR_CODEGEN_ASSERT(ops.SendMessage(request).ok());
     ops.ClientSendClose();
     call_.PerformOps(&ops);
     cq_.Pluck(&ops);
   }
 
   void WaitForInitialMetadata() GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     CallOpSet<CallOpRecvInitialMetadata> ops;
     ops.RecvInitialMetadata(context_);
@@ -155,7 +157,7 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
     Status status;
     ops.ClientRecvStatus(context_, &status);
     call_.PerformOps(&ops);
-    GPR_ASSERT(cq_.Pluck(&ops));
+    GPR_CODEGEN_ASSERT(cq_.Pluck(&ops));
     return status;
   }
 
@@ -194,7 +196,7 @@ class ClientWriter : public ClientWriterInterface<W> {
   }
 
   void WaitForInitialMetadata() {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     CallOpSet<CallOpRecvInitialMetadata> ops;
     ops.RecvInitialMetadata(context_);
@@ -227,14 +229,15 @@ class ClientWriter : public ClientWriterInterface<W> {
     }
     finish_ops_.ClientRecvStatus(context_, &status);
     call_.PerformOps(&finish_ops_);
-    GPR_ASSERT(cq_.Pluck(&finish_ops_));
+    GPR_CODEGEN_ASSERT(cq_.Pluck(&finish_ops_));
     return status;
   }
 
  private:
   ClientContext* context_;
   CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
-            CallOpClientRecvStatus> finish_ops_;
+            CallOpClientRecvStatus>
+      finish_ops_;
   CompletionQueue cq_;
   Call call_;
 };
@@ -271,7 +274,7 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
   }
 
   void WaitForInitialMetadata() GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
+    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
     CallOpSet<CallOpRecvInitialMetadata> ops;
     ops.RecvInitialMetadata(context_);
@@ -312,7 +315,7 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
     Status status;
     ops.ClientRecvStatus(context_, &status);
     call_.PerformOps(&ops);
-    GPR_ASSERT(cq_.Pluck(&ops));
+    GPR_CODEGEN_ASSERT(cq_.Pluck(&ops));
     return status;
   }
 
@@ -328,7 +331,7 @@ class ServerReader GRPC_FINAL : public ReaderInterface<R> {
   ServerReader(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
 
   void SendInitialMetadata() {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(ctx_->initial_metadata_);
@@ -355,7 +358,7 @@ class ServerWriter GRPC_FINAL : public WriterInterface<W> {
   ServerWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
 
   void SendInitialMetadata() {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(ctx_->initial_metadata_);
@@ -391,7 +394,7 @@ class ServerReaderWriter GRPC_FINAL : public WriterInterface<W>,
   ServerReaderWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
 
   void SendInitialMetadata() {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
 
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(ctx_->initial_metadata_);
diff --git a/include/grpc++/impl/grpc_library.h b/include/grpc++/impl/grpc_library.h
index e8a075f5eb..ecb5a4d64d 100644
--- a/include/grpc++/impl/grpc_library.h
+++ b/include/grpc++/impl/grpc_library.h
@@ -40,21 +40,31 @@
 #include <grpc++/impl/codegen/grpc_library.h>
 #include <grpc/grpc.h>
 
+#include "src/cpp/common/core_codegen.h"
+
 namespace grpc {
 
 namespace internal {
 class GrpcLibrary GRPC_FINAL : public GrpcLibraryInterface {
  public:
   void init() GRPC_OVERRIDE { grpc_init(); }
-
   void shutdown() GRPC_OVERRIDE { grpc_shutdown(); }
 };
 
 static GrpcLibrary g_gli;
+static CoreCodegen g_core_codegen;
 
+/// Instantiating this class ensures the proper initialization of gRPC.
 class GrpcLibraryInitializer GRPC_FINAL {
  public:
-  GrpcLibraryInitializer() { grpc::g_glip = &g_gli; }
+  GrpcLibraryInitializer() {
+    if (grpc::g_glip == nullptr) {
+      grpc::g_glip = &g_gli;
+    }
+    if (grpc::g_core_codegen_interface == nullptr) {
+      grpc::g_core_codegen_interface = &g_core_codegen;
+    }
+  }
 
   /// A no-op method to force the linker to reference this class, which will
   /// take care of initializing and shutting down the gRPC runtime.
diff --git a/include/grpc++/impl/proto_utils.h b/include/grpc++/impl/proto_utils.h
index 9124001e20..36acabba79 100644
--- a/include/grpc++/impl/proto_utils.h
+++ b/include/grpc++/impl/proto_utils.h
@@ -31,9 +31,9 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
-#define GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
+#ifndef GRPCXX_IMPL_PROTO_UTILS_H
+#define GRPCXX_IMPL_PROTO_UTILS_H
 
 #include <grpc++/impl/codegen/proto_utils.h>
 
-#endif  // GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
+#endif  // GRPCXX_IMPL_PROTO_UTILS_H
diff --git a/include/grpc++/security/auth_context.h b/include/grpc++/security/auth_context.h
index bca8fa7c0c..548f9a236c 100644
--- a/include/grpc++/security/auth_context.h
+++ b/include/grpc++/security/auth_context.h
@@ -31,9 +31,9 @@
  *
  */
 
-#ifndef GRPCXX_SUPPORT_AUTH_CONTEXT_H
-#define GRPCXX_SUPPORT_AUTH_CONTEXT_H
+#ifndef GRPCXX_SECURITY_AUTH_CONTEXT_H
+#define GRPCXX_SECURITY_AUTH_CONTEXT_H
 
 #include <grpc++/impl/codegen/security/auth_context.h>
 
-#endif  // GRPCXX_SUPPORT_AUTH_CONTEXT_H
+#endif  // GRPCXX_SECURITY_AUTH_CONTEXT_H
diff --git a/include/grpc++/security/auth_metadata_processor.h b/include/grpc++/security/auth_metadata_processor.h
index 25011f33ba..b39451f83e 100644
--- a/include/grpc++/security/auth_metadata_processor.h
+++ b/include/grpc++/security/auth_metadata_processor.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_AUTH_METADATA_PROCESSOR_H_
-#define GRPCXX_AUTH_METADATA_PROCESSOR_H_
+#ifndef GRPCXX_SECURITY_AUTH_METADATA_PROCESSOR_H
+#define GRPCXX_SECURITY_AUTH_METADATA_PROCESSOR_H
 
 #include <map>
 
@@ -70,4 +70,4 @@ class AuthMetadataProcessor {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_AUTH_METADATA_PROCESSOR_H_
+#endif  // GRPCXX_SECURITY_AUTH_METADATA_PROCESSOR_H
diff --git a/include/grpc++/security/credentials.h b/include/grpc++/security/credentials.h
index e0806c0b7b..bedfc9c2a2 100644
--- a/include/grpc++/security/credentials.h
+++ b/include/grpc++/security/credentials.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_CREDENTIALS_H
-#define GRPCXX_CREDENTIALS_H
+#ifndef GRPCXX_SECURITY_CREDENTIALS_H
+#define GRPCXX_SECURITY_CREDENTIALS_H
 
 #include <map>
 #include <memory>
@@ -57,7 +57,7 @@ class SecureCallCredentials;
 /// for all the calls on that channel.
 ///
 /// \see http://www.grpc.io/docs/guides/auth.html
-class ChannelCredentials : private GrpcLibrary {
+class ChannelCredentials : private GrpcLibraryCodegen {
  public:
   ChannelCredentials();
   ~ChannelCredentials();
@@ -83,7 +83,7 @@ class ChannelCredentials : private GrpcLibrary {
 /// authenticate with a server for a given call on a channel.
 ///
 /// \see http://www.grpc.io/docs/guides/auth.html
-class CallCredentials : private GrpcLibrary {
+class CallCredentials : private GrpcLibraryCodegen {
  public:
   CallCredentials();
   ~CallCredentials();
@@ -229,4 +229,4 @@ std::shared_ptr<CallCredentials> MetadataCredentialsFromPlugin(
 
 }  // namespace grpc
 
-#endif  // GRPCXX_CREDENTIALS_H
+#endif  // GRPCXX_SECURITY_CREDENTIALS_H
diff --git a/include/grpc++/security/server_credentials.h b/include/grpc++/security/server_credentials.h
index e933825ec3..addb11ccb4 100644
--- a/include/grpc++/security/server_credentials.h
+++ b/include/grpc++/security/server_credentials.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_SERVER_CREDENTIALS_H
-#define GRPCXX_SERVER_CREDENTIALS_H
+#ifndef GRPCXX_SECURITY_SERVER_CREDENTIALS_H
+#define GRPCXX_SECURITY_SERVER_CREDENTIALS_H
 
 #include <memory>
 #include <vector>
@@ -89,4 +89,4 @@ std::shared_ptr<ServerCredentials> InsecureServerCredentials();
 
 }  // namespace grpc
 
-#endif  // GRPCXX_SERVER_CREDENTIALS_H
+#endif  // GRPCXX_SECURITY_SERVER_CREDENTIALS_H
diff --git a/include/grpc++/server.h b/include/grpc++/server.h
index c177805236..9eb8c287e1 100644
--- a/include/grpc++/server.h
+++ b/include/grpc++/server.h
@@ -62,7 +62,7 @@ class ThreadPoolInterface;
 /// Models a gRPC server.
 ///
 /// Servers are configured and started via \a grpc::ServerBuilder.
-class Server GRPC_FINAL : public ServerInterface, private GrpcLibrary {
+class Server GRPC_FINAL : public ServerInterface, private GrpcLibraryCodegen {
  public:
   ~Server();
 
diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h
index 82591a88ef..3825518de1 100644
--- a/include/grpc++/support/byte_buffer.h
+++ b/include/grpc++/support/byte_buffer.h
@@ -34,13 +34,13 @@
 #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H
 #define GRPCXX_SUPPORT_BYTE_BUFFER_H
 
-#include <grpc/grpc.h>
-#include <grpc/byte_buffer.h>
-#include <grpc/support/log.h>
 #include <grpc++/impl/serialization_traits.h>
 #include <grpc++/support/config.h>
 #include <grpc++/support/slice.h>
 #include <grpc++/support/status.h>
+#include <grpc/byte_buffer.h>
+#include <grpc/grpc.h>
+#include <grpc/support/log.h>
 
 #include <vector>
 
@@ -99,8 +99,8 @@ class SerializationTraits<ByteBuffer, void> {
   }
   static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer,
                           bool* own_buffer) {
-    *buffer = source.buffer();
-    *own_buffer = false;
+    *buffer = grpc_byte_buffer_copy(source.buffer());
+    *own_buffer = true;
     return Status::OK;
   }
 };
diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h
index a9ede35f90..8c2f7c71eb 100644
--- a/include/grpc++/support/channel_arguments.h
+++ b/include/grpc++/support/channel_arguments.h
@@ -34,12 +34,12 @@
 #ifndef GRPCXX_SUPPORT_CHANNEL_ARGUMENTS_H
 #define GRPCXX_SUPPORT_CHANNEL_ARGUMENTS_H
 
-#include <vector>
 #include <list>
+#include <vector>
 
+#include <grpc++/support/config.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
-#include <grpc++/support/config.h>
 
 namespace grpc {
 namespace testing {
diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h
index 724691a033..6251a8bcde 100644
--- a/include/grpc++/support/slice.h
+++ b/include/grpc++/support/slice.h
@@ -34,8 +34,8 @@
 #ifndef GRPCXX_SUPPORT_SLICE_H
 #define GRPCXX_SUPPORT_SLICE_H
 
-#include <grpc/support/slice.h>
 #include <grpc++/support/config.h>
+#include <grpc/support/slice.h>
 
 namespace grpc {
 
diff --git a/include/grpc/byte_buffer_reader.h b/include/grpc/byte_buffer_reader.h
index b0e63a6da2..600bd3810a 100644
--- a/include/grpc/byte_buffer_reader.h
+++ b/include/grpc/byte_buffer_reader.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,8 +34,8 @@
 #ifndef GRPC_BYTE_BUFFER_READER_H
 #define GRPC_BYTE_BUFFER_READER_H
 
-#include <grpc/grpc.h>
 #include <grpc/byte_buffer.h>
+#include <grpc/grpc.h>
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/include/grpc/census.h b/include/grpc/census.h
index 442a754f0a..4ccf4af5fe 100644
--- a/include/grpc/census.h
+++ b/include/grpc/census.h
@@ -35,8 +35,8 @@
  * they can (ultimately) be used in many different RPC systems (with differing
  * implementations). */
 
-#ifndef CENSUS_CENSUS_H
-#define CENSUS_CENSUS_H
+#ifndef GRPC_CENSUS_H
+#define GRPC_CENSUS_H
 
 #include <grpc/grpc.h>
 
@@ -537,4 +537,4 @@ CENSUSAPI void census_view_reset(census_view *view);
 }
 #endif
 
-#endif /* CENSUS_CENSUS_H */
+#endif /* GRPC_CENSUS_H */
diff --git a/include/grpc/compression.h b/include/grpc/compression.h
index acc168a6ee..39023ded34 100644
--- a/include/grpc/compression.h
+++ b/include/grpc/compression.h
@@ -36,8 +36,8 @@
 
 #include <stdlib.h>
 
-#include <grpc/impl/codegen/port_platform.h>
 #include <grpc/impl/codegen/compression_types.h>
+#include <grpc/impl/codegen/port_platform.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -55,11 +55,12 @@ GRPCAPI int grpc_compression_algorithm_parse(
 GRPCAPI int grpc_compression_algorithm_name(
     grpc_compression_algorithm algorithm, char **name);
 
-/** Returns the compression algorithm corresponding to \a level.
+/** Returns the compression algorithm corresponding to \a level for the
+ * compression algorithms encoded in the \a accepted_encodings bitset.
  *
  * It abort()s for unknown levels . */
-GRPCAPI grpc_compression_algorithm
-grpc_compression_algorithm_for_level(grpc_compression_level level);
+GRPCAPI grpc_compression_algorithm grpc_compression_algorithm_for_level(
+    grpc_compression_level level, uint32_t accepted_encodings);
 
 GRPCAPI void grpc_compression_options_init(grpc_compression_options *opts);
 
diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h
index 5113645daf..dbd2cc0000 100644
--- a/include/grpc/grpc.h
+++ b/include/grpc/grpc.h
@@ -36,13 +36,13 @@
 
 #include <grpc/status.h>
 
-#include <stddef.h>
 #include <grpc/byte_buffer.h>
-#include <grpc/support/slice.h>
-#include <grpc/support/time.h>
 #include <grpc/impl/codegen/connectivity_state.h>
-#include <grpc/impl/codegen/propagation_bits.h>
 #include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/impl/codegen/propagation_bits.h>
+#include <grpc/support/slice.h>
+#include <grpc/support/time.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -154,9 +154,8 @@ GRPCAPI void grpc_alarm_cancel(grpc_alarm *alarm);
 GRPCAPI void grpc_alarm_destroy(grpc_alarm *alarm);
 
 /** Check the connectivity state of a channel. */
-GRPCAPI grpc_connectivity_state
-grpc_channel_check_connectivity_state(grpc_channel *channel,
-                                      int try_to_connect);
+GRPCAPI grpc_connectivity_state grpc_channel_check_connectivity_state(
+    grpc_channel *channel, int try_to_connect);
 
 /** Watch for a change in connectivity state.
     Once the channel connectivity state is different from last_observed_state,
@@ -267,9 +266,10 @@ GRPCAPI grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved);
     and description passed in.
     Importantly, this function does not send status nor description to the
     remote endpoint. */
-GRPCAPI grpc_call_error
-grpc_call_cancel_with_status(grpc_call *call, grpc_status_code status,
-                             const char *description, void *reserved);
+GRPCAPI grpc_call_error grpc_call_cancel_with_status(grpc_call *call,
+                                                     grpc_status_code status,
+                                                     const char *description,
+                                                     void *reserved);
 
 /** Destroy a call.
     THREAD SAFETY: grpc_call_destroy is thread-compatible */
@@ -283,13 +283,11 @@ GRPCAPI void grpc_call_destroy(grpc_call *call);
     to \a cq_bound_to_call.
     Note that \a cq_for_notification must have been registered to the server via
     \a grpc_server_register_completion_queue. */
-GRPCAPI grpc_call_error
-grpc_server_request_call(grpc_server *server, grpc_call **call,
-                         grpc_call_details *details,
-                         grpc_metadata_array *request_metadata,
-                         grpc_completion_queue *cq_bound_to_call,
-                         grpc_completion_queue *cq_for_notification,
-                         void *tag_new);
+GRPCAPI grpc_call_error grpc_server_request_call(
+    grpc_server *server, grpc_call **call, grpc_call_details *details,
+    grpc_metadata_array *request_metadata,
+    grpc_completion_queue *cq_bound_to_call,
+    grpc_completion_queue *cq_for_notification, void *tag_new);
 
 /** Registers a method in the server.
     Methods to this (host, method) pair will not be reported by
@@ -299,7 +297,8 @@ grpc_server_request_call(grpc_server *server, grpc_call **call,
     Must be called before grpc_server_start.
     Returns NULL on failure. */
 GRPCAPI void *grpc_server_register_method(grpc_server *server,
-                                          const char *method, const char *host);
+                                          const char *method, const char *host,
+                                          uint32_t flags);
 
 /** Request notification of a new pre-registered call. 'cq_for_notification'
     must have been registered to the server via
diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h
index ef7205ded8..f2d04e551a 100644
--- a/include/grpc/grpc_security.h
+++ b/include/grpc/grpc_security.h
@@ -48,6 +48,7 @@ extern "C" {
 
 #define GRPC_X509_CN_PROPERTY_NAME "x509_common_name"
 #define GRPC_X509_SAN_PROPERTY_NAME "x509_subject_alternative_name"
+#define GRPC_X509_PEM_CERT_PROPERTY_NAME "x509_pem_cert"
 
 typedef struct grpc_auth_context grpc_auth_context;
 
@@ -79,9 +80,8 @@ grpc_auth_context_peer_identity(const grpc_auth_context *ctx);
 
 /* Finds a property in the context. May return an empty iterator (first _next
    will return NULL) if no property with this name was found in the context. */
-GRPCAPI grpc_auth_property_iterator
-grpc_auth_context_find_properties_by_name(const grpc_auth_context *ctx,
-                                          const char *name);
+GRPCAPI grpc_auth_property_iterator grpc_auth_context_find_properties_by_name(
+    const grpc_auth_context *ctx, const char *name);
 
 /* Gets the name of the property that indicates the peer identity. Will return
    NULL if the peer is not authenticated. */
@@ -362,8 +362,8 @@ GRPCAPI int grpc_server_add_secure_http2_port(grpc_server *server,
 
 /* Sets a credentials to a call. Can only be called on the client side before
    grpc_call_start_batch. */
-GRPCAPI grpc_call_error
-grpc_call_set_credentials(grpc_call *call, grpc_call_credentials *creds);
+GRPCAPI grpc_call_error grpc_call_set_credentials(grpc_call *call,
+                                                  grpc_call_credentials *creds);
 
 /* --- Auth Metadata Processing --- */
 
diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h
index 466af3a4cd..2c5b8216b2 100644
--- a/include/grpc/impl/codegen/grpc_types.h
+++ b/include/grpc/impl/codegen/grpc_types.h
@@ -201,6 +201,12 @@ typedef enum grpc_call_error {
 /** Mask of all valid flags. */
 #define GRPC_WRITE_USED_MASK (GRPC_WRITE_BUFFER_HINT | GRPC_WRITE_NO_COMPRESS)
 
+/* Initial metadata flags */
+/** Signal that the call is idempotent */
+#define GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST (0x00000010u)
+/** Mask of all valid flags */
+#define GRPC_INITIAL_METADATA_USED_MASK GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST
+
 /** A single metadata element */
 typedef struct grpc_metadata {
   const char *key;
@@ -252,6 +258,7 @@ typedef struct {
   char *host;
   size_t host_capacity;
   gpr_timespec deadline;
+  uint32_t flags;
   void *reserved;
 } grpc_call_details;
 
diff --git a/include/grpc/impl/codegen/log.h b/include/grpc/impl/codegen/log.h
index d6e18e9ca5..afd8c5d4cf 100644
--- a/include/grpc/impl/codegen/log.h
+++ b/include/grpc/impl/codegen/log.h
@@ -34,8 +34,8 @@
 #ifndef GRPC_IMPL_CODEGEN_LOG_H
 #define GRPC_IMPL_CODEGEN_LOG_H
 
-#include <stdlib.h> /* for abort() */
 #include <stdarg.h>
+#include <stdlib.h> /* for abort() */
 
 #include <grpc/impl/codegen/port_platform.h>
 
diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h
index 92569043fc..607511edc8 100644
--- a/include/grpc/impl/codegen/port_platform.h
+++ b/include/grpc/impl/codegen/port_platform.h
@@ -133,6 +133,9 @@
 #define GPR_POSIX_TIME 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #define GPR_HAVE_MSG_NOSIGNAL 1
+#define GPR_HAVE_UNIX_SOCKET 1
+#define GPR_HAVE_IP_PKTINFO 1
+#define GPR_HAVE_IPV6_RECVPKTINFO 1
 #elif defined(__linux__)
 #define GPR_POSIX_CRASH_HANDLER 1
 #define GPR_PLATFORM_STRING "linux"
@@ -154,6 +157,9 @@
 #define GPR_POSIX_WAKEUP_FD 1
 #define GPR_POSIX_SOCKET 1
 #define GPR_POSIX_SOCKETADDR 1
+#define GPR_HAVE_UNIX_SOCKET 1
+#define GPR_HAVE_IP_PKTINFO 1
+#define GPR_HAVE_IPV6_RECVPKTINFO 1
 #ifdef __GLIBC_PREREQ
 #if __GLIBC_PREREQ(2, 9)
 #define GPR_LINUX_EVENTFD 1
@@ -214,6 +220,8 @@
 #define GPR_POSIX_TIME 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #define GPR_HAVE_SO_NOSIGPIPE 1
+#define GPR_HAVE_UNIX_SOCKET 1
+#define GPR_HAVE_IP_PKTINFO 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
 #else /* _LP64 */
@@ -242,13 +250,49 @@
 #define GPR_POSIX_TIME 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #define GPR_HAVE_SO_NOSIGPIPE 1
+#define GPR_HAVE_UNIX_SOCKET 1
+#define GPR_HAVE_IP_PKTINFO 1
+#define GPR_HAVE_IPV6_RECVPKTINFO 1
+#ifdef _LP64
+#define GPR_ARCH_64 1
+#else /* _LP64 */
+#define GPR_ARCH_32 1
+#endif /* _LP64 */
+#elif defined(__native_client__)
+#define GPR_PLATFORM_STRING "nacl"
+#ifndef _BSD_SOURCE
+#define _BSD_SOURCE
+#endif
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#define GPR_CPU_POSIX 1
+#define GPR_GCC_ATOMIC 1
+#define GPR_GCC_TLS 1
+#define GPR_POSIX_LOG 1
+#define GPR_POSIX_MULTIPOLL_WITH_POLL 1
+#define GPR_POSIX_WAKEUP_FD 1
+#define GPR_POSIX_NO_SPECIAL_WAKEUP_FD 1
+#define GPR_POSIX_SOCKET 1
+#define GPR_POSIX_SOCKETADDR 1
+#define GPR_POSIX_SOCKETUTILS 1
+#define GPR_POSIX_ENV 1
+#define GPR_POSIX_FILE 1
+#define GPR_POSIX_STRING 1
+#define GPR_POSIX_SUBPROCESS 1
+#define GPR_POSIX_SYNC 1
+#define GPR_POSIX_TIME 1
+#define GPR_GETPID_IN_UNISTD_H 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
 #else /* _LP64 */
 #define GPR_ARCH_32 1
 #endif /* _LP64 */
 #else
-#error Could not auto-detect platform
+#error "Could not auto-detect platform"
 #endif
 #endif /* GPR_NO_AUTODETECT_PLATFORM */
 
diff --git a/include/grpc/impl/codegen/propagation_bits.h b/include/grpc/impl/codegen/propagation_bits.h
index cdd699710c..4b64558764 100644
--- a/include/grpc/impl/codegen/propagation_bits.h
+++ b/include/grpc/impl/codegen/propagation_bits.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_IMPL_CODEGEN_H
-#define GRPC_IMPL_CODEGEN_H
+#ifndef GRPC_IMPL_CODEGEN_PROPAGATION_BITS_H
+#define GRPC_IMPL_CODEGEN_PROPAGATION_BITS_H
 
 #include <grpc/impl/codegen/port_platform.h>
 
@@ -64,4 +64,4 @@ extern "C" {
 }
 #endif
 
-#endif /* GRPC_IMPL_CODEGEN_H */
+#endif /* GRPC_IMPL_CODEGEN_PROPAGATION_BITS_H */
diff --git a/include/grpc/impl/codegen/slice.h b/include/grpc/impl/codegen/slice.h
index a62fdd087b..03c59e72a0 100644
--- a/include/grpc/impl/codegen/slice.h
+++ b/include/grpc/impl/codegen/slice.h
@@ -122,8 +122,8 @@ GPRAPI gpr_slice gpr_slice_new(void *p, size_t len, void (*destroy)(void *));
 
 /* Equivalent to gpr_slice_new, but with a two argument destroy function that
    also takes the slice length. */
-GPRAPI gpr_slice
-gpr_slice_new_with_len(void *p, size_t len, void (*destroy)(void *, size_t));
+GPRAPI gpr_slice gpr_slice_new_with_len(void *p, size_t len,
+                                        void (*destroy)(void *, size_t));
 
 /* Equivalent to gpr_slice_new(malloc(len), len, free), but saves one malloc()
    call.
diff --git a/include/grpc/impl/codegen/slice_buffer.h b/include/grpc/impl/codegen/slice_buffer.h
index 4fe909ee82..7191878496 100644
--- a/include/grpc/impl/codegen/slice_buffer.h
+++ b/include/grpc/impl/codegen/slice_buffer.h
@@ -73,8 +73,8 @@ GPRAPI void gpr_slice_buffer_add(gpr_slice_buffer *sb, gpr_slice slice);
    slice at the returned index in sb->slices)
    The implementation MAY decide to concatenate data at the end of a small
    slice added in this fashion. */
-GPRAPI size_t
-gpr_slice_buffer_add_indexed(gpr_slice_buffer *sb, gpr_slice slice);
+GPRAPI size_t gpr_slice_buffer_add_indexed(gpr_slice_buffer *sb,
+                                           gpr_slice slice);
 GPRAPI void gpr_slice_buffer_addn(gpr_slice_buffer *sb, gpr_slice *slices,
                                   size_t n);
 /* add a very small (less than 8 bytes) amount of data to the end of a slice
diff --git a/include/grpc/impl/codegen/time.h b/include/grpc/impl/codegen/time.h
index c22bedfe77..b4f45097c7 100644
--- a/include/grpc/impl/codegen/time.h
+++ b/include/grpc/impl/codegen/time.h
@@ -88,8 +88,8 @@ GPRAPI void gpr_time_init(void);
 GPRAPI gpr_timespec gpr_now(gpr_clock_type clock);
 
 /* Convert a timespec from one clock to another */
-GPRAPI gpr_timespec
-gpr_convert_clock_type(gpr_timespec t, gpr_clock_type target_clock);
+GPRAPI gpr_timespec gpr_convert_clock_type(gpr_timespec t,
+                                           gpr_clock_type target_clock);
 
 /* Return -ve, 0, or +ve according to whether a < b, a == b, or a > b
    respectively.  */
diff --git a/include/grpc/support/atm_gcc_atomic.h b/include/grpc/support/atm_gcc_atomic.h
index 7e1f7fd55a..2fc8f609a5 100644
--- a/include/grpc/support/atm_gcc_atomic.h
+++ b/include/grpc/support/atm_gcc_atomic.h
@@ -31,9 +31,9 @@
  *
  */
 
-#ifndef GRPC_IMPL_CODEGEN_ATM_GCC_ATOMIC_H
-#define GRPC_IMPL_CODEGEN_ATM_GCC_ATOMIC_H
+#ifndef GRPC_SUPPORT_ATM_GCC_ATOMIC_H
+#define GRPC_SUPPORT_ATM_GCC_ATOMIC_H
 
 #include <grpc/impl/codegen/atm_gcc_atomic.h>
 
-#endif /* GRPC_IMPL_CODEGEN_ATM_GCC_ATOMIC_H */
+#endif /* GRPC_SUPPORT_ATM_GCC_ATOMIC_H */
diff --git a/include/grpc/support/subprocess.h b/include/grpc/support/subprocess.h
index 6a4946014b..e013b706ec 100644
--- a/include/grpc/support/subprocess.h
+++ b/include/grpc/support/subprocess.h
@@ -56,4 +56,4 @@ GPRAPI void gpr_subprocess_interrupt(gpr_subprocess *p);
 }  // extern "C"
 #endif
 
-#endif
+#endif /* GRPC_SUPPORT_SUBPROCESS_H */
diff --git a/include/grpc/support/tls.h b/include/grpc/support/tls.h
index 43addc7f14..0ff19bbadd 100644
--- a/include/grpc/support/tls.h
+++ b/include/grpc/support/tls.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -74,4 +74,4 @@
 #include <grpc/support/tls_pthread.h>
 #endif
 
-#endif
+#endif /* GRPC_SUPPORT_TLS_H */
diff --git a/include/grpc/support/tls_gcc.h b/include/grpc/support/tls_gcc.h
index a697ad05b0..21269e4b14 100644
--- a/include/grpc/support/tls_gcc.h
+++ b/include/grpc/support/tls_gcc.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,9 +34,51 @@
 #ifndef GRPC_SUPPORT_TLS_GCC_H
 #define GRPC_SUPPORT_TLS_GCC_H
 
+#include <stdbool.h>
+
+#include <grpc/support/log.h>
+
 /* Thread local storage based on gcc compiler primitives.
    #include tls.h to use this - and see that file for documentation */
 
+#ifndef NDEBUG
+
+struct gpr_gcc_thread_local {
+  intptr_t value;
+  bool *inited;
+};
+
+#define GPR_TLS_DECL(name)           \
+  static bool name##_inited = false; \
+  static __thread struct gpr_gcc_thread_local name = {0, &(name##_inited)}
+
+#define gpr_tls_init(tls)                  \
+  do {                                     \
+    GPR_ASSERT(*((tls)->inited) == false); \
+    *((tls)->inited) = true;               \
+  } while (0)
+
+/* It is allowed to call gpr_tls_init after gpr_tls_destroy is called. */
+#define gpr_tls_destroy(tls)      \
+  do {                            \
+    GPR_ASSERT(*((tls)->inited)); \
+    *((tls)->inited) = false;     \
+  } while (0)
+
+#define gpr_tls_set(tls, new_value) \
+  do {                              \
+    GPR_ASSERT(*((tls)->inited));   \
+    (tls)->value = (new_value);     \
+  } while (0)
+
+#define gpr_tls_get(tls)          \
+  ({                              \
+    GPR_ASSERT(*((tls)->inited)); \
+    (tls)->value;                 \
+  })
+
+#else /* NDEBUG */
+
 struct gpr_gcc_thread_local {
   intptr_t value;
 };
@@ -53,4 +95,6 @@ struct gpr_gcc_thread_local {
 #define gpr_tls_set(tls, new_value) (((tls)->value) = (new_value))
 #define gpr_tls_get(tls) ((tls)->value)
 
-#endif
+#endif /* NDEBUG */
+
+#endif /* GRPC_SUPPORT_TLS_GCC_H */
diff --git a/include/grpc/support/tls_msvc.h b/include/grpc/support/tls_msvc.h
index 987a514f03..751ec35f73 100644
--- a/include/grpc/support/tls_msvc.h
+++ b/include/grpc/support/tls_msvc.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_SUPPORT_TLS_GCC_H
-#define GRPC_SUPPORT_TLS_GCC_H
+#ifndef GRPC_SUPPORT_TLS_MSVC_H
+#define GRPC_SUPPORT_TLS_MSVC_H
 
 /* Thread local storage based on ms visual c compiler primitives.
    #include tls.h to use this - and see that file for documentation */
@@ -53,4 +53,4 @@ struct gpr_msvc_thread_local {
 #define gpr_tls_set(tls, new_value) (((tls)->value) = (new_value))
 #define gpr_tls_get(tls) ((tls)->value)
 
-#endif
+#endif /* GRPC_SUPPORT_TLS_MSVC_H */
diff --git a/include/grpc/support/tls_pthread.h b/include/grpc/support/tls_pthread.h
index 699ee6b1d3..93ba7822c3 100644
--- a/include/grpc/support/tls_pthread.h
+++ b/include/grpc/support/tls_pthread.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,4 +57,4 @@ intptr_t gpr_tls_set(struct gpr_pthread_thread_local *tls, intptr_t value);
 }
 #endif
 
-#endif
+#endif /* GRPC_SUPPORT_TLS_PTHREAD_H */