pre sync_stream.cc creation. Does not compile

10 files changed, 699 insertions, 536 deletions

diff --git a/include/grpc++/impl/codegen/async_stream.h b/include/grpc++/impl/codegen/async_stream.h
new file mode 100644
index 0000000000..b0410485f8
--- /dev/null
+++ b/include/grpc++/impl/codegen/async_stream.h
@@ -0,0 +1,462 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_ASYNC_STREAM_H
+#define GRPCXX_IMPL_CODEGEN_ASYNC_STREAM_H
+
+#include <grpc++/impl/codegen/channel_interface.h>
+#include <grpc++/impl/codegen/call.h>
+#include <grpc++/impl/codegen/service_type.h>
+#include <grpc++/impl/codegen/server_context.h>
+#include <grpc++/impl/codegen/status.h>
+
+namespace grpc {
+
+class CompletionQueue;
+
+/// Common interface for all client side asynchronous streaming.
+class ClientAsyncStreamingInterface {
+ public:
+  virtual ~ClientAsyncStreamingInterface() {}
+
+  /// Request notification of the reading of the initial metadata. Completion
+  /// will be notified by \a tag on the associated completion queue.
+  ///
+  /// \param[in] tag Tag identifying this request.
+  virtual void ReadInitialMetadata(void* tag) = 0;
+
+  /// Request notification completion.
+  ///
+  /// \param[out] status To be updated with the operation status.
+  /// \param[in] tag Tag identifying this request.
+  virtual void Finish(Status* status, void* tag) = 0;
+};
+
+/// An interface that yields a sequence of messages of type \a R.
+template <class R>
+class AsyncReaderInterface {
+ public:
+  virtual ~AsyncReaderInterface() {}
+
+  /// Read a message of type \a R into \a msg. Completion will be notified by \a
+  /// tag on the associated completion queue.
+  ///
+  /// \param[out] msg Where to eventually store the read message.
+  /// \param[in] tag The tag identifying the operation.
+  virtual void Read(R* msg, void* tag) = 0;
+};
+
+/// An interface that can be fed a sequence of messages of type \a W.
+template <class W>
+class AsyncWriterInterface {
+ public:
+  virtual ~AsyncWriterInterface() {}
+
+  /// Request the writing of \a msg with identifying tag \a tag.
+  ///
+  /// Only one write may be outstanding at any given time. This means that
+  /// after calling Write, one must wait to receive \a tag from the completion
+  /// queue BEFORE calling Write again.
+  ///
+  /// \param[in] msg The message to be written.
+  /// \param[in] tag The tag identifying the operation.
+  virtual void Write(const W& msg, void* tag) = 0;
+};
+
+template <class R>
+class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface,
+                                   public AsyncReaderInterface<R> {};
+
+template <class R>
+class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
+ public:
+  /// Create a stream and write the first request out.
+  template <class W>
+  ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
+                    const RpcMethod& method, ClientContext* context,
+                    const W& request, void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(init_ops_.SendMessage(request).ok());
+    init_ops_.ClientSendClose();
+    call_.PerformOps(&init_ops_);
+  }
+
+  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      read_ops_.RecvInitialMetadata(context_);
+    }
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
+    finish_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_ops_.RecvInitialMetadata(context_);
+    }
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  ClientContext* context_;
+  Call call_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose>
+      init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
+};
+
+/// Common interface for client side asynchronous writing.
+template <class W>
+class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface,
+                                   public AsyncWriterInterface<W> {
+ public:
+  /// Signal the client is done with the writes.
+  ///
+  /// \param[in] tag The tag identifying the operation.
+  virtual void WritesDone(void* tag) = 0;
+};
+
+template <class W>
+class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
+ public:
+  template <class R>
+  ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
+                    const RpcMethod& method, ClientContext* context,
+                    R* response, void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    finish_ops_.RecvMessage(response);
+
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&init_ops_);
+  }
+
+  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void WritesDone(void* tag) GRPC_OVERRIDE {
+    writes_done_ops_.set_output_tag(tag);
+    writes_done_ops_.ClientSendClose();
+    call_.PerformOps(&writes_done_ops_);
+  }
+
+  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
+    finish_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_ops_.RecvInitialMetadata(context_);
+    }
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  ClientContext* context_;
+  Call call_;
+  CallOpSet<CallOpSendInitialMetadata> init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpClientSendClose> writes_done_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
+            CallOpClientRecvStatus> finish_ops_;
+};
+
+/// Client-side interface for asynchronous bi-directional streaming.
+template <class W, class R>
+class ClientAsyncReaderWriterInterface : public ClientAsyncStreamingInterface,
+                                         public AsyncWriterInterface<W>,
+                                         public AsyncReaderInterface<R> {
+ public:
+  /// Signal the client is done with the writes.
+  ///
+  /// \param[in] tag The tag identifying the operation.
+  virtual void WritesDone(void* tag) = 0;
+};
+
+template <class W, class R>
+class ClientAsyncReaderWriter GRPC_FINAL
+    : public ClientAsyncReaderWriterInterface<W, R> {
+ public:
+  ClientAsyncReaderWriter(ChannelInterface* channel, CompletionQueue* cq,
+                          const RpcMethod& method, ClientContext* context,
+                          void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&init_ops_);
+  }
+
+  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      read_ops_.RecvInitialMetadata(context_);
+    }
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void WritesDone(void* tag) GRPC_OVERRIDE {
+    writes_done_ops_.set_output_tag(tag);
+    writes_done_ops_.ClientSendClose();
+    call_.PerformOps(&writes_done_ops_);
+  }
+
+  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
+    finish_ops_.set_output_tag(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_ops_.RecvInitialMetadata(context_);
+    }
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  ClientContext* context_;
+  Call call_;
+  CallOpSet<CallOpSendInitialMetadata> init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpClientSendClose> writes_done_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
+};
+
+template <class W, class R>
+class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
+                                     public AsyncReaderInterface<R> {
+ public:
+  explicit ServerAsyncReader(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Finish(const W& msg, const Status& status, void* tag) {
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // The response is dropped if the status is not OK.
+    if (status.ok()) {
+      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_,
+                                   finish_ops_.SendMessage(msg));
+    } else {
+      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    }
+    call_.PerformOps(&finish_ops_);
+  }
+
+  void FinishWithError(const Status& status, void* tag) {
+    GPR_ASSERT(!status.ok());
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+            CallOpServerSendStatus> finish_ops_;
+};
+
+template <class W>
+class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
+                                     public AsyncWriterInterface<W> {
+ public:
+  explicit ServerAsyncWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void Finish(const Status& status, void* tag) {
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
+};
+
+/// Server-side interface for asynchronous bi-directional streaming.
+template <class W, class R>
+class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
+                                           public AsyncWriterInterface<W>,
+                                           public AsyncReaderInterface<R> {
+ public:
+  explicit ServerAsyncReaderWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_ops_);
+  }
+
+  void Read(R* msg, void* tag) GRPC_OVERRIDE {
+    read_ops_.set_output_tag(tag);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
+  }
+
+  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
+    write_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
+  }
+
+  void Finish(const Status& status, void* tag) {
+    finish_ops_.set_output_tag(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
+  }
+
+ private:
+  friend class ::grpc::Server;
+
+  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_ASYNC_STREAM_H
diff --git a/include/grpc++/impl/codegen/config_protobuf.h b/include/grpc++/impl/codegen/config_protobuf.h
new file mode 100644
index 0000000000..f1b6beaca7
--- /dev/null
+++ b/include/grpc++/impl/codegen/config_protobuf.h
@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H
+#define GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H
+
+#ifndef GRPC_CUSTOM_PROTOBUF_INT64
+#include <google/protobuf/stubs/common.h>
+#define GRPC_CUSTOM_PROTOBUF_INT64 ::google::protobuf::int64
+#endif
+
+#ifndef GRPC_CUSTOM_MESSAGE
+#include <google/protobuf/message.h>
+#define GRPC_CUSTOM_MESSAGE ::google::protobuf::Message
+#endif
+
+#ifndef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#define GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM \
+  ::google::protobuf::io::ZeroCopyOutputStream
+#define GRPC_CUSTOM_ZEROCOPYINPUTSTREAM \
+  ::google::protobuf::io::ZeroCopyInputStream
+#define GRPC_CUSTOM_CODEDINPUTSTREAM ::google::protobuf::io::CodedInputStream
+#endif
+
+namespace grpc {
+namespace protobuf {
+
+typedef GRPC_CUSTOM_MESSAGE Message;
+typedef GRPC_CUSTOM_PROTOBUF_INT64 int64;
+
+namespace io {
+typedef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM ZeroCopyOutputStream;
+typedef GRPC_CUSTOM_ZEROCOPYINPUTSTREAM ZeroCopyInputStream;
+typedef GRPC_CUSTOM_CODEDINPUTSTREAM CodedInputStream;
+}  // namespace io
+
+}  // namespace protobuf
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H
diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h
new file mode 100644
index 0000000000..9b81853652
--- /dev/null
+++ b/include/grpc++/impl/codegen/proto_utils.h
@@ -0,0 +1,76 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_PROTO_UTILS_H
+#define GRPCXX_IMPL_CODEGEN_PROTO_UTILS_H
+
+#include <type_traits>
+
+#include <grpc/impl/codegen/byte_buffer.h>
+#include <grpc++/impl/codegen/serialization_traits.h>
+#include <grpc++/impl/codegen/config_protobuf.h>
+#include <grpc++/impl/codegen/status.h>
+
+namespace grpc {
+
+// Serialize the msg into a buffer created inside the function. The caller
+// should destroy the returned buffer when done with it. If serialization fails,
+// false is returned and buffer is left unchanged.
+Status SerializeProto(const grpc::protobuf::Message& msg,
+                      grpc_byte_buffer** buffer);
+
+// The caller keeps ownership of buffer and msg.
+Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
+                        int max_message_size);
+
+template <class T>
+class SerializationTraits<T, typename std::enable_if<std::is_base_of<
+                                 grpc::protobuf::Message, T>::value>::type> {
+ public:
+  static Status Serialize(const grpc::protobuf::Message& msg,
+                          grpc_byte_buffer** buffer, bool* own_buffer) {
+    *own_buffer = true;
+    return SerializeProto(msg, buffer);
+  }
+  static Status Deserialize(grpc_byte_buffer* buffer,
+                            grpc::protobuf::Message* msg,
+                            int max_message_size) {
+    auto status = DeserializeProto(buffer, msg, max_message_size);
+    grpc_byte_buffer_destroy(buffer);
+    return status;
+  }
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_PROTO_UTILS_H
diff --git a/include/grpc++/impl/codegen/stub_options.h b/include/grpc++/impl/codegen/stub_options.h
new file mode 100644
index 0000000000..8e966a8dbf
--- /dev/null
+++ b/include/grpc++/impl/codegen/stub_options.h
@@ -0,0 +1,43 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_STUB_OPTIONS_H
+#define GRPCXX_IMPL_CODEGEN_STUB_OPTIONS_H
+
+namespace grpc {
+
+class StubOptions {};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_STUB_OPTIONS_H
diff --git a/include/grpc++/impl/proto_utils.h b/include/grpc++/impl/proto_utils.h
index 283e33486d..7110a4d9ac 100644
--- a/include/grpc++/impl/proto_utils.h
+++ b/include/grpc++/impl/proto_utils.h
@@ -34,43 +34,6 @@
 #ifndef GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
 #define GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
 
-#include <type_traits>
-
-#include <grpc/grpc.h>
-#include <grpc++/impl/serialization_traits.h>
-#include <grpc++/support/config_protobuf.h>
-#include <grpc++/support/status.h>
-
-namespace grpc {
-
-// Serialize the msg into a buffer created inside the function. The caller
-// should destroy the returned buffer when done with it. If serialization fails,
-// false is returned and buffer is left unchanged.
-Status SerializeProto(const grpc::protobuf::Message& msg,
-                      grpc_byte_buffer** buffer);
-
-// The caller keeps ownership of buffer and msg.
-Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
-                        int max_message_size);
-
-template <class T>
-class SerializationTraits<T, typename std::enable_if<std::is_base_of<
-                                 grpc::protobuf::Message, T>::value>::type> {
- public:
-  static Status Serialize(const grpc::protobuf::Message& msg,
-                          grpc_byte_buffer** buffer, bool* own_buffer) {
-    *own_buffer = true;
-    return SerializeProto(msg, buffer);
-  }
-  static Status Deserialize(grpc_byte_buffer* buffer,
-                            grpc::protobuf::Message* msg,
-                            int max_message_size) {
-    auto status = DeserializeProto(buffer, msg, max_message_size);
-    grpc_byte_buffer_destroy(buffer);
-    return status;
-  }
-};
-
-}  // namespace grpc
+#include <grpc++/impl/codegen/proto_utils.h>
 
 #endif  // GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
diff --git a/include/grpc++/support/async_stream.h b/include/grpc++/support/async_stream.h
index 5a811f9217..88147ea9d6 100644
--- a/include/grpc++/support/async_stream.h
+++ b/include/grpc++/support/async_stream.h
@@ -34,429 +34,6 @@
 #ifndef GRPCXX_SUPPORT_ASYNC_STREAM_H
 #define GRPCXX_SUPPORT_ASYNC_STREAM_H
 
-#include <grpc++/impl/codegen/channel_interface.h>
-#include <grpc++/impl/codegen/call.h>
-#include <grpc++/impl/codegen/service_type.h>
-#include <grpc++/impl/codegen/server_context.h>
-#include <grpc++/impl/codegen/status.h>
-
-namespace grpc {
-
-class CompletionQueue;
-
-/// Common interface for all client side asynchronous streaming.
-class ClientAsyncStreamingInterface {
- public:
-  virtual ~ClientAsyncStreamingInterface() {}
-
-  /// Request notification of the reading of the initial metadata. Completion
-  /// will be notified by \a tag on the associated completion queue.
-  ///
-  /// \param[in] tag Tag identifying this request.
-  virtual void ReadInitialMetadata(void* tag) = 0;
-
-  /// Request notification completion.
-  ///
-  /// \param[out] status To be updated with the operation status.
-  /// \param[in] tag Tag identifying this request.
-  virtual void Finish(Status* status, void* tag) = 0;
-};
-
-/// An interface that yields a sequence of messages of type \a R.
-template <class R>
-class AsyncReaderInterface {
- public:
-  virtual ~AsyncReaderInterface() {}
-
-  /// Read a message of type \a R into \a msg. Completion will be notified by \a
-  /// tag on the associated completion queue.
-  ///
-  /// \param[out] msg Where to eventually store the read message.
-  /// \param[in] tag The tag identifying the operation.
-  virtual void Read(R* msg, void* tag) = 0;
-};
-
-/// An interface that can be fed a sequence of messages of type \a W.
-template <class W>
-class AsyncWriterInterface {
- public:
-  virtual ~AsyncWriterInterface() {}
-
-  /// Request the writing of \a msg with identifying tag \a tag.
-  ///
-  /// Only one write may be outstanding at any given time. This means that
-  /// after calling Write, one must wait to receive \a tag from the completion
-  /// queue BEFORE calling Write again.
-  ///
-  /// \param[in] msg The message to be written.
-  /// \param[in] tag The tag identifying the operation.
-  virtual void Write(const W& msg, void* tag) = 0;
-};
-
-template <class R>
-class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface,
-                                   public AsyncReaderInterface<R> {};
-
-template <class R>
-class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
- public:
-  /// Create a stream and write the first request out.
-  template <class W>
-  ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
-                    const RpcMethod& method, ClientContext* context,
-                    const W& request, void* tag)
-      : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_ops_.set_output_tag(tag);
-    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
-    // TODO(ctiller): don't assert
-    GPR_ASSERT(init_ops_.SendMessage(request).ok());
-    init_ops_.ClientSendClose();
-    call_.PerformOps(&init_ops_);
-  }
-
-  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.RecvInitialMetadata(context_);
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      read_ops_.RecvInitialMetadata(context_);
-    }
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      finish_ops_.RecvInitialMetadata(context_);
-    }
-    finish_ops_.ClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  ClientContext* context_;
-  Call call_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose>
-      init_ops_;
-  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
-};
-
-/// Common interface for client side asynchronous writing.
-template <class W>
-class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface,
-                                   public AsyncWriterInterface<W> {
- public:
-  /// Signal the client is done with the writes.
-  ///
-  /// \param[in] tag The tag identifying the operation.
-  virtual void WritesDone(void* tag) = 0;
-};
-
-template <class W>
-class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
- public:
-  template <class R>
-  ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
-                    const RpcMethod& method, ClientContext* context,
-                    R* response, void* tag)
-      : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    finish_ops_.RecvMessage(response);
-
-    init_ops_.set_output_tag(tag);
-    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
-    call_.PerformOps(&init_ops_);
-  }
-
-  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.RecvInitialMetadata(context_);
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void WritesDone(void* tag) GRPC_OVERRIDE {
-    writes_done_ops_.set_output_tag(tag);
-    writes_done_ops_.ClientSendClose();
-    call_.PerformOps(&writes_done_ops_);
-  }
-
-  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      finish_ops_.RecvInitialMetadata(context_);
-    }
-    finish_ops_.ClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  ClientContext* context_;
-  Call call_;
-  CallOpSet<CallOpSendInitialMetadata> init_ops_;
-  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
-  CallOpSet<CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpClientSendClose> writes_done_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
-            CallOpClientRecvStatus> finish_ops_;
-};
-
-/// Client-side interface for asynchronous bi-directional streaming.
-template <class W, class R>
-class ClientAsyncReaderWriterInterface : public ClientAsyncStreamingInterface,
-                                         public AsyncWriterInterface<W>,
-                                         public AsyncReaderInterface<R> {
- public:
-  /// Signal the client is done with the writes.
-  ///
-  /// \param[in] tag The tag identifying the operation.
-  virtual void WritesDone(void* tag) = 0;
-};
-
-template <class W, class R>
-class ClientAsyncReaderWriter GRPC_FINAL
-    : public ClientAsyncReaderWriterInterface<W, R> {
- public:
-  ClientAsyncReaderWriter(ChannelInterface* channel, CompletionQueue* cq,
-                          const RpcMethod& method, ClientContext* context,
-                          void* tag)
-      : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_ops_.set_output_tag(tag);
-    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
-    call_.PerformOps(&init_ops_);
-  }
-
-  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!context_->initial_metadata_received_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.RecvInitialMetadata(context_);
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      read_ops_.RecvInitialMetadata(context_);
-    }
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void WritesDone(void* tag) GRPC_OVERRIDE {
-    writes_done_ops_.set_output_tag(tag);
-    writes_done_ops_.ClientSendClose();
-    call_.PerformOps(&writes_done_ops_);
-  }
-
-  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      finish_ops_.RecvInitialMetadata(context_);
-    }
-    finish_ops_.ClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  ClientContext* context_;
-  Call call_;
-  CallOpSet<CallOpSendInitialMetadata> init_ops_;
-  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpClientSendClose> writes_done_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
-};
-
-template <class W, class R>
-class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
-                                     public AsyncReaderInterface<R> {
- public:
-  explicit ServerAsyncReader(ServerContext* ctx)
-      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
-
-  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Finish(const W& msg, const Status& status, void* tag) {
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    // The response is dropped if the status is not OK.
-    if (status.ok()) {
-      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_,
-                                   finish_ops_.SendMessage(msg));
-    } else {
-      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    }
-    call_.PerformOps(&finish_ops_);
-  }
-
-  void FinishWithError(const Status& status, void* tag) {
-    GPR_ASSERT(!status.ok());
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
-
-  Call call_;
-  ServerContext* ctx_;
-  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-            CallOpServerSendStatus> finish_ops_;
-};
-
-template <class W>
-class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
-                                     public AsyncWriterInterface<W> {
- public:
-  explicit ServerAsyncWriter(ServerContext* ctx)
-      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
-
-  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void Finish(const Status& status, void* tag) {
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
-
-  Call call_;
-  ServerContext* ctx_;
-  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
-};
-
-/// Server-side interface for asynchronous bi-directional streaming.
-template <class W, class R>
-class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
-                                           public AsyncWriterInterface<W>,
-                                           public AsyncReaderInterface<R> {
- public:
-  explicit ServerAsyncReaderWriter(ServerContext* ctx)
-      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
-
-  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    // TODO(ctiller): don't assert
-    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void Finish(const Status& status, void* tag) {
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  friend class ::grpc::Server;
-
-  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
-
-  Call call_;
-  ServerContext* ctx_;
-  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
-};
-
-}  // namespace grpc
+#include <grpc++/impl/codegen/async_stream.h>
 
 #endif  // GRPCXX_SUPPORT_ASYNC_STREAM_H
diff --git a/include/grpc++/support/async_unary_call.h b/include/grpc++/support/async_unary_call.h
index 55ca56b6a6..3a601e063d 100644
--- a/include/grpc++/support/async_unary_call.h
+++ b/include/grpc++/support/async_unary_call.h
@@ -34,17 +34,18 @@
 #ifndef GRPCXX_SUPPORT_ASYNC_UNARY_CALL_H
 #define GRPCXX_SUPPORT_ASYNC_UNARY_CALL_H
 
-#include <grpc/support/log.h>
+#include <grpc/impl/codegen/log.h>
 #include <grpc++/impl/codegen/channel_interface.h>
-#include <grpc++/client_context.h>
-#include <grpc++/completion_queue.h>
-#include <grpc++/server_context.h>
-#include <grpc++/impl/call.h>
-#include <grpc++/impl/service_type.h>
-#include <grpc++/support/status.h>
+#include <grpc++/impl/codegen/client_context.h>
+#include <grpc++/impl/codegen/server_context.h>
+#include <grpc++/impl/codegen/call.h>
+#include <grpc++/impl/codegen/service_type.h>
+#include <grpc++/impl/codegen/status.h>
 
 namespace grpc {
 
+class CompletionQueue;
+
 template <class R>
 class ClientAsyncResponseReaderInterface {
  public:
diff --git a/include/grpc++/support/config_protobuf.h b/include/grpc++/support/config_protobuf.h
index 8235590d41..3e7f169652 100644
--- a/include/grpc++/support/config_protobuf.h
+++ b/include/grpc++/support/config_protobuf.h
@@ -34,39 +34,6 @@
 #ifndef GRPCXX_SUPPORT_CONFIG_PROTOBUF_H
 #define GRPCXX_SUPPORT_CONFIG_PROTOBUF_H
 
-#ifndef GRPC_CUSTOM_PROTOBUF_INT64
-#include <google/protobuf/stubs/common.h>
-#define GRPC_CUSTOM_PROTOBUF_INT64 ::google::protobuf::int64
-#endif
-
-#ifndef GRPC_CUSTOM_MESSAGE
-#include <google/protobuf/message.h>
-#define GRPC_CUSTOM_MESSAGE ::google::protobuf::Message
-#endif
-
-#ifndef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream.h>
-#define GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM \
-  ::google::protobuf::io::ZeroCopyOutputStream
-#define GRPC_CUSTOM_ZEROCOPYINPUTSTREAM \
-  ::google::protobuf::io::ZeroCopyInputStream
-#define GRPC_CUSTOM_CODEDINPUTSTREAM ::google::protobuf::io::CodedInputStream
-#endif
-
-namespace grpc {
-namespace protobuf {
-
-typedef GRPC_CUSTOM_MESSAGE Message;
-typedef GRPC_CUSTOM_PROTOBUF_INT64 int64;
-
-namespace io {
-typedef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM ZeroCopyOutputStream;
-typedef GRPC_CUSTOM_ZEROCOPYINPUTSTREAM ZeroCopyInputStream;
-typedef GRPC_CUSTOM_CODEDINPUTSTREAM CodedInputStream;
-}  // namespace io
-
-}  // namespace protobuf
-}  // namespace grpc
+#include <grpc++/impl/codegen/config_protobuf.h>
 
 #endif  // GRPCXX_SUPPORT_CONFIG_PROTOBUF_H
diff --git a/include/grpc++/support/stub_options.h b/include/grpc++/support/stub_options.h
index 973aa9bc83..17a13c357e 100644
--- a/include/grpc++/support/stub_options.h
+++ b/include/grpc++/support/stub_options.h
@@ -34,10 +34,6 @@
 #ifndef GRPCXX_SUPPORT_STUB_OPTIONS_H
 #define GRPCXX_SUPPORT_STUB_OPTIONS_H
 
-namespace grpc {
-
-class StubOptions {};
-
-}  // namespace grpc
+#include <grpc++/impl/codegen/stub_options.h>
 
 #endif  // GRPCXX_SUPPORT_STUB_OPTIONS_H
diff --git a/include/grpc++/support/sync_stream.h b/include/grpc++/support/sync_stream.h
index 3f3acc5308..77066ada62 100644
--- a/include/grpc++/support/sync_stream.h
+++ b/include/grpc++/support/sync_stream.h
@@ -34,18 +34,18 @@
 #ifndef GRPCXX_SUPPORT_SYNC_STREAM_H
 #define GRPCXX_SUPPORT_SYNC_STREAM_H
 
-#include <grpc/support/log.h>
-#include <grpc++/channel.h>
+#include <grpc/impl/codegen/log.h>
 #include <grpc++/impl/codegen/channel_interface.h>
-#include <grpc++/client_context.h>
-#include <grpc++/completion_queue.h>
-#include <grpc++/impl/call.h>
-#include <grpc++/impl/service_type.h>
-#include <grpc++/server_context.h>
-#include <grpc++/support/status.h>
+#include <grpc++/impl/codegen/client_context.h>
+#include <grpc++/impl/codegen/call.h>
+#include <grpc++/impl/codegen/service_type.h>
+#include <grpc++/impl/codegen/server_context.h>
+#include <grpc++/impl/codegen/status.h>
 
 namespace grpc {
 
+class CompletionQueue;
+
 /// Common interface for all synchronous client side streaming.
 class ClientStreamingInterface {
  public:
@@ -121,7 +121,9 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
   template <class W>
   ClientReader(ChannelInterface* channel, const RpcMethod& method,
                ClientContext* context, const W& request)
-      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+      : context_(context),
+        cq_(new CompletionQueue),
+        call_(channel->CreateCall(method, context, cq_.get())) {
     CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
               CallOpClientSendClose> ops;
     ops.SendInitialMetadata(context->send_initial_metadata_);
@@ -129,7 +131,7 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
     GPR_ASSERT(ops.SendMessage(request).ok());
     ops.ClientSendClose();
     call_.PerformOps(&ops);
-    cq_.Pluck(&ops);
+    cq_->Pluck(&ops);
   }
 
   void WaitForInitialMetadata() GRPC_OVERRIDE {
@@ -138,7 +140,7 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
     CallOpSet<CallOpRecvInitialMetadata> ops;
     ops.RecvInitialMetadata(context_);
     call_.PerformOps(&ops);
-    cq_.Pluck(&ops);  /// status ignored
+    cq_->Pluck(&ops);  /// status ignored
   }
 
   bool Read(R* msg) GRPC_OVERRIDE {
@@ -148,7 +150,7 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
     }
     ops.RecvMessage(msg);
     call_.PerformOps(&ops);
-    return cq_.Pluck(&ops) && ops.got_message;
+    return cq_->Pluck(&ops) && ops.got_message;
   }
 
   Status Finish() GRPC_OVERRIDE {
@@ -156,13 +158,13 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
     Status status;
     ops.ClientRecvStatus(context_, &status);
     call_.PerformOps(&ops);
-    GPR_ASSERT(cq_.Pluck(&ops));
+    GPR_ASSERT(cq_->Pluck(&ops));
     return status;
   }
 
  private:
   ClientContext* context_;
-  CompletionQueue cq_;
+  std::unique_ptr<CompletionQueue> cq_;
   Call call_;
 };
 
@@ -185,13 +187,15 @@ class ClientWriter : public ClientWriterInterface<W> {
   template <class R>
   ClientWriter(ChannelInterface* channel, const RpcMethod& method,
                ClientContext* context, R* response)
-      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+      : context_(context),
+        cq_(new CompletionQueue),
+        call_(channel->CreateCall(method, context, cq_.get())) {
     finish_ops_.RecvMessage(response);
 
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(context->send_initial_metadata_);
     call_.PerformOps(&ops);
-    cq_.Pluck(&ops);
+    cq_->Pluck(&ops);
   }
 
   using WriterInterface<W>::Write;
@@ -201,14 +205,14 @@ class ClientWriter : public ClientWriterInterface<W> {
       return false;
     }
     call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
+    return cq_->Pluck(&ops);
   }
 
   bool WritesDone() GRPC_OVERRIDE {
     CallOpSet<CallOpClientSendClose> ops;
     ops.ClientSendClose();
     call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
+    return cq_->Pluck(&ops);
   }
 
   /// Read the final response and wait for the final status.
@@ -216,14 +220,14 @@ class ClientWriter : public ClientWriterInterface<W> {
     Status status;
     finish_ops_.ClientRecvStatus(context_, &status);
     call_.PerformOps(&finish_ops_);
-    GPR_ASSERT(cq_.Pluck(&finish_ops_));
+    GPR_ASSERT(cq_->Pluck(&finish_ops_));
     return status;
   }
 
  private:
   ClientContext* context_;
   CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_;
-  CompletionQueue cq_;
+  std::unique_ptr<CompletionQueue> cq_;
   Call call_;
 };
 
@@ -251,11 +255,13 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
   /// Blocking create a stream.
   ClientReaderWriter(ChannelInterface* channel, const RpcMethod& method,
                      ClientContext* context)
-      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+      : context_(context),
+        cq_(new CompletionQueue),
+        call_(channel->CreateCall(method, context, cq_.get())) {
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(context->send_initial_metadata_);
     call_.PerformOps(&ops);
-    cq_.Pluck(&ops);
+    cq_->Pluck(&ops);
   }
 
   void WaitForInitialMetadata() GRPC_OVERRIDE {
@@ -264,7 +270,7 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
     CallOpSet<CallOpRecvInitialMetadata> ops;
     ops.RecvInitialMetadata(context_);
     call_.PerformOps(&ops);
-    cq_.Pluck(&ops);  // status ignored
+    cq_->Pluck(&ops);  // status ignored
   }
 
   bool Read(R* msg) GRPC_OVERRIDE {
@@ -274,7 +280,7 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
     }
     ops.RecvMessage(msg);
     call_.PerformOps(&ops);
-    return cq_.Pluck(&ops) && ops.got_message;
+    return cq_->Pluck(&ops) && ops.got_message;
   }
 
   using WriterInterface<W>::Write;
@@ -282,14 +288,14 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
     CallOpSet<CallOpSendMessage> ops;
     if (!ops.SendMessage(msg, options).ok()) return false;
     call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
+    return cq_->Pluck(&ops);
   }
 
   bool WritesDone() GRPC_OVERRIDE {
     CallOpSet<CallOpClientSendClose> ops;
     ops.ClientSendClose();
     call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
+    return cq_->Pluck(&ops);
   }
 
   Status Finish() GRPC_OVERRIDE {
@@ -297,13 +303,13 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
     Status status;
     ops.ClientRecvStatus(context_, &status);
     call_.PerformOps(&ops);
-    GPR_ASSERT(cq_.Pluck(&ops));
+    GPR_ASSERT(cq_->Pluck(&ops));
     return status;
   }
 
  private:
   ClientContext* context_;
-  CompletionQueue cq_;
+  std::unique_ptr<CompletionQueue> cq_;
   Call call_;
 };