Merge branch 'master' of github.com:grpc/grpc into compression-accept-encoding

43 files changed, 2025 insertions, 1082 deletions

diff --git a/include/grpc++/channel.h b/include/grpc++/channel.h
new file mode 100644
index 0000000000..60c816d58a
--- /dev/null
+++ b/include/grpc++/channel.h
@@ -0,0 +1,139 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_CHANNEL_H
+#define GRPCXX_CHANNEL_H
+
+#include <memory>
+
+#include <grpc/grpc.h>
+#include <grpc++/impl/call.h>
+#include <grpc++/impl/grpc_library.h>
+#include <grpc++/support/config.h>
+
+struct grpc_channel;
+
+namespace grpc {
+class CallOpSetInterface;
+class ChannelArguments;
+class CompletionQueue;
+class Credentials;
+class SecureCredentials;
+
+template <class R>
+class ClientReader;
+template <class W>
+class ClientWriter;
+template <class R, class W>
+class ClientReaderWriter;
+template <class R>
+class ClientAsyncReader;
+template <class W>
+class ClientAsyncWriter;
+template <class R, class W>
+class ClientAsyncReaderWriter;
+template <class R>
+class ClientAsyncResponseReader;
+
+/// Channels represent a connection to an endpoint. Created by \a CreateChannel.
+class Channel GRPC_FINAL : public GrpcLibrary,
+                           public CallHook,
+                           public std::enable_shared_from_this<Channel> {
+ public:
+  ~Channel();
+
+  /// Get the current channel state. If the channel is in IDLE and
+  /// \a try_to_connect is set to true, try to connect.
+  grpc_connectivity_state GetState(bool try_to_connect);
+
+  /// Return the \a tag on \a cq when the channel state is changed or \a
+  /// deadline expires. \a GetState needs to called to get the current state.
+  template <typename T>
+  void NotifyOnStateChange(grpc_connectivity_state last_observed, T deadline,
+                           CompletionQueue* cq, void* tag) {
+    TimePoint<T> deadline_tp(deadline);
+    NotifyOnStateChangeImpl(last_observed, deadline_tp.raw_time(), cq, tag);
+  }
+
+  /// Blocking wait for channel state change or \a deadline expiration.
+  /// \a GetState needs to called to get the current state.
+  template <typename T>
+  bool WaitForStateChange(grpc_connectivity_state last_observed, T deadline) {
+    TimePoint<T> deadline_tp(deadline);
+    return WaitForStateChangeImpl(last_observed, deadline_tp.raw_time());
+  }
+
+ private:
+  template <class R>
+  friend class ::grpc::ClientReader;
+  template <class W>
+  friend class ::grpc::ClientWriter;
+  template <class R, class W>
+  friend class ::grpc::ClientReaderWriter;
+  template <class R>
+  friend class ::grpc::ClientAsyncReader;
+  template <class W>
+  friend class ::grpc::ClientAsyncWriter;
+  template <class R, class W>
+  friend class ::grpc::ClientAsyncReaderWriter;
+  template <class R>
+  friend class ::grpc::ClientAsyncResponseReader;
+  template <class InputMessage, class OutputMessage>
+  friend Status BlockingUnaryCall(Channel* channel, const RpcMethod& method,
+                                  ClientContext* context,
+                                  const InputMessage& request,
+                                  OutputMessage* result);
+  friend class ::grpc::RpcMethod;
+  friend std::shared_ptr<Channel> CreateChannelInternal(
+      const grpc::string& host, grpc_channel* c_channel);
+
+  Channel(const grpc::string& host, grpc_channel* c_channel);
+
+  Call CreateCall(const RpcMethod& method, ClientContext* context,
+                  CompletionQueue* cq);
+  void PerformOpsOnCall(CallOpSetInterface* ops, Call* call);
+  void* RegisterMethod(const char* method);
+
+  void NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
+                               gpr_timespec deadline, CompletionQueue* cq,
+                               void* tag);
+  bool WaitForStateChangeImpl(grpc_connectivity_state last_observed,
+                              gpr_timespec deadline);
+
+  const grpc::string host_;
+  grpc_channel* const c_channel_;  // owned
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_CHANNEL_H
diff --git a/include/grpc++/channel_interface.h b/include/grpc++/channel_interface.h
deleted file mode 100644
index 4176cded7b..0000000000
--- a/include/grpc++/channel_interface.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPCXX_CHANNEL_INTERFACE_H
-#define GRPCXX_CHANNEL_INTERFACE_H
-
-#include <memory>
-
-#include <grpc/grpc.h>
-#include <grpc++/status.h>
-#include <grpc++/impl/call.h>
-
-struct grpc_call;
-
-namespace grpc {
-class Call;
-class CallOpBuffer;
-class ClientContext;
-class CompletionQueue;
-class RpcMethod;
-
-class ChannelInterface : public CallHook,
-                         public std::enable_shared_from_this<ChannelInterface> {
- public:
-  virtual ~ChannelInterface() {}
-
-  virtual void* RegisterMethod(const char* method_name) = 0;
-  virtual Call CreateCall(const RpcMethod& method, ClientContext* context,
-                          CompletionQueue* cq) = 0;
-
-  // Get the current channel state. If the channel is in IDLE and try_to_connect
-  // is set to true, try to connect.
-  virtual grpc_connectivity_state GetState(bool try_to_connect) = 0;
-
-  // Return the tag on cq when the channel state is changed or deadline expires.
-  // GetState needs to called to get the current state.
-  template <typename T>
-  void NotifyOnStateChange(grpc_connectivity_state last_observed, T deadline,
-                           CompletionQueue* cq, void* tag) {
-    TimePoint<T> deadline_tp(deadline);
-    NotifyOnStateChangeImpl(last_observed, deadline_tp.raw_time(), cq, tag);
-  }
-
-  // Blocking wait for channel state change or deadline expiration.
-  // GetState needs to called to get the current state.
-  template <typename T>
-  bool WaitForStateChange(grpc_connectivity_state last_observed, T deadline) {
-    TimePoint<T> deadline_tp(deadline);
-    return WaitForStateChangeImpl(last_observed, deadline_tp.raw_time());
-  }
-
- private:
-  virtual void NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
-                                       gpr_timespec deadline,
-                                       CompletionQueue* cq, void* tag) = 0;
-  virtual bool WaitForStateChangeImpl(grpc_connectivity_state last_observed,
-                                      gpr_timespec deadline) = 0;
-};
-
-}  // namespace grpc
-
-#endif  // GRPCXX_CHANNEL_INTERFACE_H
diff --git a/include/grpc++/client_context.h b/include/grpc++/client_context.h
index 8de2ba4877..7046f939e5 100644
--- a/include/grpc++/client_context.h
+++ b/include/grpc++/client_context.h
@@ -31,6 +31,21 @@
  *
  */
 
+/// A ClientContext allows the person implementing a service client to:
+///
+/// - Add custom metadata key-value pairs that will propagated to the server
+/// side.
+/// - Control call settings such as compression and authentication.
+/// - Initial and trailing metadata coming from the server.
+/// - Get performance metrics (ie, census).
+///
+/// Context settings are only relevant to the call they are invoked with, that
+/// is to say, they aren't sticky. Some of these settings, such as the
+/// compression options, can be made persistant at channel construction time
+/// (see \a grpc::CreateCustomChannel).
+///
+/// \warning ClientContext instances should \em not be reused across rpcs.
+
 #ifndef GRPCXX_CLIENT_CONTEXT_H
 #define GRPCXX_CLIENT_CONTEXT_H
 
@@ -42,16 +57,17 @@
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
-#include <grpc++/auth_context.h>
-#include <grpc++/config.h>
-#include <grpc++/status.h>
-#include <grpc++/time.h>
+#include <grpc++/security/auth_context.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
+#include <grpc++/support/string_ref.h>
+#include <grpc++/support/time.h>
 
 struct census_context;
 
 namespace grpc {
 
-class ChannelInterface;
+class Channel;
 class CompletionQueue;
 class Credentials;
 class RpcMethod;
@@ -71,6 +87,11 @@ template <class R>
 class ClientAsyncResponseReader;
 class ServerContext;
 
+/// Options for \a ClientContext::FromServerContext specifying which traits from
+/// the \a ServerContext to propagate (copy) from it into a new \a
+/// ClientContext.
+///
+/// \see ClientContext::FromServerContext
 class PropagationOptions {
  public:
   PropagationOptions() : propagate_(GRPC_PROPAGATE_DEFAULTS) {}
@@ -130,24 +151,66 @@ class ClientContext {
   ClientContext();
   ~ClientContext();
 
-  /// Create a new ClientContext that propagates some or all of its attributes
+  /// Create a new \a ClientContext as a child of an incoming server call,
+  /// according to \a options (\see PropagationOptions).
+  ///
+  /// \param server_context The source server context to use as the basis for
+  /// constructing the client context.
+  /// \param options The options controlling what to copy from the \a
+  /// server_context.
+  ///
+  /// \return A newly constructed \a ClientContext instance based on \a
+  /// server_context, with traits propagated (copied) according to \a options.
   static std::unique_ptr<ClientContext> FromServerContext(
       const ServerContext& server_context,
       PropagationOptions options = PropagationOptions());
 
+  /// Add the (\a meta_key, \a meta_value) pair to the metadata associated with
+  /// a client call. These are made available at the server side by the \a
+  /// grpc::ServerContext::client_metadata() method.
+  ///
+  /// \warning This method should only be called before invoking the rpc.
+  ///
+  /// \param meta_key The metadata key. If \a meta_value is binary data, it must
+  /// end in "-bin".
+  /// \param meta_value The metadata value. If its value is binary, it must be
+  /// base64-encoding (see https://tools.ietf.org/html/rfc4648#section-4) and \a
+  /// meta_key must end in "-bin".
   void AddMetadata(const grpc::string& meta_key,
                    const grpc::string& meta_value);
 
-  const std::multimap<grpc::string, grpc::string>& GetServerInitialMetadata() {
+  /// Return a collection of initial metadata key-value pairs. Note that keys
+  /// may happen more than once (ie, a \a std::multimap is returned).
+  ///
+  /// \warning This method should only be called after initial metadata has been
+  /// received. For streaming calls, see \a
+  /// ClientReaderInterface::WaitForInitialMetadata().
+  ///
+  /// \return A multimap of initial metadata key-value pairs from the server.
+  const std::multimap<grpc::string_ref, grpc::string_ref>&
+  GetServerInitialMetadata() {
     GPR_ASSERT(initial_metadata_received_);
     return recv_initial_metadata_;
   }
 
-  const std::multimap<grpc::string, grpc::string>& GetServerTrailingMetadata() {
+  /// Return a collection of trailing metadata key-value pairs. Note that keys
+  /// may happen more than once (ie, a \a std::multimap is returned).
+  ///
+  /// \warning This method is only callable once the stream has finished.
+  ///
+  /// \return A multimap of metadata trailing key-value pairs from the server.
+  const std::multimap<grpc::string_ref, grpc::string_ref>&
+  GetServerTrailingMetadata() {
     // TODO(yangg) check finished
     return trailing_metadata_;
   }
 
+  /// Set the deadline for the client call.
+  ///
+  /// \warning This method should only be called before invoking the rpc.
+  ///
+  /// \param deadline the deadline for the client call. Units are determined by
+  /// the type used.
   template <typename T>
   void set_deadline(const T& deadline) {
     TimePoint<T> deadline_tp(deadline);
@@ -155,40 +218,65 @@ class ClientContext {
   }
 
 #ifndef GRPC_CXX0X_NO_CHRONO
+  /// Return the deadline for the client call.
   std::chrono::system_clock::time_point deadline() {
     return Timespec2Timepoint(deadline_);
   }
 #endif  // !GRPC_CXX0X_NO_CHRONO
 
+  /// Return a \a gpr_timespec representation of the client call's deadline.
   gpr_timespec raw_deadline() { return deadline_; }
 
+  /// Set the per call authority header (see
+  /// https://tools.ietf.org/html/rfc7540#section-8.1.2.3).
   void set_authority(const grpc::string& authority) { authority_ = authority; }
 
-  // Set credentials for the rpc.
+  /// Return the authentication context for this client call.
+  ///
+  /// \see grpc::AuthContext.
+  std::shared_ptr<const AuthContext> auth_context() const;
+
+  /// Set credentials for the client call.
+  ///
+  /// A credentials object encapsulates all the state needed by a client to
+  /// authenticate with a server and make various assertions, e.g., about the
+  /// client’s identity, role, or whether it is authorized to make a particular
+  /// call.
+  ///
+  /// \see  https://github.com/grpc/grpc/blob/master/doc/grpc-auth-support.md
   void set_credentials(const std::shared_ptr<Credentials>& creds) {
     creds_ = creds;
   }
 
+  /// Return the compression algorithm to be used by the client call.
   grpc_compression_algorithm compression_algorithm() const {
     return compression_algorithm_;
   }
 
+  /// Set \a algorithm to be the compression algorithm used for the client call.
+  ///
+  /// \param algorith The compression algorithm used for the client call.
   void set_compression_algorithm(grpc_compression_algorithm algorithm);
 
-  std::shared_ptr<const AuthContext> auth_context() const;
-
-  // Return the peer uri in a string.
-  // WARNING: this value is never authenticated or subject to any security
-  // related code. It must not be used for any authentication related
-  // functionality. Instead, use auth_context.
+  /// Return the peer uri in a string.
+  ///
+  /// \warning This value is never authenticated or subject to any security
+  /// related code. It must not be used for any authentication related
+  /// functionality. Instead, use auth_context.
+  ///
+  /// \return The call's peer URI.
   grpc::string peer() const;
 
-  // Get and set census context
+  /// Get and set census context
   void set_census_context(struct census_context* ccp) { census_context_ = ccp; }
   struct census_context* census_context() const {
     return census_context_;
   }
 
+  /// Send a best-effort out-of-band cancel. The call could be in any stage.
+  /// e.g. if it is already finished, it may still return success.
+  ///
+  /// There is no guarantee the call will be cancelled.
   void TryCancel();
 
  private:
@@ -215,20 +303,18 @@ class ClientContext {
   template <class R>
   friend class ::grpc::ClientAsyncResponseReader;
   template <class InputMessage, class OutputMessage>
-  friend Status BlockingUnaryCall(ChannelInterface* channel,
-                                  const RpcMethod& method,
+  friend Status BlockingUnaryCall(Channel* channel, const RpcMethod& method,
                                   ClientContext* context,
                                   const InputMessage& request,
                                   OutputMessage* result);
 
   grpc_call* call() { return call_; }
-  void set_call(grpc_call* call,
-                const std::shared_ptr<ChannelInterface>& channel);
+  void set_call(grpc_call* call, const std::shared_ptr<Channel>& channel);
 
   grpc::string authority() { return authority_; }
 
   bool initial_metadata_received_;
-  std::shared_ptr<ChannelInterface> channel_;
+  std::shared_ptr<Channel> channel_;
   grpc_call* call_;
   gpr_timespec deadline_;
   grpc::string authority_;
@@ -236,8 +322,8 @@ class ClientContext {
   mutable std::shared_ptr<const AuthContext> auth_context_;
   struct census_context* census_context_;
   std::multimap<grpc::string, grpc::string> send_initial_metadata_;
-  std::multimap<grpc::string, grpc::string> recv_initial_metadata_;
-  std::multimap<grpc::string, grpc::string> trailing_metadata_;
+  std::multimap<grpc::string_ref, grpc::string_ref> recv_initial_metadata_;
+  std::multimap<grpc::string_ref, grpc::string_ref> trailing_metadata_;
 
   grpc_call* propagate_from_call_;
   PropagationOptions propagation_options_;
diff --git a/include/grpc++/completion_queue.h b/include/grpc++/completion_queue.h
index 2f30211145..0ea970417e 100644
--- a/include/grpc++/completion_queue.h
+++ b/include/grpc++/completion_queue.h
@@ -31,13 +31,15 @@
  *
  */
 
+/// A completion queue implements a concurrent producer-consumer queue, with two
+/// main methods, \a Next and \a AsyncNext.
 #ifndef GRPCXX_COMPLETION_QUEUE_H
 #define GRPCXX_COMPLETION_QUEUE_H
 
 #include <grpc/support/time.h>
 #include <grpc++/impl/grpc_library.h>
-#include <grpc++/status.h>
-#include <grpc++/time.h>
+#include <grpc++/support/status.h>
+#include <grpc++/support/time.h>
 
 struct grpc_completion_queue;
 
@@ -65,55 +67,81 @@ template <class ServiceType, class RequestType, class ResponseType>
 class BidiStreamingHandler;
 class UnknownMethodHandler;
 
-class ChannelInterface;
+class Channel;
 class ClientContext;
+class CompletionQueueTag;
 class CompletionQueue;
 class RpcMethod;
 class Server;
 class ServerBuilder;
 class ServerContext;
 
-class CompletionQueueTag {
- public:
-  virtual ~CompletionQueueTag() {}
-  // Called prior to returning from Next(), return value
-  // is the status of the operation (return status is the default thing
-  // to do)
-  // If this function returns false, the tag is dropped and not returned
-  // from the completion queue
-  virtual bool FinalizeResult(void** tag, bool* status) = 0;
-};
-
-// grpc_completion_queue wrapper class
+/// A thin wrapper around \a grpc_completion_queue (see / \a
+/// src/core/surface/completion_queue.h).
 class CompletionQueue : public GrpcLibrary {
  public:
+  /// Default constructor. Implicitly creates a \a grpc_completion_queue
+  /// instance.
   CompletionQueue();
+
+  /// Wrap \a take, taking ownership of the instance.
+  ///
+  /// \param take The completion queue instance to wrap. Ownership is taken.
   explicit CompletionQueue(grpc_completion_queue* take);
-  ~CompletionQueue() GRPC_OVERRIDE;
 
-  // Tri-state return for AsyncNext: SHUTDOWN, GOT_EVENT, TIMEOUT
-  enum NextStatus { SHUTDOWN, GOT_EVENT, TIMEOUT };
+  /// Destructor. Destroys the owned wrapped completion queue / instance.
+  ~CompletionQueue() GRPC_OVERRIDE;
 
-  // Nonblocking (until deadline) read from queue.
-  // Cannot rely on result of tag or ok if return is TIMEOUT
+  /// Tri-state return for AsyncNext: SHUTDOWN, GOT_EVENT, TIMEOUT.
+  enum NextStatus {
+    SHUTDOWN,   ///< The completion queue has been shutdown.
+    GOT_EVENT,  ///< Got a new event; \a tag will be filled in with its
+                ///< associated value; \a ok indicating its success.
+    TIMEOUT     ///< deadline was reached.
+  };
+
+  /// Read from the queue, blocking up to \a deadline (or the queue's shutdown).
+  /// Both \a tag and \a ok are updated upon success (if an event is available
+  /// within the \a deadline).  A \a tag points to an arbitrary location usually
+  /// employed to uniquely identify an event.
+  ///
+  /// \param tag[out] Upon sucess, updated to point to the event's tag.
+  /// \param ok[out] Upon sucess, true if read a regular event, false otherwise.
+  /// \param deadline[in] How long to block in wait for an event.
+  ///
+  /// \return The type of event read.
   template <typename T>
   NextStatus AsyncNext(void** tag, bool* ok, const T& deadline) {
     TimePoint<T> deadline_tp(deadline);
     return AsyncNextInternal(tag, ok, deadline_tp.raw_time());
   }
 
-  // Blocking read from queue.
-  // Returns false if the queue is ready for destruction, true if event
-
+  /// Read from the queue, blocking until an event is available or the queue is
+  /// shutting down.
+  ///
+  /// \param tag[out] Updated to point to the read event's tag.
+  /// \param ok[out] true if read a regular event, false otherwise.
+  ///
+  /// \return true if read a regular event, false if the queue is shutting down.
   bool Next(void** tag, bool* ok) {
     return (AsyncNextInternal(tag, ok, gpr_inf_future(GPR_CLOCK_REALTIME)) !=
             SHUTDOWN);
   }
 
-  // Shutdown has to be called, and the CompletionQueue can only be
-  // destructed when false is returned from Next().
+  /// Request the shutdown of the queue.
+  ///
+  /// \warning This method must be called at some point. Once invoked, \a Next
+  /// will start to return false and \a AsyncNext will return \a
+  /// NextStatus::SHUTDOWN. Only once either one of these methods does that
+  /// (that is, once the queue has been \em drained) can an instance of this
+  /// class be destroyed.
   void Shutdown();
 
+  /// Returns a \em raw pointer to the underlying \a grpc_completion_queue
+  /// instance.
+  ///
+  /// \warning Remember that the returned instance is owned. No transfer of
+  /// owership is performed.
   grpc_completion_queue* cq() { return cq_; }
 
  private:
@@ -143,24 +171,36 @@ class CompletionQueue : public GrpcLibrary {
   friend class ::grpc::Server;
   friend class ::grpc::ServerContext;
   template <class InputMessage, class OutputMessage>
-  friend Status BlockingUnaryCall(ChannelInterface* channel,
-                                  const RpcMethod& method,
+  friend Status BlockingUnaryCall(Channel* channel, const RpcMethod& method,
                                   ClientContext* context,
                                   const InputMessage& request,
                                   OutputMessage* result);
 
   NextStatus AsyncNextInternal(void** tag, bool* ok, gpr_timespec deadline);
 
-  // Wraps grpc_completion_queue_pluck.
-  // Cannot be mixed with calls to Next().
+  /// Wraps \a grpc_completion_queue_pluck.
+  /// \warning Must not be mixed with calls to \a Next.
   bool Pluck(CompletionQueueTag* tag);
 
-  // Does a single polling pluck on tag
+  /// Performs a single polling pluck on \a tag.
   void TryPluck(CompletionQueueTag* tag);
 
   grpc_completion_queue* cq_;  // owned
 };
 
+/// An interface allowing implementors to process and filter event tags.
+class CompletionQueueTag {
+ public:
+  virtual ~CompletionQueueTag() {}
+  // Called prior to returning from Next(), return value is the status of the
+  // operation (return status is the default thing to do). If this function
+  // returns false, the tag is dropped and not returned from the completion
+  // queue
+  virtual bool FinalizeResult(void** tag, bool* status) = 0;
+};
+
+/// A specific type of completion queue used by the processing of notifications
+/// by servers. Instantiated by \a ServerBuilder.
 class ServerCompletionQueue : public CompletionQueue {
  private:
   friend class ServerBuilder;
diff --git a/include/grpc++/create_channel.h b/include/grpc++/create_channel.h
index 424a93a64c..196d2927a9 100644
--- a/include/grpc++/create_channel.h
+++ b/include/grpc++/create_channel.h
@@ -36,15 +36,31 @@
 
 #include <memory>
 
-#include <grpc++/config.h>
-#include <grpc++/credentials.h>
+#include <grpc++/security/credentials.h>
+#include <grpc++/support/channel_arguments.h>
+#include <grpc++/support/config.h>
 
 namespace grpc {
-class ChannelArguments;
-class ChannelInterface;
 
-// If creds does not hold an object or is invalid, a lame channel is returned.
-std::shared_ptr<ChannelInterface> CreateChannel(
+/// Create a new \a Channel pointing to \a target
+///
+/// \param target The URI of the endpoint to connect to.
+/// \param creds Credentials to use for the created channel. If it does not hold
+/// an object or is invalid, a lame channel is returned.
+/// \param args Options for channel creation.
+std::shared_ptr<Channel> CreateChannel(
+    const grpc::string& target, const std::shared_ptr<Credentials>& creds);
+
+/// Create a new \em custom \a Channel pointing to \a target
+///
+/// \warning For advanced use and testing ONLY. Override default channel
+/// arguments only if necessary.
+///
+/// \param target The URI of the endpoint to connect to.
+/// \param creds Credentials to use for the created channel. If it does not hold
+/// an object or is invalid, a lame channel is returned.
+/// \param args Options for channel creation.
+std::shared_ptr<Channel> CreateCustomChannel(
     const grpc::string& target, const std::shared_ptr<Credentials>& creds,
     const ChannelArguments& args);
 
diff --git a/include/grpc++/credentials.h b/include/grpc++/credentials.h
deleted file mode 100644
index a4f1e73118..0000000000
--- a/include/grpc++/credentials.h
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPCXX_CREDENTIALS_H
-#define GRPCXX_CREDENTIALS_H
-
-#include <memory>
-
-#include <grpc++/config.h>
-#include <grpc++/impl/grpc_library.h>
-
-namespace grpc {
-class ChannelArguments;
-class ChannelInterface;
-class SecureCredentials;
-
-class Credentials : public GrpcLibrary {
- public:
-  ~Credentials() GRPC_OVERRIDE;
-  virtual bool ApplyToCall(grpc_call* call) = 0;
-
- protected:
-  friend std::shared_ptr<Credentials> CompositeCredentials(
-      const std::shared_ptr<Credentials>& creds1,
-      const std::shared_ptr<Credentials>& creds2);
-
-  virtual SecureCredentials* AsSecureCredentials() = 0;
-
- private:
-  friend std::shared_ptr<ChannelInterface> CreateChannel(
-      const grpc::string& target, const std::shared_ptr<Credentials>& creds,
-      const ChannelArguments& args);
-
-  virtual std::shared_ptr<ChannelInterface> CreateChannel(
-      const grpc::string& target, const ChannelArguments& args) = 0;
-};
-
-// Options used to build SslCredentials
-// pem_roots_cert is the buffer containing the PEM encoding of the server root
-// certificates. If this parameter is empty, the default roots will be used.
-// pem_private_key is the buffer containing the PEM encoding of the client's
-// private key. This parameter can be empty if the client does not have a
-// private key.
-// pem_cert_chain is the buffer containing the PEM encoding of the client's
-// certificate chain. This parameter can be empty if the client does not have
-// a certificate chain.
-struct SslCredentialsOptions {
-  grpc::string pem_root_certs;
-  grpc::string pem_private_key;
-  grpc::string pem_cert_chain;
-};
-
-// Factories for building different types of Credentials
-// The functions may return empty shared_ptr when credentials cannot be created.
-// If a Credentials pointer is returned, it can still be invalid when used to
-// create a channel. A lame channel will be created then and all rpcs will
-// fail on it.
-
-// Builds credentials with reasonable defaults.
-std::shared_ptr<Credentials> GoogleDefaultCredentials();
-
-// Builds SSL Credentials given SSL specific options
-std::shared_ptr<Credentials> SslCredentials(
-    const SslCredentialsOptions& options);
-
-// Builds credentials for use when running in GCE
-std::shared_ptr<Credentials> ComputeEngineCredentials();
-
-// Builds service account credentials.
-// json_key is the JSON key string containing the client's private key.
-// scope is a space-delimited list of the requested permissions.
-// token_lifetime_seconds is the lifetime in seconds of each token acquired
-// through this service account credentials. It should be positive and should
-// not exceed grpc_max_auth_token_lifetime or will be cropped to this value.
-std::shared_ptr<Credentials> ServiceAccountCredentials(
-    const grpc::string& json_key, const grpc::string& scope,
-    long token_lifetime_seconds);
-
-// Builds Service Account JWT Access credentials.
-// json_key is the JSON key string containing the client's private key.
-// token_lifetime_seconds is the lifetime in seconds of each Json Web Token
-// (JWT) created with this credentials. It should not exceed
-// grpc_max_auth_token_lifetime or will be cropped to this value.
-std::shared_ptr<Credentials> ServiceAccountJWTAccessCredentials(
-    const grpc::string& json_key, long token_lifetime_seconds);
-
-// Builds refresh token credentials.
-// json_refresh_token is the JSON string containing the refresh token along
-// with a client_id and client_secret.
-std::shared_ptr<Credentials> RefreshTokenCredentials(
-    const grpc::string& json_refresh_token);
-
-// Builds access token credentials.
-// access_token is an oauth2 access token that was fetched using an out of band
-// mechanism.
-std::shared_ptr<Credentials> AccessTokenCredentials(
-    const grpc::string& access_token);
-
-// Builds IAM credentials.
-std::shared_ptr<Credentials> IAMCredentials(
-    const grpc::string& authorization_token,
-    const grpc::string& authority_selector);
-
-// Combines two credentials objects into a composite credentials
-std::shared_ptr<Credentials> CompositeCredentials(
-    const std::shared_ptr<Credentials>& creds1,
-    const std::shared_ptr<Credentials>& creds2);
-
-// Credentials for an unencrypted, unauthenticated channel
-std::shared_ptr<Credentials> InsecureCredentials();
-
-}  // namespace grpc
-
-#endif  // GRPCXX_CREDENTIALS_H
diff --git a/include/grpc++/async_generic_service.h b/include/grpc++/generic/async_generic_service.h
index b435c6e73d..8578d850ff 100644
--- a/include/grpc++/async_generic_service.h
+++ b/include/grpc++/generic/async_generic_service.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPCXX_ASYNC_GENERIC_SERVICE_H
-#define GRPCXX_ASYNC_GENERIC_SERVICE_H
+#ifndef GRPCXX_GENERIC_ASYNC_GENERIC_SERVICE_H
+#define GRPCXX_GENERIC_ASYNC_GENERIC_SERVICE_H
 
-#include <grpc++/byte_buffer.h>
-#include <grpc++/stream.h>
+#include <grpc++/support/byte_buffer.h>
+#include <grpc++/support/async_stream.h>
 
 struct grpc_server;
 
@@ -75,4 +75,4 @@ class AsyncGenericService GRPC_FINAL {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_ASYNC_GENERIC_SERVICE_H
+#endif  // GRPCXX_GENERIC_ASYNC_GENERIC_SERVICE_H
diff --git a/include/grpc++/generic_stub.h b/include/grpc++/generic/generic_stub.h
index 172f10e45a..1bb7900b06 100644
--- a/include/grpc++/generic_stub.h
+++ b/include/grpc++/generic/generic_stub.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPCXX_GENERIC_STUB_H
-#define GRPCXX_GENERIC_STUB_H
+#ifndef GRPCXX_GENERIC_GENERIC_STUB_H
+#define GRPCXX_GENERIC_GENERIC_STUB_H
 
-#include <grpc++/byte_buffer.h>
-#include <grpc++/stream.h>
+#include <grpc++/support/async_stream.h>
+#include <grpc++/support/byte_buffer.h>
 
 namespace grpc {
 
@@ -47,8 +47,7 @@ typedef ClientAsyncReaderWriter<ByteBuffer, ByteBuffer>
 // by name.
 class GenericStub GRPC_FINAL {
  public:
-  explicit GenericStub(std::shared_ptr<ChannelInterface> channel)
-      : channel_(channel) {}
+  explicit GenericStub(std::shared_ptr<Channel> channel) : channel_(channel) {}
 
   // begin a call to a named method
   std::unique_ptr<GenericClientAsyncReaderWriter> Call(
@@ -56,9 +55,9 @@ class GenericStub GRPC_FINAL {
       void* tag);
 
  private:
-  std::shared_ptr<ChannelInterface> channel_;
+  std::shared_ptr<Channel> channel_;
 };
 
 }  // namespace grpc
 
-#endif  // GRPCXX_GENERIC_STUB_H
+#endif  // GRPCXX_GENERIC_GENERIC_STUB_H
diff --git a/include/grpc++/impl/internal_stub.h b/include/grpc++/grpc++.h
index 370a3b8ac5..b7d5fb0bbc 100644
--- a/include/grpc++/impl/internal_stub.h
+++ b/include/grpc++/grpc++.h
@@ -31,27 +31,34 @@
  *
  */
 
-#ifndef GRPCXX_IMPL_INTERNAL_STUB_H
-#define GRPCXX_IMPL_INTERNAL_STUB_H
+/// \mainpage gRPC C++ API
+///
+/// The gRPC C++ API mainly consists of the following classes:
+/// - grpc::Channel, which represents the connection to an endpoint. See [the
+/// gRPC Concepts page](http://www.grpc.io/docs/guides/concepts.html) for more
+/// details. Channels are created by the factory function grpc::CreateChannel.
+/// - grpc::CompletionQueue, the producer-consumer queue used for all
+/// asynchronous communication with the gRPC runtime.
+/// - grpc::ClientContext and grpc::ServerContext, where optional configuration
+/// for an RPC can be set, such as setting custom metadata to be conveyed to the
+/// peer, compression settings, authentication, etc.
+/// - grpc::Server, representing a gRPC server, created by grpc::ServerBuilder.
+///
+/// Refer to the
+/// [examples](https://github.com/grpc/grpc/blob/master/examples/cpp)
+/// for code putting these pieces into play.
 
-#include <memory>
+#ifndef GRPCXX_GRPCXX_H
+#define GRPCXX_GRPCXX_H
 
-#include <grpc++/channel_interface.h>
+#include <grpc/grpc.h>
 
-namespace grpc {
+#include <grpc++/channel.h>
+#include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
 
-class InternalStub {
- public:
-  InternalStub(const std::shared_ptr<ChannelInterface>& channel)
-      : channel_(channel) {}
-  virtual ~InternalStub() {}
-
-  ChannelInterface* channel() { return channel_.get(); }
-
- private:
-  const std::shared_ptr<ChannelInterface> channel_;
-};
-
-}  // namespace grpc
-
-#endif  // GRPCXX_IMPL_INTERNAL_STUB_H
+#endif  // GRPCXX_GRPCXX_H
diff --git a/include/grpc++/impl/call.h b/include/grpc++/impl/call.h
index 35338a413e..fca5603047 100644
--- a/include/grpc++/impl/call.h
+++ b/include/grpc++/impl/call.h
@@ -34,18 +34,17 @@
 #ifndef GRPCXX_IMPL_CALL_H
 #define GRPCXX_IMPL_CALL_H
 
-#include <grpc/support/alloc.h>
-#include <grpc++/client_context.h>
-#include <grpc++/completion_queue.h>
-#include <grpc++/config.h>
-#include <grpc++/status.h>
-#include <grpc++/impl/serialization_traits.h>
-
 #include <functional>
 #include <memory>
 #include <map>
+#include <cstring>
 
-#include <string.h>
+#include <grpc/support/alloc.h>
+#include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
+#include <grpc++/impl/serialization_traits.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
 
 struct grpc_call;
 struct grpc_op;
@@ -55,8 +54,9 @@ namespace grpc {
 class ByteBuffer;
 class Call;
 
-void FillMetadataMap(grpc_metadata_array* arr,
-                     std::multimap<grpc::string, grpc::string>* metadata);
+void FillMetadataMap(
+    grpc_metadata_array* arr,
+    std::multimap<grpc::string_ref, grpc::string_ref>* metadata);
 grpc_metadata* FillMetadataArray(
     const std::multimap<grpc::string, grpc::string>& metadata);
 
@@ -419,7 +419,7 @@ class CallOpRecvInitialMetadata {
   }
 
  private:
-  std::multimap<grpc::string, grpc::string>* recv_initial_metadata_;
+  std::multimap<grpc::string_ref, grpc::string_ref>* recv_initial_metadata_;
   grpc_metadata_array recv_initial_metadata_arr_;
 };
 
@@ -462,7 +462,7 @@ class CallOpClientRecvStatus {
   }
 
  private:
-  std::multimap<grpc::string, grpc::string>* recv_trailing_metadata_;
+  std::multimap<grpc::string_ref, grpc::string_ref>* recv_trailing_metadata_;
   Status* recv_status_;
   grpc_metadata_array recv_trailing_metadata_arr_;
   grpc_status_code status_code_;
diff --git a/include/grpc++/impl/client_unary_call.h b/include/grpc++/impl/client_unary_call.h
index b77ce7d02c..4cdc800267 100644
--- a/include/grpc++/impl/client_unary_call.h
+++ b/include/grpc++/impl/client_unary_call.h
@@ -34,21 +34,20 @@
 #ifndef GRPCXX_IMPL_CLIENT_UNARY_CALL_H
 #define GRPCXX_IMPL_CLIENT_UNARY_CALL_H
 
-#include <grpc++/config.h>
-#include <grpc++/status.h>
-
 #include <grpc++/impl/call.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
 
 namespace grpc {
 
-class ChannelInterface;
+class Channel;
 class ClientContext;
 class CompletionQueue;
 class RpcMethod;
 
 // Wrapper that performs a blocking unary call
 template <class InputMessage, class OutputMessage>
-Status BlockingUnaryCall(ChannelInterface* channel, const RpcMethod& method,
+Status BlockingUnaryCall(Channel* channel, const RpcMethod& method,
                          ClientContext* context, const InputMessage& request,
                          OutputMessage* result) {
   CompletionQueue cq;
diff --git a/include/grpc++/impl/proto_utils.h b/include/grpc++/impl/proto_utils.h
index ebefa3e1be..283e33486d 100644
--- a/include/grpc++/impl/proto_utils.h
+++ b/include/grpc++/impl/proto_utils.h
@@ -38,8 +38,8 @@
 
 #include <grpc/grpc.h>
 #include <grpc++/impl/serialization_traits.h>
-#include <grpc++/config_protobuf.h>
-#include <grpc++/status.h>
+#include <grpc++/support/config_protobuf.h>
+#include <grpc++/support/status.h>
 
 namespace grpc {
 
diff --git a/include/grpc++/impl/rpc_method.h b/include/grpc++/impl/rpc_method.h
index 50a160b08c..9800268062 100644
--- a/include/grpc++/impl/rpc_method.h
+++ b/include/grpc++/impl/rpc_method.h
@@ -34,6 +34,10 @@
 #ifndef GRPCXX_IMPL_RPC_METHOD_H
 #define GRPCXX_IMPL_RPC_METHOD_H
 
+#include <memory>
+
+#include <grpc++/channel.h>
+
 namespace grpc {
 
 class RpcMethod {
@@ -45,8 +49,14 @@ class RpcMethod {
     BIDI_STREAMING
   };
 
-  RpcMethod(const char* name, RpcType type, void* channel_tag)
-      : name_(name), method_type_(type), channel_tag_(channel_tag) {}
+  RpcMethod(const char* name, RpcType type)
+      : name_(name), method_type_(type), channel_tag_(NULL) {}
+
+  RpcMethod(const char* name, RpcType type,
+            const std::shared_ptr<Channel>& channel)
+      : name_(name),
+        method_type_(type),
+        channel_tag_(channel->RegisterMethod(name)) {}
 
   const char* name() const { return name_; }
   RpcType method_type() const { return method_type_; }
diff --git a/include/grpc++/impl/rpc_service_method.h b/include/grpc++/impl/rpc_service_method.h
index c02ebec19e..fcb0b7ccce 100644
--- a/include/grpc++/impl/rpc_service_method.h
+++ b/include/grpc++/impl/rpc_service_method.h
@@ -39,10 +39,10 @@
 #include <memory>
 #include <vector>
 
-#include <grpc++/config.h>
 #include <grpc++/impl/rpc_method.h>
-#include <grpc++/status.h>
-#include <grpc++/stream.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
+#include <grpc++/support/sync_stream.h>
 
 namespace grpc {
 class ServerContext;
@@ -235,7 +235,7 @@ class RpcServiceMethod : public RpcMethod {
   // Takes ownership of the handler
   RpcServiceMethod(const char* name, RpcMethod::RpcType type,
                    MethodHandler* handler)
-      : RpcMethod(name, type, nullptr), handler_(handler) {}
+      : RpcMethod(name, type), handler_(handler) {}
 
   MethodHandler* handler() { return handler_.get(); }
 
diff --git a/include/grpc++/impl/service_type.h b/include/grpc++/impl/service_type.h
index c33a278f5b..3b6ac1de77 100644
--- a/include/grpc++/impl/service_type.h
+++ b/include/grpc++/impl/service_type.h
@@ -34,10 +34,10 @@
 #ifndef GRPCXX_IMPL_SERVICE_TYPE_H
 #define GRPCXX_IMPL_SERVICE_TYPE_H
 
-#include <grpc++/config.h>
 #include <grpc++/impl/serialization_traits.h>
 #include <grpc++/server.h>
-#include <grpc++/status.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
 
 namespace grpc {
 
diff --git a/include/grpc++/impl/sync.h b/include/grpc++/impl/sync.h
index 2f41d2bdeb..999c4303cb 100644
--- a/include/grpc++/impl/sync.h
+++ b/include/grpc++/impl/sync.h
@@ -34,7 +34,7 @@
 #ifndef GRPCXX_IMPL_SYNC_H
 #define GRPCXX_IMPL_SYNC_H
 
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
 
 #ifdef GRPC_CXX0X_NO_THREAD
 #include <grpc++/impl/sync_no_cxx11.h>
diff --git a/include/grpc++/impl/thd.h b/include/grpc++/impl/thd.h
index 4c4578a92d..f8d4258ac6 100644
--- a/include/grpc++/impl/thd.h
+++ b/include/grpc++/impl/thd.h
@@ -34,7 +34,7 @@
 #ifndef GRPCXX_IMPL_THD_H
 #define GRPCXX_IMPL_THD_H
 
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
 
 #ifdef GRPC_CXX0X_NO_THREAD
 #include <grpc++/impl/thd_no_cxx11.h>
diff --git a/include/grpc++/auth_context.h b/include/grpc++/security/auth_context.h
index 7dced90ce5..b924ec90c6 100644
--- a/include/grpc++/auth_context.h
+++ b/include/grpc++/security/auth_context.h
@@ -31,13 +31,14 @@
  *
  */
 
-#ifndef GRPCXX_AUTH_CONTEXT_H
-#define GRPCXX_AUTH_CONTEXT_H
+#ifndef GRPCXX_SUPPORT_AUTH_CONTEXT_H
+#define GRPCXX_SUPPORT_AUTH_CONTEXT_H
 
 #include <iterator>
 #include <vector>
 
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/string_ref.h>
 
 struct grpc_auth_context;
 struct grpc_auth_property;
@@ -46,7 +47,7 @@ struct grpc_auth_property_iterator;
 namespace grpc {
 class SecureAuthContext;
 
-typedef std::pair<grpc::string, grpc::string> AuthProperty;
+typedef std::pair<grpc::string_ref, grpc::string_ref> AuthProperty;
 
 class AuthPropertyIterator
     : public std::iterator<std::input_iterator_tag, const AuthProperty> {
@@ -72,24 +73,38 @@ class AuthPropertyIterator
   const char* name_;
 };
 
+/// Class encapsulating the Authentication Information.
+///
+/// It includes the secure identity of the peer, the type of secure transport
+/// used as well as any other properties required by the authorization layer.
 class AuthContext {
  public:
   virtual ~AuthContext() {}
 
-  // A peer identity, in general is one or more properties (in which case they
-  // have the same name).
-  virtual std::vector<grpc::string> GetPeerIdentity() const = 0;
+  /// Returns true if the peer is authenticated.
+  virtual bool IsPeerAuthenticated() const = 0;
+
+  /// A peer identity.
+  ///
+  /// It is, in general, comprised of one or more properties (in which case they
+  /// have the same name).
+  virtual std::vector<grpc::string_ref> GetPeerIdentity() const = 0;
   virtual grpc::string GetPeerIdentityPropertyName() const = 0;
 
-  // Returns all the property values with the given name.
-  virtual std::vector<grpc::string> FindPropertyValues(
+  /// Returns all the property values with the given name.
+  virtual std::vector<grpc::string_ref> FindPropertyValues(
       const grpc::string& name) const = 0;
 
-  // Iteration over all the properties.
+  /// Iteration over all the properties.
   virtual AuthPropertyIterator begin() const = 0;
   virtual AuthPropertyIterator end() const = 0;
+
+  // Mutation functions: should only be used by an AuthMetadataProcessor.
+  virtual void AddProperty(const grpc::string& key,
+                           const grpc::string_ref& value) = 0;
+  virtual bool SetPeerIdentityPropertyName(const grpc::string& name) = 0;
 };
 
 }  // namespace grpc
 
-#endif  // GRPCXX_AUTH_CONTEXT_H
+#endif  // GRPCXX_SUPPORT_AUTH_CONTEXT_H
diff --git a/include/grpc++/security/auth_metadata_processor.h b/include/grpc++/security/auth_metadata_processor.h
new file mode 100644
index 0000000000..18ad922321
--- /dev/null
+++ b/include/grpc++/security/auth_metadata_processor.h
@@ -0,0 +1,74 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_AUTH_METADATA_PROCESSOR_H_
+#define GRPCXX_AUTH_METADATA_PROCESSOR_H_
+
+#include <map>
+
+#include <grpc++/security/auth_context.h>
+#include <grpc++/support/status.h>
+#include <grpc++/support/string_ref.h>
+
+namespace grpc {
+
+class AuthMetadataProcessor {
+ public:
+  typedef std::multimap<grpc::string_ref, grpc::string_ref> InputMetadata;
+  typedef std::multimap<grpc::string, grpc::string_ref> OutputMetadata;
+
+  virtual ~AuthMetadataProcessor() {}
+
+  // If this method returns true, the Process function will be scheduled in
+  // a different thread from the one processing the call.
+  virtual bool IsBlocking() const { return true; }
+
+  // context is read/write: it contains the properties of the channel peer and
+  // it is the job of the Process method to augment it with properties derived
+  // from the passed-in auth_metadata.
+  // consumed_auth_metadata needs to be filled with metadata that has been
+  // consumed by the processor and will be removed from the call.
+  // response_metadata is the metadata that will be sent as part of the
+  // response.
+  // If the return value is not Status::OK, the rpc call will be aborted with
+  // the error code and error message sent back to the client.
+  virtual Status Process(const InputMetadata& auth_metadata,
+                         AuthContext* context,
+                         OutputMetadata* consumed_auth_metadata,
+                         OutputMetadata* response_metadata) = 0;
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_AUTH_METADATA_PROCESSOR_H_
+
diff --git a/include/grpc++/security/credentials.h b/include/grpc++/security/credentials.h
new file mode 100644
index 0000000000..e423849714
--- /dev/null
+++ b/include/grpc++/security/credentials.h
@@ -0,0 +1,170 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_CREDENTIALS_H
+#define GRPCXX_CREDENTIALS_H
+
+#include <memory>
+
+#include <grpc++/impl/grpc_library.h>
+#include <grpc++/support/config.h>
+
+namespace grpc {
+class ChannelArguments;
+class Channel;
+class SecureCredentials;
+
+/// A credentials object encapsulates all the state needed by a client to
+/// authenticate with a server and make various assertions, e.g., about the
+/// client’s identity, role, or whether it is authorized to make a particular
+/// call.
+///
+/// \see https://github.com/grpc/grpc/blob/master/doc/grpc-auth-support.md
+class Credentials : public GrpcLibrary {
+ public:
+  ~Credentials() GRPC_OVERRIDE;
+
+  /// Apply this instance's credentials to \a call.
+  virtual bool ApplyToCall(grpc_call* call) = 0;
+
+ protected:
+  friend std::shared_ptr<Credentials> CompositeCredentials(
+      const std::shared_ptr<Credentials>& creds1,
+      const std::shared_ptr<Credentials>& creds2);
+
+  virtual SecureCredentials* AsSecureCredentials() = 0;
+
+ private:
+  friend std::shared_ptr<Channel> CreateCustomChannel(
+      const grpc::string& target, const std::shared_ptr<Credentials>& creds,
+      const ChannelArguments& args);
+
+  virtual std::shared_ptr<Channel> CreateChannel(
+      const grpc::string& target, const ChannelArguments& args) = 0;
+};
+
+/// Options used to build SslCredentials.
+struct SslCredentialsOptions {
+  /// The buffer containing the PEM encoding of the server root certificates. If
+  /// this parameter is empty, the default roots will be used.  The default
+  /// roots can be overridden using the \a GRPC_DEFAULT_SSL_ROOTS_FILE_PATH
+  /// environment variable pointing to a file on the file system containing the
+  /// roots.
+  grpc::string pem_root_certs;
+
+  /// The buffer containing the PEM encoding of the client's private key. This
+  /// parameter can be empty if the client does not have a private key.
+  grpc::string pem_private_key;
+
+  /// The buffer containing the PEM encoding of the client's certificate chain.
+  /// This parameter can be empty if the client does not have a certificate
+  /// chain.
+  grpc::string pem_cert_chain;
+};
+
+// Factories for building different types of Credentials The functions may
+// return empty shared_ptr when credentials cannot be created. If a
+// Credentials pointer is returned, it can still be invalid when used to create
+// a channel. A lame channel will be created then and all rpcs will fail on it.
+
+/// Builds credentials with reasonable defaults.
+///
+/// \warning Only use these credentials when connecting to a Google endpoint.
+/// Using these credentials to connect to any other service may result in this
+/// service being able to impersonate your client for requests to Google
+/// services.
+std::shared_ptr<Credentials> GoogleDefaultCredentials();
+
+/// Builds SSL Credentials given SSL specific options
+std::shared_ptr<Credentials> SslCredentials(
+    const SslCredentialsOptions& options);
+
+/// Builds credentials for use when running in GCE
+///
+/// \warning Only use these credentials when connecting to a Google endpoint.
+/// Using these credentials to connect to any other service may result in this
+/// service being able to impersonate your client for requests to Google
+/// services.
+std::shared_ptr<Credentials> GoogleComputeEngineCredentials();
+
+/// Builds Service Account JWT Access credentials.
+/// json_key is the JSON key string containing the client's private key.
+/// token_lifetime_seconds is the lifetime in seconds of each Json Web Token
+/// (JWT) created with this credentials. It should not exceed
+/// grpc_max_auth_token_lifetime or will be cropped to this value.
+std::shared_ptr<Credentials> ServiceAccountJWTAccessCredentials(
+    const grpc::string& json_key, long token_lifetime_seconds);
+
+/// Builds refresh token credentials.
+/// json_refresh_token is the JSON string containing the refresh token along
+/// with a client_id and client_secret.
+///
+/// \warning Only use these credentials when connecting to a Google endpoint.
+/// Using these credentials to connect to any other service may result in this
+/// service being able to impersonate your client for requests to Google
+/// services.
+std::shared_ptr<Credentials> GoogleRefreshTokenCredentials(
+    const grpc::string& json_refresh_token);
+
+/// Builds access token credentials.
+/// access_token is an oauth2 access token that was fetched using an out of band
+/// mechanism.
+///
+/// \warning Only use these credentials when connecting to a Google endpoint.
+/// Using these credentials to connect to any other service may result in this
+/// service being able to impersonate your client for requests to Google
+/// services.
+std::shared_ptr<Credentials> AccessTokenCredentials(
+    const grpc::string& access_token);
+
+/// Builds IAM credentials.
+///
+/// \warning Only use these credentials when connecting to a Google endpoint.
+/// Using these credentials to connect to any other service may result in this
+/// service being able to impersonate your client for requests to Google
+/// services.
+std::shared_ptr<Credentials> GoogleIAMCredentials(
+    const grpc::string& authorization_token,
+    const grpc::string& authority_selector);
+
+/// Combines two credentials objects into a composite credentials
+std::shared_ptr<Credentials> CompositeCredentials(
+    const std::shared_ptr<Credentials>& creds1,
+    const std::shared_ptr<Credentials>& creds2);
+
+/// Credentials for an unencrypted, unauthenticated channel
+std::shared_ptr<Credentials> InsecureCredentials();
+
+}  // namespace grpc
+
+#endif  // GRPCXX_CREDENTIALS_H
diff --git a/include/grpc++/server_credentials.h b/include/grpc++/security/server_credentials.h
index 11acd67e8a..e933825ec3 100644
--- a/include/grpc++/server_credentials.h
+++ b/include/grpc++/security/server_credentials.h
@@ -37,26 +37,37 @@
 #include <memory>
 #include <vector>
 
-#include <grpc++/config.h>
+#include <grpc++/security/auth_metadata_processor.h>
+#include <grpc++/support/config.h>
 
 struct grpc_server;
 
 namespace grpc {
 class Server;
 
-// grpc_server_credentials wrapper class.
+// Wrapper around \a grpc_server_credentials, a way to authenticate a server.
 class ServerCredentials {
  public:
   virtual ~ServerCredentials();
 
+  // This method is not thread-safe and has to be called before the server is
+  // started. The last call to this function wins.
+  virtual void SetAuthMetadataProcessor(
+      const std::shared_ptr<AuthMetadataProcessor>& processor) = 0;
+
  private:
   friend class ::grpc::Server;
 
+  /// Tries to bind \a server to the given \a addr (eg, localhost:1234,
+  /// 192.168.1.1:31416, [::1]:27182, etc.)
+  ///
+  /// \return bound port number on sucess, 0 on failure.
+  // TODO(dgq): the "port" part seems to be a misnomer.
   virtual int AddPortToServer(const grpc::string& addr,
                               grpc_server* server) = 0;
 };
 
-// Options to create ServerCredentials with SSL
+/// Options to create ServerCredentials with SSL
 struct SslServerCredentialsOptions {
   SslServerCredentialsOptions() : force_client_auth(false) {}
 
@@ -69,10 +80,11 @@ struct SslServerCredentialsOptions {
   bool force_client_auth;
 };
 
-// Builds SSL ServerCredentials given SSL specific options
+/// Builds SSL ServerCredentials given SSL specific options
 std::shared_ptr<ServerCredentials> SslServerCredentials(
     const SslServerCredentialsOptions& options);
 
+/// Builds insecure server credentials.
 std::shared_ptr<ServerCredentials> InsecureServerCredentials();
 
 }  // namespace grpc
diff --git a/include/grpc++/server.h b/include/grpc++/server.h
index f29346d3b6..210fe020ad 100644
--- a/include/grpc++/server.h
+++ b/include/grpc++/server.h
@@ -37,13 +37,14 @@
 #include <list>
 #include <memory>
 
+#include <grpc/compression.h>
 #include <grpc++/completion_queue.h>
-#include <grpc++/config.h>
 #include <grpc++/impl/call.h>
 #include <grpc++/impl/grpc_library.h>
 #include <grpc++/impl/sync.h>
-#include <grpc++/status.h>
-#include <grpc/compression.h>
+#include <grpc++/security/server_credentials.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
 
 struct grpc_server;
 
@@ -55,27 +56,32 @@ class AsyncGenericService;
 class RpcService;
 class RpcServiceMethod;
 class ServerAsyncStreamingInterface;
-class ServerCredentials;
 class ThreadPoolInterface;
 
-// Currently it only supports handling rpcs in a single thread.
+/// Models a gRPC server.
+///
+/// Servers are configured and started via \a grpc::ServerBuilder.
 class Server GRPC_FINAL : public GrpcLibrary, private CallHook {
  public:
   ~Server();
 
-  // Shutdown the server, block until all rpc processing finishes.
-  // Forcefully terminate pending calls after deadline expires.
+  /// Shutdown the server, blocking until all rpc processing finishes.
+  /// Forcefully terminate pending calls after \a deadline expires.
+  ///
+  /// \param deadline How long to wait until pending rpcs are forcefully
+  /// terminated.
   template <class T>
   void Shutdown(const T& deadline) {
     ShutdownInternal(TimePoint<T>(deadline).raw_time());
   }
 
-  // Shutdown the server, waiting for all rpc processing to finish.
+  /// Shutdown the server, waiting for all rpc processing to finish.
   void Shutdown() { ShutdownInternal(gpr_inf_future(GPR_CLOCK_MONOTONIC)); }
 
-  // Block waiting for all work to complete (the server must either
-  // be shutting down or some other thread must call Shutdown for this
-  // function to ever return)
+  /// Block waiting for all work to complete.
+  ///
+  /// \warning The server must be either shutting down or some other thread must
+  /// call \a Shutdown for this function to ever return.
   void Wait();
 
  private:
@@ -87,22 +93,57 @@ class Server GRPC_FINAL : public GrpcLibrary, private CallHook {
   class AsyncRequest;
   class ShutdownRequest;
 
-  // ServerBuilder use only
+  /// Server constructors. To be used by \a ServerBuilder only.
+  ///
+  /// \param thread_pool The threadpool instance to use for call processing.
+  /// \param thread_pool_owned Does the server own the \a thread_pool instance?
+  /// \param max_message_size Maximum message length that the channel can
+  /// receive.
   Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
          int max_message_size, grpc_compression_options compression_options);
-  // Register a service. This call does not take ownership of the service.
-  // The service must exist for the lifetime of the Server instance.
+
+  /// Register a service. This call does not take ownership of the service.
+  /// The service must exist for the lifetime of the Server instance.
   bool RegisterService(const grpc::string* host, RpcService* service);
+
+  /// Register an asynchronous service. This call does not take ownership of the
+  /// service. The service must exist for the lifetime of the Server instance.
   bool RegisterAsyncService(const grpc::string* host,
                             AsynchronousService* service);
+
+  /// Register a generic service. This call does not take ownership of the
+  /// service. The service must exist for the lifetime of the Server instance.
   void RegisterAsyncGenericService(AsyncGenericService* service);
-  // Add a listening port. Can be called multiple times.
+
+  /// Tries to bind \a server to the given \a addr.
+  ///
+  /// It can be invoked multiple times.
+  ///
+  /// \param addr The address to try to bind to the server (eg, localhost:1234,
+  /// 192.168.1.1:31416, [::1]:27182, etc.).
+  /// \params creds The credentials associated with the server.
+  ///
+  /// \return bound port number on sucess, 0 on failure.
+  ///
+  /// \warning It's an error to call this method on an already started server.
   int AddListeningPort(const grpc::string& addr, ServerCredentials* creds);
-  // Start the server.
+
+  /// Start the server.
+  ///
+  /// \param cqs Completion queues for handling asynchronous services. The
+  /// caller is required to keep all completion queues live until the server is
+  /// destroyed.
+  /// \param num_cqs How many completion queues does \a cqs hold.
+  ///
+  /// \return true on a successful shutdown.
   bool Start(ServerCompletionQueue** cqs, size_t num_cqs);
 
   void HandleQueueClosed();
+
+  /// Process one or more incoming calls.
   void RunRpc();
+
+  /// Schedule \a RunRpc to run in the threadpool.
   void ScheduleCallback();
 
   void PerformOpsOnCall(CallOpSetInterface* ops, Call* call) GRPC_OVERRIDE;
diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h
index 8968cbf121..05937f150b 100644
--- a/include/grpc++/server_builder.h
+++ b/include/grpc++/server_builder.h
@@ -38,7 +38,7 @@
 #include <vector>
 
 #include <grpc/compression.h>
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
 
 namespace grpc {
 
@@ -52,56 +52,73 @@ class ServerCredentials;
 class SynchronousService;
 class ThreadPoolInterface;
 
+/// A builder class for the creation and startup of \a grpc::Server instances.
 class ServerBuilder {
  public:
   ServerBuilder();
 
-  // Register a service. This call does not take ownership of the service.
-  // The service must exist for the lifetime of the Server instance returned by
-  // BuildAndStart().
-  // Matches requests with any :authority
-  ServerBuilder& RegisterService(SynchronousService* service);
-
-  // Register an asynchronous service.
-  // This call does not take ownership of the service or completion queue.
-  // The service and completion queuemust exist for the lifetime of the Server
-  // instance returned by BuildAndStart().
-  // Matches requests with any :authority
-  ServerBuilder& RegisterAsyncService(AsynchronousService* service);
-
-  // Register a generic service.
-  // Matches requests with any :authority
-  ServerBuilder& RegisterAsyncGenericService(AsyncGenericService* service);
-
-  // Register a service. This call does not take ownership of the service.
-  // The service must exist for the lifetime of the Server instance returned by
-  // BuildAndStart().
-  // Only matches requests with :authority \a host
-  ServerBuilder& RegisterService(const grpc::string& host,
-                                 SynchronousService* service);
-
-  // Register an asynchronous service.
-  // This call does not take ownership of the service or completion queue.
-  // The service and completion queuemust exist for the lifetime of the Server
-  // instance returned by BuildAndStart().
-  // Only matches requests with :authority \a host
-  ServerBuilder& RegisterAsyncService(const grpc::string& host,
-                                      AsynchronousService* service);
-
-  // Set max message size in bytes.
-  ServerBuilder& SetMaxMessageSize(int max_message_size);
-
-  // Add a listening port. Can be called multiple times.
-  ServerBuilder& AddListeningPort(const grpc::string& addr,
+  /// Register a service. This call does not take ownership of the service.
+  /// The service must exist for the lifetime of the \a Server instance returned
+  /// by \a BuildAndStart().
+  /// Matches requests with any :authority
+  void RegisterService(SynchronousService* service);
+
+  /// Register an asynchronous service.
+  /// This call does not take ownership of the service or completion queue.
+  /// The service and completion queuemust exist for the lifetime of the \a
+  /// Server instance returned by \a BuildAndStart().
+  /// Matches requests with any :authority
+  void RegisterAsyncService(AsynchronousService* service);
+
+  /// Register a generic service.
+  /// Matches requests with any :authority
+  void RegisterAsyncGenericService(AsyncGenericService* service);
+
+  /// Register a service. This call does not take ownership of the service.
+  /// The service must exist for the lifetime of the \a Server instance returned
+  /// by BuildAndStart().
+  /// Only matches requests with :authority \a host
+  void RegisterService(const grpc::string& host, SynchronousService* service);
+
+  /// Register an asynchronous service.
+  /// This call does not take ownership of the service or completion queue.
+  /// The service and completion queuemust exist for the lifetime of the \a
+  /// Server instance returned by \a BuildAndStart().
+  /// Only matches requests with :authority equal to \a host
+  void RegisterAsyncService(const grpc::string& host,
+                            AsynchronousService* service);
+
+  /// Set max message size in bytes.
+  void SetMaxMessageSize(int max_message_size) {
+    max_message_size_ = max_message_size;
+  }
+
+  /// Set the compression options to be used by the server.
+  void SetCompressionOptions(const grpc_compression_options& options) {
+    compression_options_ = options;
+  }
+
+  /// Tries to bind \a server to the given \a addr.
+  ///
+  /// It can be invoked multiple times.
+  ///
+  /// \param addr The address to try to bind to the server (eg, localhost:1234,
+  /// 192.168.1.1:31416, [::1]:27182, etc.).
+  /// \params creds The credentials associated with the server.
+  /// \param selected_port[out] Upon success, updated to contain the port
+  /// number. \a nullptr otherwise.
+  ///
+  // TODO(dgq): the "port" part seems to be a misnomer.
+  void AddListeningPort(const grpc::string& addr,
                         std::shared_ptr<ServerCredentials> creds,
                         int* selected_port = nullptr);
 
-  // Add a completion queue for handling asynchronous services
-  // Caller is required to keep this completion queue live until
-  // the server is destroyed.
+  /// Add a completion queue for handling asynchronous services
+  /// Caller is required to keep this completion queue live until
+  /// the server is destroyed.
   std::unique_ptr<ServerCompletionQueue> AddCompletionQueue();
 
-  // Return a running server which is ready for processing rpcs.
+  /// Return a running server which is ready for processing calls.
   std::unique_ptr<Server> BuildAndStart();
 
  private:
diff --git a/include/grpc++/server_context.h b/include/grpc++/server_context.h
index b87a1f0379..85f384d477 100644
--- a/include/grpc++/server_context.h
+++ b/include/grpc++/server_context.h
@@ -39,9 +39,10 @@
 
 #include <grpc/compression.h>
 #include <grpc/support/time.h>
-#include <grpc++/auth_context.h>
-#include <grpc++/config.h>
-#include <grpc++/time.h>
+#include <grpc++/security/auth_context.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/string_ref.h>
+#include <grpc++/support/time.h>
 
 struct gpr_timespec;
 struct grpc_metadata;
@@ -103,7 +104,7 @@ class ServerContext {
 
   bool IsCancelled() const;
 
-  const std::multimap<grpc::string, grpc::string>& client_metadata() {
+  const std::multimap<grpc::string_ref, grpc::string_ref>& client_metadata() {
     return client_metadata_;
   }
 
@@ -185,7 +186,7 @@ class ServerContext {
   CompletionQueue* cq_;
   bool sent_initial_metadata_;
   mutable std::shared_ptr<const AuthContext> auth_context_;
-  std::multimap<grpc::string, grpc::string> client_metadata_;
+  std::multimap<grpc::string_ref, grpc::string_ref> client_metadata_;
   std::multimap<grpc::string, grpc::string> initial_metadata_;
   std::multimap<grpc::string, grpc::string> trailing_metadata_;
 
diff --git a/include/grpc++/status_code_enum.h b/include/grpc++/status_code_enum.h
deleted file mode 100644
index 2211c964cd..0000000000
--- a/include/grpc++/status_code_enum.h
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPCXX_STATUS_CODE_ENUM_H
-#define GRPCXX_STATUS_CODE_ENUM_H
-
-namespace grpc {
-
-enum StatusCode {
-  /* Not an error; returned on success */
-  OK = 0,
-
-  /* The operation was cancelled (typically by the caller). */
-  CANCELLED = 1,
-
-  /* Unknown error.  An example of where this error may be returned is
-     if a Status value received from another address space belongs to
-     an error-space that is not known in this address space.  Also
-     errors raised by APIs that do not return enough error information
-     may be converted to this error. */
-  UNKNOWN = 2,
-
-  /* Client specified an invalid argument.  Note that this differs
-     from FAILED_PRECONDITION.  INVALID_ARGUMENT indicates arguments
-     that are problematic regardless of the state of the system
-     (e.g., a malformed file name). */
-  INVALID_ARGUMENT = 3,
-
-  /* Deadline expired before operation could complete.  For operations
-     that change the state of the system, this error may be returned
-     even if the operation has completed successfully.  For example, a
-     successful response from a server could have been delayed long
-     enough for the deadline to expire. */
-  DEADLINE_EXCEEDED = 4,
-
-  /* Some requested entity (e.g., file or directory) was not found. */
-  NOT_FOUND = 5,
-
-  /* Some entity that we attempted to create (e.g., file or directory)
-     already exists. */
-  ALREADY_EXISTS = 6,
-
-  /* The caller does not have permission to execute the specified
-     operation.  PERMISSION_DENIED must not be used for rejections
-     caused by exhausting some resource (use RESOURCE_EXHAUSTED
-     instead for those errors).  PERMISSION_DENIED must not be
-     used if the caller can not be identified (use UNAUTHENTICATED
-     instead for those errors). */
-  PERMISSION_DENIED = 7,
-
-  /* The request does not have valid authentication credentials for the
-     operation. */
-  UNAUTHENTICATED = 16,
-
-  /* Some resource has been exhausted, perhaps a per-user quota, or
-     perhaps the entire file system is out of space. */
-  RESOURCE_EXHAUSTED = 8,
-
-  /* Operation was rejected because the system is not in a state
-     required for the operation's execution.  For example, directory
-     to be deleted may be non-empty, an rmdir operation is applied to
-     a non-directory, etc.
-
-     A litmus test that may help a service implementor in deciding
-     between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE:
-      (a) Use UNAVAILABLE if the client can retry just the failing call.
-      (b) Use ABORTED if the client should retry at a higher-level
-          (e.g., restarting a read-modify-write sequence).
-      (c) Use FAILED_PRECONDITION if the client should not retry until
-          the system state has been explicitly fixed.  E.g., if an "rmdir"
-          fails because the directory is non-empty, FAILED_PRECONDITION
-          should be returned since the client should not retry unless
-          they have first fixed up the directory by deleting files from it.
-      (d) Use FAILED_PRECONDITION if the client performs conditional
-          REST Get/Update/Delete on a resource and the resource on the
-          server does not match the condition. E.g., conflicting
-          read-modify-write on the same resource. */
-  FAILED_PRECONDITION = 9,
-
-  /* The operation was aborted, typically due to a concurrency issue
-     like sequencer check failures, transaction aborts, etc.
-
-     See litmus test above for deciding between FAILED_PRECONDITION,
-     ABORTED, and UNAVAILABLE. */
-  ABORTED = 10,
-
-  /* Operation was attempted past the valid range.  E.g., seeking or
-     reading past end of file.
-
-     Unlike INVALID_ARGUMENT, this error indicates a problem that may
-     be fixed if the system state changes. For example, a 32-bit file
-     system will generate INVALID_ARGUMENT if asked to read at an
-     offset that is not in the range [0,2^32-1], but it will generate
-     OUT_OF_RANGE if asked to read from an offset past the current
-     file size.
-
-     There is a fair bit of overlap between FAILED_PRECONDITION and
-     OUT_OF_RANGE.  We recommend using OUT_OF_RANGE (the more specific
-     error) when it applies so that callers who are iterating through
-     a space can easily look for an OUT_OF_RANGE error to detect when
-     they are done. */
-  OUT_OF_RANGE = 11,
-
-  /* Operation is not implemented or not supported/enabled in this service. */
-  UNIMPLEMENTED = 12,
-
-  /* Internal errors.  Means some invariants expected by underlying
-     system has been broken.  If you see one of these errors,
-     something is very broken. */
-  INTERNAL = 13,
-
-  /* The service is currently unavailable.  This is a most likely a
-     transient condition and may be corrected by retrying with
-     a backoff.
-
-     See litmus test above for deciding between FAILED_PRECONDITION,
-     ABORTED, and UNAVAILABLE. */
-  UNAVAILABLE = 14,
-
-  /* Unrecoverable data loss or corruption. */
-  DATA_LOSS = 15,
-
-  /* Force users to include a default branch: */
-  DO_NOT_USE = -1
-};
-
-}  // namespace grpc
-
-#endif  // GRPCXX_STATUS_CODE_ENUM_H
diff --git a/include/grpc++/stream.h b/include/grpc++/support/async_stream.h
index 4bffaffb40..b4dae30cd5 100644
--- a/include/grpc++/stream.h
+++ b/include/grpc++/support/async_stream.h
@@ -31,388 +31,62 @@
  *
  */
 
-#ifndef GRPCXX_STREAM_H
-#define GRPCXX_STREAM_H
+#ifndef GRPCXX_SUPPORT_ASYNC_STREAM_H
+#define GRPCXX_SUPPORT_ASYNC_STREAM_H
 
-#include <grpc++/channel_interface.h>
+#include <grpc/support/log.h>
+#include <grpc++/channel.h>
 #include <grpc++/client_context.h>
 #include <grpc++/completion_queue.h>
-#include <grpc++/server_context.h>
 #include <grpc++/impl/call.h>
 #include <grpc++/impl/service_type.h>
-#include <grpc++/status.h>
-#include <grpc/support/log.h>
+#include <grpc++/server_context.h>
+#include <grpc++/support/status.h>
 
 namespace grpc {
 
-// Common interface for all client side streaming.
-class ClientStreamingInterface {
- public:
-  virtual ~ClientStreamingInterface() {}
-
-  // Wait until the stream finishes, and return the final status. When the
-  // client side declares it has no more message to send, either implicitly or
-  // by calling WritesDone, it needs to make sure there is no more message to
-  // be received from the server, either implicitly or by getting a false from
-  // a Read().
-  // This function will return either:
-  // - when all incoming messages have been read and the server has returned
-  //   status
-  // - OR when the server has returned a non-OK status
-  virtual Status Finish() = 0;
-};
-
-// An interface that yields a sequence of R messages.
-template <class R>
-class ReaderInterface {
- public:
-  virtual ~ReaderInterface() {}
-
-  // Blocking read a message and parse to msg. Returns true on success.
-  // The method returns false when there will be no more incoming messages,
-  // either because the other side has called WritesDone or the stream has
-  // failed (or been cancelled).
-  virtual bool Read(R* msg) = 0;
-};
-
-// An interface that can be fed a sequence of W messages.
-template <class W>
-class WriterInterface {
- public:
-  virtual ~WriterInterface() {}
-
-  // Blocking write msg to the stream. Returns true on success.
-  // Returns false when the stream has been closed.
-  virtual bool Write(const W& msg, const WriteOptions& options) = 0;
-
-  inline bool Write(const W& msg) { return Write(msg, WriteOptions()); }
-};
-
-template <class R>
-class ClientReaderInterface : public ClientStreamingInterface,
-                              public ReaderInterface<R> {
- public:
-  virtual void WaitForInitialMetadata() = 0;
-};
-
-template <class R>
-class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
- public:
-  // Blocking create a stream and write the first request out.
-  template <class W>
-  ClientReader(ChannelInterface* channel, const RpcMethod& method,
-               ClientContext* context, const W& request)
-      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-              CallOpClientSendClose> ops;
-    ops.SendInitialMetadata(context->send_initial_metadata_);
-    // TODO(ctiller): don't assert
-    GPR_ASSERT(ops.SendMessage(request).ok());
-    ops.ClientSendClose();
-    call_.PerformOps(&ops);
-    cq_.Pluck(&ops);
-  }
-
-  // Blocking wait for initial metadata from server. The received metadata
-  // can only be accessed after this call returns. Should only be called before
-  // the first read. Calling this method is optional, and if it is not called
-  // the metadata will be available in ClientContext after the first read.
-  void WaitForInitialMetadata() {
-    GPR_ASSERT(!context_->initial_metadata_received_);
-
-    CallOpSet<CallOpRecvInitialMetadata> ops;
-    ops.RecvInitialMetadata(context_);
-    call_.PerformOps(&ops);
-    cq_.Pluck(&ops);  // status ignored
-  }
-
-  bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> ops;
-    if (!context_->initial_metadata_received_) {
-      ops.RecvInitialMetadata(context_);
-    }
-    ops.RecvMessage(msg);
-    call_.PerformOps(&ops);
-    return cq_.Pluck(&ops) && ops.got_message;
-  }
-
-  Status Finish() GRPC_OVERRIDE {
-    CallOpSet<CallOpClientRecvStatus> ops;
-    Status status;
-    ops.ClientRecvStatus(context_, &status);
-    call_.PerformOps(&ops);
-    GPR_ASSERT(cq_.Pluck(&ops));
-    return status;
-  }
-
- private:
-  ClientContext* context_;
-  CompletionQueue cq_;
-  Call call_;
-};
-
-template <class W>
-class ClientWriterInterface : public ClientStreamingInterface,
-                              public WriterInterface<W> {
- public:
-  virtual bool WritesDone() = 0;
-};
-
-template <class W>
-class ClientWriter : public ClientWriterInterface<W> {
- public:
-  // Blocking create a stream.
-  template <class R>
-  ClientWriter(ChannelInterface* channel, const RpcMethod& method,
-               ClientContext* context, R* response)
-      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    finish_ops_.RecvMessage(response);
-
-    CallOpSet<CallOpSendInitialMetadata> ops;
-    ops.SendInitialMetadata(context->send_initial_metadata_);
-    call_.PerformOps(&ops);
-    cq_.Pluck(&ops);
-  }
-
-  using WriterInterface<W>::Write;
-  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
-    CallOpSet<CallOpSendMessage> ops;
-    if (!ops.SendMessage(msg, options).ok()) {
-      return false;
-    }
-    call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
-  }
-
-  bool WritesDone() GRPC_OVERRIDE {
-    CallOpSet<CallOpClientSendClose> ops;
-    ops.ClientSendClose();
-    call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
-  }
-
-  // Read the final response and wait for the final status.
-  Status Finish() GRPC_OVERRIDE {
-    Status status;
-    finish_ops_.ClientRecvStatus(context_, &status);
-    call_.PerformOps(&finish_ops_);
-    GPR_ASSERT(cq_.Pluck(&finish_ops_));
-    return status;
-  }
-
- private:
-  ClientContext* context_;
-  CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_;
-  CompletionQueue cq_;
-  Call call_;
-};
-
-// Client-side interface for bi-directional streaming.
-template <class W, class R>
-class ClientReaderWriterInterface : public ClientStreamingInterface,
-                                    public WriterInterface<W>,
-                                    public ReaderInterface<R> {
- public:
-  virtual void WaitForInitialMetadata() = 0;
-  virtual bool WritesDone() = 0;
-};
-
-template <class W, class R>
-class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
- public:
-  // Blocking create a stream.
-  ClientReaderWriter(ChannelInterface* channel, const RpcMethod& method,
-                     ClientContext* context)
-      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    CallOpSet<CallOpSendInitialMetadata> ops;
-    ops.SendInitialMetadata(context->send_initial_metadata_);
-    call_.PerformOps(&ops);
-    cq_.Pluck(&ops);
-  }
-
-  // Blocking wait for initial metadata from server. The received metadata
-  // can only be accessed after this call returns. Should only be called before
-  // the first read. Calling this method is optional, and if it is not called
-  // the metadata will be available in ClientContext after the first read.
-  void WaitForInitialMetadata() {
-    GPR_ASSERT(!context_->initial_metadata_received_);
-
-    CallOpSet<CallOpRecvInitialMetadata> ops;
-    ops.RecvInitialMetadata(context_);
-    call_.PerformOps(&ops);
-    cq_.Pluck(&ops);  // status ignored
-  }
-
-  bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> ops;
-    if (!context_->initial_metadata_received_) {
-      ops.RecvInitialMetadata(context_);
-    }
-    ops.RecvMessage(msg);
-    call_.PerformOps(&ops);
-    return cq_.Pluck(&ops) && ops.got_message;
-  }
-
-  using WriterInterface<W>::Write;
-  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
-    CallOpSet<CallOpSendMessage> ops;
-    if (!ops.SendMessage(msg, options).ok()) return false;
-    call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
-  }
-
-  bool WritesDone() GRPC_OVERRIDE {
-    CallOpSet<CallOpClientSendClose> ops;
-    ops.ClientSendClose();
-    call_.PerformOps(&ops);
-    return cq_.Pluck(&ops);
-  }
-
-  Status Finish() GRPC_OVERRIDE {
-    CallOpSet<CallOpClientRecvStatus> ops;
-    Status status;
-    ops.ClientRecvStatus(context_, &status);
-    call_.PerformOps(&ops);
-    GPR_ASSERT(cq_.Pluck(&ops));
-    return status;
-  }
-
- private:
-  ClientContext* context_;
-  CompletionQueue cq_;
-  Call call_;
-};
-
-template <class R>
-class ServerReader GRPC_FINAL : public ReaderInterface<R> {
- public:
-  ServerReader(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
-
-  void SendInitialMetadata() {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
-
-    CallOpSet<CallOpSendInitialMetadata> ops;
-    ops.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_->PerformOps(&ops);
-    call_->cq()->Pluck(&ops);
-  }
-
-  bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpSet<CallOpRecvMessage<R>> ops;
-    ops.RecvMessage(msg);
-    call_->PerformOps(&ops);
-    return call_->cq()->Pluck(&ops) && ops.got_message;
-  }
-
- private:
-  Call* const call_;
-  ServerContext* const ctx_;
-};
-
-template <class W>
-class ServerWriter GRPC_FINAL : public WriterInterface<W> {
- public:
-  ServerWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
-
-  void SendInitialMetadata() {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
-
-    CallOpSet<CallOpSendInitialMetadata> ops;
-    ops.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_->PerformOps(&ops);
-    call_->cq()->Pluck(&ops);
-  }
-
-  using WriterInterface<W>::Write;
-  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
-    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> ops;
-    if (!ops.SendMessage(msg, options).ok()) {
-      return false;
-    }
-    if (!ctx_->sent_initial_metadata_) {
-      ops.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    call_->PerformOps(&ops);
-    return call_->cq()->Pluck(&ops);
-  }
-
- private:
-  Call* const call_;
-  ServerContext* const ctx_;
-};
-
-// Server-side interface for bi-directional streaming.
-template <class W, class R>
-class ServerReaderWriter GRPC_FINAL : public WriterInterface<W>,
-                                      public ReaderInterface<R> {
- public:
-  ServerReaderWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
-
-  void SendInitialMetadata() {
-    GPR_ASSERT(!ctx_->sent_initial_metadata_);
-
-    CallOpSet<CallOpSendInitialMetadata> ops;
-    ops.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_->PerformOps(&ops);
-    call_->cq()->Pluck(&ops);
-  }
-
-  bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpSet<CallOpRecvMessage<R>> ops;
-    ops.RecvMessage(msg);
-    call_->PerformOps(&ops);
-    return call_->cq()->Pluck(&ops) && ops.got_message;
-  }
-
-  using WriterInterface<W>::Write;
-  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
-    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> ops;
-    if (!ops.SendMessage(msg, options).ok()) {
-      return false;
-    }
-    if (!ctx_->sent_initial_metadata_) {
-      ops.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    call_->PerformOps(&ops);
-    return call_->cq()->Pluck(&ops);
-  }
-
- private:
-  Call* const call_;
-  ServerContext* const ctx_;
-};
-
-// Async interfaces
-// Common interface for all client side streaming.
+/// Common interface for all client side asynchronous streaming.
 class ClientAsyncStreamingInterface {
  public:
   virtual ~ClientAsyncStreamingInterface() {}
 
+  /// Request notification of the reading of the initial metadata. Completion
+  /// will be notified by \a tag on the associated completion queue.
+  ///
+  /// \param[in] tag Tag identifying this request.
   virtual void ReadInitialMetadata(void* tag) = 0;
 
+  /// Request notification completion.
+  ///
+  /// \param[out] status To be updated with the operation status.
+  /// \param[in] tag Tag identifying this request.
   virtual void Finish(Status* status, void* tag) = 0;
 };
 
-// An interface that yields a sequence of R messages.
+/// An interface that yields a sequence of messages of type \a R.
 template <class R>
 class AsyncReaderInterface {
  public:
   virtual ~AsyncReaderInterface() {}
 
+  /// Read a message of type \a R into \a msg. Completion will be notified by \a
+  /// tag on the associated completion queue.
+  ///
+  /// \param[out] msg Where to eventually store the read message.
+  /// \param[in] tag The tag identifying the operation.
   virtual void Read(R* msg, void* tag) = 0;
 };
 
-// An interface that can be fed a sequence of W messages.
+/// An interface that can be fed a sequence of messages of type \a W.
 template <class W>
 class AsyncWriterInterface {
  public:
   virtual ~AsyncWriterInterface() {}
 
+  /// Request the writing of \a msg with identifying tag \a tag.
+  ///
+  /// \param[in] msg The message to be written.
+  /// \param[in] tag The tag identifying the operation.
   virtual void Write(const W& msg, void* tag) = 0;
 };
 
@@ -423,9 +97,9 @@ class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface,
 template <class R>
 class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
  public:
-  // Create a stream and write the first request out.
+  /// Create a stream and write the first request out.
   template <class W>
-  ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
+  ClientAsyncReader(Channel* channel, CompletionQueue* cq,
                     const RpcMethod& method, ClientContext* context,
                     const W& request, void* tag)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
@@ -473,10 +147,14 @@ class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
   CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
 };
 
+/// Common interface for client side asynchronous writing.
 template <class W>
 class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface,
                                    public AsyncWriterInterface<W> {
  public:
+  /// Signal the client is done with the writes.
+  ///
+  /// \param[in] tag The tag identifying the operation.
   virtual void WritesDone(void* tag) = 0;
 };
 
@@ -484,7 +162,7 @@ template <class W>
 class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
  public:
   template <class R>
-  ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
+  ClientAsyncWriter(Channel* channel, CompletionQueue* cq,
                     const RpcMethod& method, ClientContext* context,
                     R* response, void* tag)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
@@ -536,12 +214,15 @@ class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
             CallOpClientRecvStatus> finish_ops_;
 };
 
-// Client-side interface for bi-directional streaming.
+/// Client-side interface for asynchronous bi-directional streaming.
 template <class W, class R>
 class ClientAsyncReaderWriterInterface : public ClientAsyncStreamingInterface,
                                          public AsyncWriterInterface<W>,
                                          public AsyncReaderInterface<R> {
  public:
+  /// Signal the client is done with the writes.
+  ///
+  /// \param[in] tag The tag identifying the operation.
   virtual void WritesDone(void* tag) = 0;
 };
 
@@ -549,7 +230,7 @@ template <class W, class R>
 class ClientAsyncReaderWriter GRPC_FINAL
     : public ClientAsyncReaderWriterInterface<W, R> {
  public:
-  ClientAsyncReaderWriter(ChannelInterface* channel, CompletionQueue* cq,
+  ClientAsyncReaderWriter(Channel* channel, CompletionQueue* cq,
                           const RpcMethod& method, ClientContext* context,
                           void* tag)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
@@ -715,7 +396,7 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
   CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
 };
 
-// Server-side interface for bi-directional streaming.
+/// Server-side interface for asynchronous bi-directional streaming.
 template <class W, class R>
 class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
                                            public AsyncWriterInterface<W>,
@@ -775,4 +456,4 @@ class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
 
 }  // namespace grpc
 
-#endif  // GRPCXX_STREAM_H
+#endif  // GRPCXX_SUPPORT_ASYNC_STREAM_H
diff --git a/include/grpc++/async_unary_call.h b/include/grpc++/support/async_unary_call.h
index 3d22df4b33..0f4ad2656f 100644
--- a/include/grpc++/async_unary_call.h
+++ b/include/grpc++/support/async_unary_call.h
@@ -31,17 +31,17 @@
  *
  */
 
-#ifndef GRPCXX_ASYNC_UNARY_CALL_H
-#define GRPCXX_ASYNC_UNARY_CALL_H
+#ifndef GRPCXX_SUPPORT_ASYNC_UNARY_CALL_H
+#define GRPCXX_SUPPORT_ASYNC_UNARY_CALL_H
 
-#include <grpc++/channel_interface.h>
+#include <grpc/support/log.h>
+#include <grpc++/channel.h>
 #include <grpc++/client_context.h>
 #include <grpc++/completion_queue.h>
 #include <grpc++/server_context.h>
 #include <grpc++/impl/call.h>
 #include <grpc++/impl/service_type.h>
-#include <grpc++/status.h>
-#include <grpc/support/log.h>
+#include <grpc++/support/status.h>
 
 namespace grpc {
 
@@ -58,7 +58,7 @@ class ClientAsyncResponseReader GRPC_FINAL
     : public ClientAsyncResponseReaderInterface<R> {
  public:
   template <class W>
-  ClientAsyncResponseReader(ChannelInterface* channel, CompletionQueue* cq,
+  ClientAsyncResponseReader(Channel* channel, CompletionQueue* cq,
                             const RpcMethod& method, ClientContext* context,
                             const W& request)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
@@ -152,4 +152,4 @@ class ServerAsyncResponseWriter GRPC_FINAL
 
 }  // namespace grpc
 
-#endif  // GRPCXX_ASYNC_UNARY_CALL_H
+#endif  // GRPCXX_SUPPORT_ASYNC_UNARY_CALL_H
diff --git a/include/grpc++/byte_buffer.h b/include/grpc++/support/byte_buffer.h
index 6467776398..c413703970 100644
--- a/include/grpc++/byte_buffer.h
+++ b/include/grpc++/support/byte_buffer.h
@@ -31,36 +31,39 @@
  *
  */
 
-#ifndef GRPCXX_BYTE_BUFFER_H
-#define GRPCXX_BYTE_BUFFER_H
+#ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H
+#define GRPCXX_SUPPORT_BYTE_BUFFER_H
 
 #include <grpc/grpc.h>
 #include <grpc/byte_buffer.h>
 #include <grpc/support/log.h>
-#include <grpc++/config.h>
-#include <grpc++/slice.h>
-#include <grpc++/status.h>
 #include <grpc++/impl/serialization_traits.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/slice.h>
+#include <grpc++/support/status.h>
 
 #include <vector>
 
 namespace grpc {
 
+/// A sequence of bytes.
 class ByteBuffer GRPC_FINAL {
  public:
+  /// Constuct an empty buffer.
   ByteBuffer() : buffer_(nullptr) {}
 
+  /// Construct buffer from \a slices, of which there are \a nslices.
   ByteBuffer(const Slice* slices, size_t nslices);
 
-  ~ByteBuffer() {
-    if (buffer_) {
-      grpc_byte_buffer_destroy(buffer_);
-    }
-  }
+  ~ByteBuffer();
 
+  /// Dump (read) the buffer contents into \a slices.
   void Dump(std::vector<Slice>* slices) const;
 
+  /// Remove all data.
   void Clear();
+
+  /// Buffer size in bytes.
   size_t Length() const;
 
  private:
@@ -78,6 +81,7 @@ class ByteBuffer GRPC_FINAL {
     buffer_ = buf;
   }
 
+  // For \a SerializationTraits's usage.
   grpc_byte_buffer* buffer() const { return buffer_; }
 
   grpc_byte_buffer* buffer_;
@@ -101,4 +105,4 @@ class SerializationTraits<ByteBuffer, void> {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_BYTE_BUFFER_H
+#endif  // GRPCXX_SUPPORT_BYTE_BUFFER_H
diff --git a/include/grpc++/channel_arguments.h b/include/grpc++/support/channel_arguments.h
index 4d926377ec..9957712a96 100644
--- a/include/grpc++/channel_arguments.h
+++ b/include/grpc++/support/channel_arguments.h
@@ -31,24 +31,24 @@
  *
  */
 
-#ifndef GRPCXX_CHANNEL_ARGUMENTS_H
-#define GRPCXX_CHANNEL_ARGUMENTS_H
+#ifndef GRPCXX_SUPPORT_CHANNEL_ARGUMENTS_H
+#define GRPCXX_SUPPORT_CHANNEL_ARGUMENTS_H
 
 #include <vector>
 #include <list>
 
-#include <grpc++/config.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
+#include <grpc++/support/config.h>
 
 namespace grpc {
 namespace testing {
 class ChannelArgumentsTest;
 }  // namespace testing
 
-// Options for channel creation. The user can use generic setters to pass
-// key value pairs down to c channel creation code. For grpc related options,
-// concrete setters are provided.
+/// Options for channel creation. The user can use generic setters to pass
+/// key value pairs down to c channel creation code. For grpc related options,
+/// concrete setters are provided.
 class ChannelArguments {
  public:
   ChannelArguments() {}
@@ -62,21 +62,26 @@ class ChannelArguments {
 
   void Swap(ChannelArguments& other);
 
-  // grpc specific channel argument setters
-  // Set target name override for SSL host name checking.
+  /// Populates this instance with the arguments from \a channel_args. Does not
+  /// take ownership of \a channel_args.
+  ///
+  /// Note that the underlying arguments are shared. Changes made to either \a
+  /// channel_args or this instance would be reflected on both.
+  void SetChannelArgs(grpc_channel_args* channel_args) const;
+
+  // gRPC specific channel argument setters
+  /// Set target name override for SSL host name checking.
   void SetSslTargetNameOverride(const grpc::string& name);
   // TODO(yangg) add flow control options
-
-  // Set the compression algorithm for the channel.
+  /// Set the compression algorithm for the channel.
   void SetCompressionAlgorithm(grpc_compression_algorithm algorithm);
 
   // Generic channel argument setters. Only for advanced use cases.
+  /// Set an integer argument \a value under \a key.
   void SetInt(const grpc::string& key, int value);
+  /// Set a textual argument \a value under \a key.
   void SetString(const grpc::string& key, const grpc::string& value);
 
-  // Populates given channel_args with args_, does not take ownership.
-  void SetChannelArgs(grpc_channel_args* channel_args) const;
-
  private:
   friend class SecureCredentials;
   friend class testing::ChannelArgumentsTest;
@@ -90,4 +95,4 @@ class ChannelArguments {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_CHANNEL_ARGUMENTS_H
+#endif  // GRPCXX_SUPPORT_CHANNEL_ARGUMENTS_H
diff --git a/include/grpc++/config.h b/include/grpc++/support/config.h
index 889dc39eb7..836bd47283 100644
--- a/include/grpc++/config.h
+++ b/include/grpc++/support/config.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_CONFIG_H
-#define GRPCXX_CONFIG_H
+#ifndef GRPCXX_SUPPORT_CONFIG_H
+#define GRPCXX_SUPPORT_CONFIG_H
 
 #if !defined(GRPC_NO_AUTODETECT_PLATFORM)
 
@@ -113,4 +113,4 @@ typedef GRPC_CUSTOM_STRING string;
 
 }  // namespace grpc
 
-#endif  // GRPCXX_CONFIG_H
+#endif  // GRPCXX_SUPPORT_CONFIG_H
diff --git a/include/grpc++/config_protobuf.h b/include/grpc++/support/config_protobuf.h
index 3afc7a58e2..8235590d41 100644
--- a/include/grpc++/config_protobuf.h
+++ b/include/grpc++/support/config_protobuf.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_CONFIG_PROTOBUF_H
-#define GRPCXX_CONFIG_PROTOBUF_H
+#ifndef GRPCXX_SUPPORT_CONFIG_PROTOBUF_H
+#define GRPCXX_SUPPORT_CONFIG_PROTOBUF_H
 
 #ifndef GRPC_CUSTOM_PROTOBUF_INT64
 #include <google/protobuf/stubs/common.h>
@@ -69,4 +69,4 @@ typedef GRPC_CUSTOM_CODEDINPUTSTREAM CodedInputStream;
 }  // namespace protobuf
 }  // namespace grpc
 
-#endif  // GRPCXX_CONFIG_PROTOBUF_H
+#endif  // GRPCXX_SUPPORT_CONFIG_PROTOBUF_H
diff --git a/include/grpc++/slice.h b/include/grpc++/support/slice.h
index 3e01bcf0ad..456379cc5b 100644
--- a/include/grpc++/slice.h
+++ b/include/grpc++/support/slice.h
@@ -31,36 +31,50 @@
  *
  */
 
-#ifndef GRPCXX_SLICE_H
-#define GRPCXX_SLICE_H
+#ifndef GRPCXX_SUPPORT_SLICE_H
+#define GRPCXX_SUPPORT_SLICE_H
 
 #include <grpc/support/slice.h>
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
 
 namespace grpc {
 
+/// A wrapper around \a grpc_slice.
+///
+/// A slice represents a contiguous reference counted array of bytes.
+/// It is cheap to take references to a slice, and it is cheap to create a
+/// slice pointing to a subset of another slice.
 class Slice GRPC_FINAL {
  public:
-  // construct empty slice
+  /// Construct an empty slice.
   Slice();
-  // destructor - drops one ref
+  // Destructor - drops one reference.
   ~Slice();
-  // construct slice from grpc slice, adding a ref
+
   enum AddRef { ADD_REF };
+  /// Construct a slice from \a slice, adding a reference.
   Slice(gpr_slice slice, AddRef);
-  // construct slice from grpc slice, stealing a ref
+
   enum StealRef { STEAL_REF };
+  /// Construct a slice from \a slice, stealing a reference.
   Slice(gpr_slice slice, StealRef);
-  // copy constructor - adds a ref
+
+  /// Copy constructor, adds a reference.
   Slice(const Slice& other);
-  // assignment - ref count is unchanged
+
+  /// Assignment, reference count is unchanged.
   Slice& operator=(Slice other) {
     std::swap(slice_, other.slice_);
     return *this;
   }
 
+  /// Byte size.
   size_t size() const { return GPR_SLICE_LENGTH(slice_); }
+
+  /// Raw pointer to the beginning (first element) of the slice.
   const gpr_uint8* begin() const { return GPR_SLICE_START_PTR(slice_); }
+
+  /// Raw pointer to the end (one byte \em past the last element) of the slice.
   const gpr_uint8* end() const { return GPR_SLICE_END_PTR(slice_); }
 
  private:
@@ -71,4 +85,4 @@ class Slice GRPC_FINAL {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_SLICE_H
+#endif  // GRPCXX_SUPPORT_SLICE_H
diff --git a/include/grpc++/status.h b/include/grpc++/support/status.h
index fb8526ddce..e59bac92d1 100644
--- a/include/grpc++/status.h
+++ b/include/grpc++/support/status.h
@@ -31,27 +31,39 @@
  *
  */
 
-#ifndef GRPCXX_STATUS_H
-#define GRPCXX_STATUS_H
+#ifndef GRPCXX_SUPPORT_STATUS_H
+#define GRPCXX_SUPPORT_STATUS_H
 
-#include <grpc++/status_code_enum.h>
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
+#include <grpc++/support/status_code_enum.h>
 
 namespace grpc {
 
+/// Did it work? If it didn't, why?
+///
+/// See \a grpc::StatusCode for details on the available code and their meaning.
 class Status {
  public:
+  /// Construct an OK instance.
   Status() : code_(StatusCode::OK) {}
+
+  /// Construct an instance with associated \a code and \a details (also
+  // referred to as "error_message").
   Status(StatusCode code, const grpc::string& details)
       : code_(code), details_(details) {}
 
   // Pre-defined special status objects.
+  /// An OK pre-defined instance.
   static const Status& OK;
+  /// A CANCELLED pre-defined instance.
   static const Status& CANCELLED;
 
+  /// Return the instance's error code.
   StatusCode error_code() const { return code_; }
+  /// Return the instance's error message.
   grpc::string error_message() const { return details_; }
 
+  /// Is the status OK?
   bool ok() const { return code_ == StatusCode::OK; }
 
  private:
@@ -61,4 +73,4 @@ class Status {
 
 }  // namespace grpc
 
-#endif  // GRPCXX_STATUS_H
+#endif  // GRPCXX_SUPPORT_STATUS_H
diff --git a/include/grpc++/support/status_code_enum.h b/include/grpc++/support/status_code_enum.h
new file mode 100644
index 0000000000..ee05b40b51
--- /dev/null
+++ b/include/grpc++/support/status_code_enum.h
@@ -0,0 +1,152 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_SUPPORT_STATUS_CODE_ENUM_H
+#define GRPCXX_SUPPORT_STATUS_CODE_ENUM_H
+
+namespace grpc {
+
+enum StatusCode {
+  /// Not an error; returned on success.
+  OK = 0,
+
+  /// The operation was cancelled (typically by the caller).
+  CANCELLED = 1,
+
+  /// Unknown error. An example of where this error may be returned is if a
+  /// Status value received from another address space belongs to an error-space
+  /// that is not known in this address space. Also errors raised by APIs that
+  /// do not return enough error information may be converted to this error.
+  UNKNOWN = 2,
+
+  /// Client specified an invalid argument. Note that this differs from
+  /// FAILED_PRECONDITION. INVALID_ARGUMENT indicates arguments that are
+  /// problematic regardless of the state of the system (e.g., a malformed file
+  /// name).
+  INVALID_ARGUMENT = 3,
+
+  /// Deadline expired before operation could complete. For operations that
+  /// change the state of the system, this error may be returned even if the
+  /// operation has completed successfully. For example, a successful response
+  /// from a server could have been delayed long enough for the deadline to
+  /// expire.
+  DEADLINE_EXCEEDED = 4,
+
+  /// Some requested entity (e.g., file or directory) was not found.
+  NOT_FOUND = 5,
+
+  /// Some entity that we attempted to create (e.g., file or directory) already
+  /// exists.
+  ALREADY_EXISTS = 6,
+
+  /// The caller does not have permission to execute the specified operation.
+  /// PERMISSION_DENIED must not be used for rejections caused by exhausting
+  /// some resource (use RESOURCE_EXHAUSTED instead for those errors).
+  /// PERMISSION_DENIED must not be used if the caller can not be identified
+  /// (use UNAUTHENTICATED instead for those errors).
+  PERMISSION_DENIED = 7,
+
+  /// The request does not have valid authentication credentials for the
+  /// operation.
+  UNAUTHENTICATED = 16,
+
+  /// Some resource has been exhausted, perhaps a per-user quota, or perhaps the
+  /// entire file system is out of space.
+  RESOURCE_EXHAUSTED = 8,
+
+  /// Operation was rejected because the system is not in a state required for
+  /// the operation's execution. For example, directory to be deleted may be
+  /// non-empty, an rmdir operation is applied to a non-directory, etc.
+  ///
+  /// A litmus test that may help a service implementor in deciding
+  /// between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE:
+  ///  (a) Use UNAVAILABLE if the client can retry just the failing call.
+  ///  (b) Use ABORTED if the client should retry at a higher-level
+  ///      (e.g., restarting a read-modify-write sequence).
+  ///  (c) Use FAILED_PRECONDITION if the client should not retry until
+  ///      the system state has been explicitly fixed. E.g., if an "rmdir"
+  ///      fails because the directory is non-empty, FAILED_PRECONDITION
+  ///      should be returned since the client should not retry unless
+  ///      they have first fixed up the directory by deleting files from it.
+  ///  (d) Use FAILED_PRECONDITION if the client performs conditional
+  ///      REST Get/Update/Delete on a resource and the resource on the
+  ///      server does not match the condition. E.g., conflicting
+  ///      read-modify-write on the same resource.
+  FAILED_PRECONDITION = 9,
+
+  /// The operation was aborted, typically due to a concurrency issue like
+  /// sequencer check failures, transaction aborts, etc.
+  ///
+  /// See litmus test above for deciding between FAILED_PRECONDITION, ABORTED,
+  /// and UNAVAILABLE.
+  ABORTED = 10,
+
+  /// Operation was attempted past the valid range. E.g., seeking or reading
+  /// past end of file.
+  ///
+  /// Unlike INVALID_ARGUMENT, this error indicates a problem that may be fixed
+  /// if the system state changes. For example, a 32-bit file system will
+  /// generate INVALID_ARGUMENT if asked to read at an offset that is not in the
+  /// range [0,2^32-1], but it will generate OUT_OF_RANGE if asked to read from
+  /// an offset past the current file size.
+  ///
+  /// There is a fair bit of overlap between FAILED_PRECONDITION and
+  /// OUT_OF_RANGE. We recommend using OUT_OF_RANGE (the more specific error)
+  /// when it applies so that callers who are iterating through a space can
+  /// easily look for an OUT_OF_RANGE error to detect when they are done.
+  OUT_OF_RANGE = 11,
+
+  /// Operation is not implemented or not supported/enabled in this service.
+  UNIMPLEMENTED = 12,
+
+  /// Internal errors. Means some invariants expected by underlying System has
+  /// been broken. If you see one of these errors, Something is very broken.
+  INTERNAL = 13,
+
+  /// The service is currently unavailable. This is a most likely a transient
+  /// condition and may be corrected by retrying with a backoff.
+  ///
+  /// See litmus test above for deciding between FAILED_PRECONDITION, ABORTED,
+  /// and UNAVAILABLE.
+  UNAVAILABLE = 14,
+
+  /// Unrecoverable data loss or corruption.
+  DATA_LOSS = 15,
+
+  /// Force users to include a default branch:
+  DO_NOT_USE = -1
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_SUPPORT_STATUS_CODE_ENUM_H
diff --git a/include/grpc++/support/string_ref.h b/include/grpc++/support/string_ref.h
new file mode 100644
index 0000000000..a17e167d2b
--- /dev/null
+++ b/include/grpc++/support/string_ref.h
@@ -0,0 +1,123 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_SUPPORT_STRING_REF_H
+#define GRPCXX_SUPPORT_STRING_REF_H
+
+#include <iterator>
+#include <iosfwd>
+
+#include <grpc++/support/config.h>
+
+namespace grpc {
+
+/// This class is a non owning reference to a string.
+///
+/// It should be a strict subset of the upcoming std::string_ref.
+///
+/// \see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3442.html
+///
+/// The constexpr is dropped or replaced with const for legacy compiler
+/// compatibility.
+class string_ref {
+ public:
+  // types
+  typedef const char* const_iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+  // constants
+  const static size_t npos = size_t(-1);
+
+  // construct/copy.
+  string_ref() : data_(nullptr), length_(0) {}
+  string_ref(const string_ref& other)
+      : data_(other.data_), length_(other.length_) {}
+  string_ref& operator=(const string_ref& rhs);
+  string_ref(const char* s);
+  string_ref(const char* s, size_t l) : data_(s), length_(l) {}
+  string_ref(const grpc::string& s) : data_(s.data()), length_(s.length()) {}
+
+  // iterators
+  const_iterator begin() const { return data_; }
+  const_iterator end() const { return data_ + length_; }
+  const_iterator cbegin() const { return data_; }
+  const_iterator cend() const { return data_ + length_; }
+  const_reverse_iterator rbegin() const {
+    return const_reverse_iterator(end());
+  }
+  const_reverse_iterator rend() const {
+    return const_reverse_iterator(begin());
+  }
+  const_reverse_iterator crbegin() const {
+    return const_reverse_iterator(end());
+  }
+  const_reverse_iterator crend() const {
+    return const_reverse_iterator(begin());
+  }
+
+  // capacity
+  size_t size() const { return length_; }
+  size_t length() const { return length_; }
+  size_t max_size() const { return length_; }
+  bool empty() const { return length_ == 0; }
+
+  // element access
+  const char* data() const { return data_; }
+
+  // string operations
+  int compare(string_ref x) const;
+  bool starts_with(string_ref x) const;
+  bool ends_with(string_ref x) const;
+  size_t find(string_ref s) const;
+  size_t find(char c) const;
+
+  string_ref substr(size_t pos, size_t n = npos) const;
+
+ private:
+  const char* data_;
+  size_t length_;
+};
+
+// Comparison operators
+bool operator==(string_ref x, string_ref y);
+bool operator!=(string_ref x, string_ref y);
+bool operator<(string_ref x, string_ref y);
+bool operator>(string_ref x, string_ref y);
+bool operator<=(string_ref x, string_ref y);
+bool operator>=(string_ref x, string_ref y);
+
+std::ostream& operator<<(std::ostream& stream, const string_ref& string);
+
+}  // namespace grpc
+
+#endif  // GRPCXX_SUPPORT_STRING_REF_H
diff --git a/include/grpc++/stub_options.h b/include/grpc++/support/stub_options.h
index c7c16dcd55..973aa9bc83 100644
--- a/include/grpc++/stub_options.h
+++ b/include/grpc++/support/stub_options.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPCXX_STUB_OPTIONS_H
-#define GRPCXX_STUB_OPTIONS_H
+#ifndef GRPCXX_SUPPORT_STUB_OPTIONS_H
+#define GRPCXX_SUPPORT_STUB_OPTIONS_H
 
 namespace grpc {
 
@@ -40,4 +40,4 @@ class StubOptions {};
 
 }  // namespace grpc
 
-#endif  // GRPCXX_STUB_OPTIONS_H
+#endif  // GRPCXX_SUPPORT_STUB_OPTIONS_H
diff --git a/include/grpc++/support/sync_stream.h b/include/grpc++/support/sync_stream.h
new file mode 100644
index 0000000000..514363338d
--- /dev/null
+++ b/include/grpc++/support/sync_stream.h
@@ -0,0 +1,415 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_SUPPORT_SYNC_STREAM_H
+#define GRPCXX_SUPPORT_SYNC_STREAM_H
+
+#include <grpc/support/log.h>
+#include <grpc++/channel.h>
+#include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
+#include <grpc++/impl/call.h>
+#include <grpc++/impl/service_type.h>
+#include <grpc++/server_context.h>
+#include <grpc++/support/status.h>
+
+namespace grpc {
+
+/// Common interface for all synchronous client side streaming.
+class ClientStreamingInterface {
+ public:
+  virtual ~ClientStreamingInterface() {}
+
+  /// Wait until the stream finishes, and return the final status. When the
+  /// client side declares it has no more message to send, either implicitly or
+  /// by calling \a WritesDone(), it needs to make sure there is no more message
+  /// to be received from the server, either implicitly or by getting a false
+  /// from a \a Read().
+  ///
+  /// This function will return either:
+  /// - when all incoming messages have been read and the server has returned
+  ///   status.
+  /// - OR when the server has returned a non-OK status.
+  virtual Status Finish() = 0;
+};
+
+/// An interface that yields a sequence of messages of type \a R.
+template <class R>
+class ReaderInterface {
+ public:
+  virtual ~ReaderInterface() {}
+
+  /// Blocking read a message and parse to \a msg. Returns \a true on success.
+  ///
+  /// \param[out] msg The read message.
+  ///
+  /// \return \a false when there will be no more incoming messages, either
+  /// because the other side has called \a WritesDone() or the stream has failed
+  /// (or been cancelled).
+  virtual bool Read(R* msg) = 0;
+};
+
+/// An interface that can be fed a sequence of messages of type \a W.
+template <class W>
+class WriterInterface {
+ public:
+  virtual ~WriterInterface() {}
+
+  /// Blocking write \a msg to the stream with options.
+  ///
+  /// \param msg The message to be written to the stream.
+  /// \param options Options affecting the write operation.
+  ///
+  /// \return \a true on success, \a false when the stream has been closed.
+  virtual bool Write(const W& msg, const WriteOptions& options) = 0;
+
+  /// Blocking write \a msg to the stream with default options.
+  ///
+  /// \param msg The message to be written to the stream.
+  ///
+  /// \return \a true on success, \a false when the stream has been closed.
+  inline bool Write(const W& msg) { return Write(msg, WriteOptions()); }
+};
+
+/// Client-side interface for streaming reads of message of type \a R.
+template <class R>
+class ClientReaderInterface : public ClientStreamingInterface,
+                              public ReaderInterface<R> {
+ public:
+  /// Blocking wait for initial metadata from server. The received metadata
+  /// can only be accessed after this call returns. Should only be called before
+  /// the first read. Calling this method is optional, and if it is not called
+  /// the metadata will be available in ClientContext after the first read.
+  virtual void WaitForInitialMetadata() = 0;
+};
+
+template <class R>
+class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
+ public:
+  /// Blocking create a stream and write the first request out.
+  template <class W>
+  ClientReader(Channel* channel, const RpcMethod& method,
+               ClientContext* context, const W& request)
+      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+              CallOpClientSendClose> ops;
+    ops.SendInitialMetadata(context->send_initial_metadata_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(ops.SendMessage(request).ok());
+    ops.ClientSendClose();
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);
+  }
+
+  void WaitForInitialMetadata() {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    CallOpSet<CallOpRecvInitialMetadata> ops;
+    ops.RecvInitialMetadata(context_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);  /// status ignored
+  }
+
+  bool Read(R* msg) GRPC_OVERRIDE {
+    CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> ops;
+    if (!context_->initial_metadata_received_) {
+      ops.RecvInitialMetadata(context_);
+    }
+    ops.RecvMessage(msg);
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops) && ops.got_message;
+  }
+
+  Status Finish() GRPC_OVERRIDE {
+    CallOpSet<CallOpClientRecvStatus> ops;
+    Status status;
+    ops.ClientRecvStatus(context_, &status);
+    call_.PerformOps(&ops);
+    GPR_ASSERT(cq_.Pluck(&ops));
+    return status;
+  }
+
+ private:
+  ClientContext* context_;
+  CompletionQueue cq_;
+  Call call_;
+};
+
+/// Client-side interface for streaming writes of message of type \a W.
+template <class W>
+class ClientWriterInterface : public ClientStreamingInterface,
+                              public WriterInterface<W> {
+ public:
+  /// Half close writing from the client.
+  /// Block until writes are completed.
+  ///
+  /// \return Whether the writes were successful.
+  virtual bool WritesDone() = 0;
+};
+
+template <class W>
+class ClientWriter : public ClientWriterInterface<W> {
+ public:
+  /// Blocking create a stream.
+  template <class R>
+  ClientWriter(Channel* channel, const RpcMethod& method,
+               ClientContext* context, R* response)
+      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+    finish_ops_.RecvMessage(response);
+
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);
+  }
+
+  using WriterInterface<W>::Write;
+  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
+    CallOpSet<CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg, options).ok()) {
+      return false;
+    }
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
+  }
+
+  bool WritesDone() GRPC_OVERRIDE {
+    CallOpSet<CallOpClientSendClose> ops;
+    ops.ClientSendClose();
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
+  }
+
+  /// Read the final response and wait for the final status.
+  Status Finish() GRPC_OVERRIDE {
+    Status status;
+    finish_ops_.ClientRecvStatus(context_, &status);
+    call_.PerformOps(&finish_ops_);
+    GPR_ASSERT(cq_.Pluck(&finish_ops_));
+    return status;
+  }
+
+ private:
+  ClientContext* context_;
+  CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_;
+  CompletionQueue cq_;
+  Call call_;
+};
+
+/// Client-side interface for bi-directional streaming.
+template <class W, class R>
+class ClientReaderWriterInterface : public ClientStreamingInterface,
+                                    public WriterInterface<W>,
+                                    public ReaderInterface<R> {
+ public:
+  /// Blocking wait for initial metadata from server. The received metadata
+  /// can only be accessed after this call returns. Should only be called before
+  /// the first read. Calling this method is optional, and if it is not called
+  /// the metadata will be available in ClientContext after the first read.
+  virtual void WaitForInitialMetadata() = 0;
+
+  /// Block until writes are completed.
+  ///
+  /// \return Whether the writes were successful.
+  virtual bool WritesDone() = 0;
+};
+
+template <class W, class R>
+class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
+ public:
+  /// Blocking create a stream.
+  ClientReaderWriter(Channel* channel, const RpcMethod& method,
+                     ClientContext* context)
+      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);
+  }
+
+  void WaitForInitialMetadata() {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    CallOpSet<CallOpRecvInitialMetadata> ops;
+    ops.RecvInitialMetadata(context_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);  // status ignored
+  }
+
+  bool Read(R* msg) GRPC_OVERRIDE {
+    CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> ops;
+    if (!context_->initial_metadata_received_) {
+      ops.RecvInitialMetadata(context_);
+    }
+    ops.RecvMessage(msg);
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops) && ops.got_message;
+  }
+
+  using WriterInterface<W>::Write;
+  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
+    CallOpSet<CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg, options).ok()) return false;
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
+  }
+
+  bool WritesDone() GRPC_OVERRIDE {
+    CallOpSet<CallOpClientSendClose> ops;
+    ops.ClientSendClose();
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
+  }
+
+  Status Finish() GRPC_OVERRIDE {
+    CallOpSet<CallOpClientRecvStatus> ops;
+    Status status;
+    ops.ClientRecvStatus(context_, &status);
+    call_.PerformOps(&ops);
+    GPR_ASSERT(cq_.Pluck(&ops));
+    return status;
+  }
+
+ private:
+  ClientContext* context_;
+  CompletionQueue cq_;
+  Call call_;
+};
+
+template <class R>
+class ServerReader GRPC_FINAL : public ReaderInterface<R> {
+ public:
+  ServerReader(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
+
+  void SendInitialMetadata() {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_->PerformOps(&ops);
+    call_->cq()->Pluck(&ops);
+  }
+
+  bool Read(R* msg) GRPC_OVERRIDE {
+    CallOpSet<CallOpRecvMessage<R>> ops;
+    ops.RecvMessage(msg);
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops) && ops.got_message;
+  }
+
+ private:
+  Call* const call_;
+  ServerContext* const ctx_;
+};
+
+template <class W>
+class ServerWriter GRPC_FINAL : public WriterInterface<W> {
+ public:
+  ServerWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
+
+  void SendInitialMetadata() {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_->PerformOps(&ops);
+    call_->cq()->Pluck(&ops);
+  }
+
+  using WriterInterface<W>::Write;
+  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg, options).ok()) {
+      return false;
+    }
+    if (!ctx_->sent_initial_metadata_) {
+      ops.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops);
+  }
+
+ private:
+  Call* const call_;
+  ServerContext* const ctx_;
+};
+
+/// Server-side interface for bi-directional streaming.
+template <class W, class R>
+class ServerReaderWriter GRPC_FINAL : public WriterInterface<W>,
+                                      public ReaderInterface<R> {
+ public:
+  ServerReaderWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
+
+  void SendInitialMetadata() {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_->PerformOps(&ops);
+    call_->cq()->Pluck(&ops);
+  }
+
+  bool Read(R* msg) GRPC_OVERRIDE {
+    CallOpSet<CallOpRecvMessage<R>> ops;
+    ops.RecvMessage(msg);
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops) && ops.got_message;
+  }
+
+  using WriterInterface<W>::Write;
+  bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE {
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg, options).ok()) {
+      return false;
+    }
+    if (!ctx_->sent_initial_metadata_) {
+      ops.SendInitialMetadata(ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops);
+  }
+
+ private:
+  Call* const call_;
+  ServerContext* const ctx_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_SUPPORT_SYNC_STREAM_H
diff --git a/include/grpc++/time.h b/include/grpc++/support/time.h
index 8fb2f8505c..2d4196b93b 100644
--- a/include/grpc++/time.h
+++ b/include/grpc++/support/time.h
@@ -31,10 +31,10 @@
  *
  */
 
-#ifndef GRPCXX_TIME_H
-#define GRPCXX_TIME_H
+#ifndef GRPCXX_SUPPORT_TIME_H
+#define GRPCXX_SUPPORT_TIME_H
 
-#include <grpc++/config.h>
+#include <grpc++/support/config.h>
 
 namespace grpc {
 
@@ -107,4 +107,4 @@ class TimePoint<std::chrono::system_clock::time_point> {
 
 #endif  // !GRPC_CXX0X_NO_CHRONO
 
-#endif  // GRPCXX_TIME_H
+#endif  // GRPCXX_SUPPORT_TIME_H
diff --git a/include/grpc/census.h b/include/grpc/census.h
index a18b997b79..2f36665d46 100644
--- a/include/grpc/census.h
+++ b/include/grpc/census.h
@@ -69,12 +69,14 @@ int census_supported(void);
 /** Return the census features currently enabled. */
 int census_enabled(void);
 
-/* Internally, Census relies on a context, which should be propagated across
- * RPC's. From the RPC subsystems viewpoint, this is an opaque data structure.
- * A context must be used as the first argument to all other census
- * functions. Conceptually, contexts should be thought of as specific to
- * single RPC/thread. The context can be serialized for passing across the
- * wire. */
+/**
+  Context is a handle used by census to represent the current tracing and
+  tagging information. Contexts should be propagated across RPC's. Contexts
+  are created by any of the census_start_*_op() functions. A context is
+  typically used as argument to most census functions. Conceptually, contexts
+  should be thought of as specific to single RPC/thread. The context can be
+  serialized for passing across the wire, via census_context_serialize().
+*/
 typedef struct census_context census_context;
 
 /* This function is called by the RPC subsystem whenever it needs to get a
@@ -91,18 +93,236 @@ typedef struct census_context census_context;
 size_t census_context_serialize(const census_context *context, char *buffer,
                                 size_t buf_size);
 
-/* Create a new census context, possibly from a serialized buffer. If 'buffer'
- * is non-NULL, it is assumed that it is a buffer encoded by
- * census_context_serialize(). If `buffer` is NULL, a new, empty context is
- * created. The decoded/new contest is returned in 'context'.
- *
- * Returns 0 if no errors, non-zero if buffer is incorrectly formatted, in
- * which case a new empty context will be returned. */
-int census_context_deserialize(const char *buffer, census_context **context);
+/* Distributed traces can have a number of options. */
+enum census_trace_mask_values {
+  CENSUS_TRACE_MASK_NONE = 0,      /* Default, empty flags */
+  CENSUS_TRACE_MASK_IS_SAMPLED = 1 /* RPC tracing enabled for this context. */
+};
+
+/** Get the current trace mask associated with this context. The value returned
+    will be the logical or of census_trace_mask_values values. */
+int census_trace_mask(const census_context *context);
+
+/** Set the trace mask associated with a context. */
+void census_set_trace_mask(int trace_mask);
+
+/* The concept of "operation" is a fundamental concept for Census. In an RPC
+   system, and operation typcially represents a single RPC, or a significant
+   sub-part thereof (e.g. a single logical "read" RPC to a distributed storage
+   system might do several other actions in parallel, from looking up metadata
+   indices to making requests of other services - each of these could be a
+   sub-operation with the larger RPC operation). Census uses operations for the
+   following:
+
+   CPU accounting: If enabled, census will measure the thread CPU time
+   consumed between operation start and end times.
+
+   Active operations: Census will maintain information on all currently
+   active operations.
+
+   Distributed tracing: Each operation serves as a logical trace span.
+
+   Stats collection: Stats are broken down by operation (e.g. latency
+   breakdown for each unique RPC path).
+
+   The following functions serve to delineate the start and stop points for
+   each logical operation. */
+
+/**
+  This structure represents a timestamp as used by census to record the time
+  at which an operation begins.
+*/
+typedef struct {
+  /* Use gpr_timespec for default implementation. High performance
+   * implementations should use a cycle-counter based timestamp. */
+  gpr_timespec ts;
+} census_timestamp;
+
+/**
+  Mark the beginning of an RPC operation. The information required to call the
+  functions to record the start of RPC operations (both client and server) may
+  not be callable at the true start time of the operation, due to information
+  not being available (e.g. the census context data will not be available in a
+  server RPC until at least initial metadata has been processed). To ensure
+  correct CPU accounting and latency recording, RPC systems can call this
+  function to get the timestamp of operation beginning. This can later be used
+  as an argument to census_start_{client,server}_rpc_op(). NB: for correct
+  CPU accounting, the system must guarantee that the same thread is used
+  for all request processing after this function is called.
+
+  @return A timestamp representing the operation start time.
+*/
+census_timestamp census_start_rpc_op_timestamp(void);
+
+/**
+  Represent functions to map RPC name ID to service/method names. Census
+  breaks down all RPC stats by service and method names. We leave the
+  definition and format of these to the RPC system. For efficiency purposes,
+  we encode these as a single 64 bit identifier, and allow the RPC system to
+  provide a structure for functions that can convert these to service and
+  method strings.
+
+  TODO(aveitch): Instead of providing this as an argument to the rpc_start_op()
+  functions, maybe it should be set once at census initialization.
+*/
+typedef struct {
+  const char *(*get_rpc_service_name)(gpr_int64 id);
+  const char *(*get_rpc_method_name)(gpr_int64 id);
+} census_rpc_name_info;
+
+/**
+   Start a client rpc operation. This function should be called as early in the
+   client RPC path as possible. This function will create a new context. If
+   the context argument is non-null, then the new context will inherit all
+   its properties, with the following changes:
+   - create a new operation ID for the new context, marking it as a child of
+     the previous operation.
+   - use the new RPC path and peer information for tracing and stats
+     collection purposes, rather than those from the original context
+
+   If the context argument is NULL, then a new root context is created. This
+   is particularly important for tracing purposes (the trace spans generated
+   will be unassociated with any other trace spans, except those
+   downstream). The trace_mask will be used for tracing operations associated
+   with the new context.
+
+   In some RPC systems (e.g. where load balancing is used), peer information
+   may not be available at the time the operation starts. In this case, use a
+   NULL value for peer, and set it later using the
+   census_set_rpc_client_peer() function.
+
+   @param context The parent context. Can be NULL.
+   @param rpc_name_id The rpc name identifier to be associated with this RPC.
+   @param rpc_name_info Used to decode rpc_name_id.
+   @param peer RPC peer. If not available at the time, NULL can be used,
+               and a later census_set_rpc_client_peer() call made.
+   @param trace_mask An OR of census_trace_mask_values values. Only used in
+                     the creation of a new root context (context == NULL).
+   @param start_time A timestamp returned from census_start_rpc_op_timestamp().
+                     Can be NULL. Used to set the true time the operation
+                     begins.
+
+   @return A new census context.
+ */
+census_context *census_start_client_rpc_op(
+    const census_context *context, gpr_int64 rpc_name_id,
+    const census_rpc_name_info *rpc_name_info, const char *peer, int trace_mask,
+    const census_timestamp *start_time);
+
+/**
+  Add peer information to a context representing a client RPC operation.
+*/
+void census_set_rpc_client_peer(census_context *context, const char *peer);
+
+/**
+   Start a server RPC operation. Returns a new context to be used in future
+   census calls. If buffer is non-NULL, then the buffer contents should
+   represent the client context, as generated by census_context_serialize().
+   If buffer is NULL, a new root context is created.
+
+   @param buffer Buffer containing bytes output from census_context_serialize().
+   @param rpc_name_id The rpc name identifier to be associated with this RPC.
+   @param rpc_name_info Used to decode rpc_name_id.
+   @param peer RPC peer.
+   @param trace_mask An OR of census_trace_mask_values values. Only used in
+                     the creation of a new root context (buffer == NULL).
+   @param start_time A timestamp returned from census_start_rpc_op_timestamp().
+                     Can be NULL. Used to set the true time the operation
+                     begins.
+
+   @return A new census context.
+ */
+census_context *census_start_server_rpc_op(
+    const char *buffer, gpr_int64 rpc_name_id,
+    const census_rpc_name_info *rpc_name_info, const char *peer, int trace_mask,
+    census_timestamp *start_time);
+
+/**
+   Start a new, non-RPC operation. In general, this function works very
+   similarly to census_start_client_rpc_op, with the primary difference being
+   the replacement of host/path information with the more generic family/name
+   tags. If the context argument is non-null, then the new context will
+   inherit all its properties, with the following changes:
+   - create a new operation ID for the new context, marking it as a child of
+     the previous operation.
+   - use the family and name information for tracing and stats collection
+     purposes, rather than those from the original context
+
+   If the context argument is NULL, then a new root context is created. This
+   is particularly important for tracing purposes (the trace spans generated
+   will be unassociated with any other trace spans, except those
+   downstream). The trace_mask will be used for tracing
+   operations associated with the new context.
 
-/* The given context is destroyed. Once destroyed, using the context in
- * future census calls will result in undefined behavior. */
-void census_context_destroy(census_context *context);
+   @param context The base context. Can be NULL.
+   @param family Family name to associate with the trace
+   @param name Name within family to associated with traces/stats
+   @param trace_mask An OR of census_trace_mask_values values. Only used if
+                     context is NULL.
+
+   @return A new census context.
+ */
+census_context *census_start_op(census_context *context, const char *family,
+                                const char *name, int trace_mask);
+
+/**
+  End an operation started by any of the census_start_*_op*() calls. The
+  context used in this call will no longer be valid once this function
+  completes.
+
+  @param context Context associated with operation which is ending.
+  @param status status associated with the operation. Not interpreted by
+                census.
+*/
+void census_end_op(census_context *context, int status);
+
+#define CENSUS_TRACE_RECORD_START_OP ((gpr_uint32)0)
+#define CENSUS_TRACE_RECORD_END_OP ((gpr_uint32)1)
+
+/** Insert a trace record into the trace stream. The record consists of an
+    arbitrary size buffer, the size of which is provided in 'n'.
+    @param context Trace context
+    @param type User-defined type to associate with trace entry.
+    @param buffer Pointer to buffer to use
+    @param n Number of bytes in buffer
+*/
+void census_trace_print(census_context *context, gpr_uint32 type,
+                        const char *buffer, size_t n);
+
+/** Trace record. */
+typedef struct {
+  census_timestamp timestamp; /* Time of record creation */
+  gpr_uint64 trace_id;        /* Trace ID associated with record */
+  gpr_uint64 op_id;           /* Operation ID associated with record */
+  gpr_uint32 type;            /* Type (as used in census_trace_print() */
+  const char *buffer;         /* Buffer (from census_trace_print() */
+  size_t buf_size;            /* Number of bytes inside buffer */
+} census_trace_record;
+
+/** Start a scan of existing trace records. While a scan is ongoing, addition
+    of new trace records will be blocked if the underlying trace buffers
+    fill up, so trace processing systems should endeavor to complete
+    reading as soon as possible.
+  @param consume if non-zero, indicates that reading records also "consumes"
+         the previously read record - i.e. releases space in the trace log
+         while scanning is ongoing.
+  @returns 0 on success, non-zero on failure (e.g. if a scan is already ongoing)
+*/
+int census_trace_scan_start(int consume);
+
+/** Get a trace record. The data pointed to by the trace buffer is guaranteed
+    stable until the next census_get_trace_record() call (if the consume
+    argument to census_trace_scan_start was non-zero) or census_trace_scan_end()
+    is called (otherwise).
+  @param trace_record structure that will be filled in with oldest trace record.
+  @returns -1 if an error occurred (e.g. no previous call to
+           census_trace_scan_start()), 0 if there is no more trace data (and
+           trace_record will not be modified) or 1 otherwise.
+*/
+int census_get_trace_record(census_trace_record *trace_record);
+
+/** End a scan previously started by census_trace_scan_start() */
+void census_trace_scan_end();
 
 /* Max number of characters in tag key */
 #define CENSUS_MAX_TAG_KEY_LENGTH 20
@@ -159,16 +379,107 @@ int census_tag_set_next(census_tag_set_iterator *it, census_tag_const *tag);
    invalidated, and should not be used once close is called. */
 void census_tag_set_close(census_tag_set_iterator *it);
 
-/* A census statistic to be recorded comprises two parts: an ID for the
- * particular statistic and the value to be recorded against it. */
+/* Core stats collection API's. The following concepts are used:
+   * Aggregation: A collection of values. Census supports the following
+       aggregation types:
+         Sum - a single summation type. Typically used for keeping (e.g.)
+           counts of events.
+         Distribution - statistical distribution information, used for
+           recording average, standard deviation etc.
+         Histogram - a histogram of measurements falling in defined bucket
+           boundaries.
+         Window - a count of events that happen in reolling time window.
+     New aggregation types can be added by the user, if desired (see
+     census_register_aggregation()).
+   * Metric: Each measurement is for a single metric. Examples include RPC
+     latency, CPU seconds consumed, and bytes transmitted.
+   * View: A view is a combination of a metric, a tag set (in which the tag
+     values are regular expressions) and a set of aggregations. When a
+     measurement for a metric matches the view tags, it is recorded (for each
+     unique set of tags) against each aggregation. Each metric can have an
+     arbitrary number of views by which it will be broken down.
+*/
+
+/* A single value to be recorded comprises two parts: an ID for the particular
+ * metric and the value to be recorded against it. */
 typedef struct {
-  int id;
+  gpr_uint32 metric_id;
   double value;
-} census_stat;
+} census_value;
+
+/* Record new usage values against the given context. */
+void census_record_values(census_context *context, census_value *values,
+                          size_t nvalues);
+
+/** Type representing a particular aggregation */
+typedef struct census_aggregation_ops census_aggregation_ops;
+
+/* Predefined aggregation types, for use with census_view_create(). */
+extern census_aggregation_ops census_agg_sum;
+extern census_aggregation_ops census_agg_distribution;
+extern census_aggregation_ops census_agg_histogram;
+extern census_aggregation_ops census_agg_window;
+
+/** Information needed to instantiate a new aggregation. Used in view
+    construction via census_define_view(). */
+typedef struct {
+  const census_aggregation_ops *ops;
+  const void
+      *create_arg; /* Argument to be used for aggregation initialization. */
+} census_aggregation;
+
+/** A census view type. Opaque. */
+typedef struct census_view census_view;
+
+/** Create a new view.
+  @param metric_id Metric with which this view is associated.
+  @param tags tags that define the view
+  @param aggregations aggregations to associate with the view
+  @param naggregations number of aggregations
+
+  @return A new census view
+*/
+census_view *census_view_create(gpr_uint32 metric_id,
+                                const census_tag_set *tags,
+                                const census_aggregation *aggregations,
+                                size_t naggregations);
+
+/** Destroy a previously created view. */
+void census_view_delete(census_view *view);
+
+/** Metric ID associated with a view */
+size_t census_view_metric(const census_view *view);
+
+/** Number of aggregations associated with view. */
+size_t census_view_naggregations(const census_view *view);
+
+/** Get tags associated with view. */
+const census_tag_set *census_view_tags(const census_view *view);
+
+/** Get aggregation descriptors associated with a view. */
+const census_aggregation *census_view_aggregrations(const census_view *view);
+
+/** Holds all the aggregation data for a particular view instantiation. Forms
+  part of the data returned by census_view_data(). */
+typedef struct {
+  const census_tag_set *tags; /* Tags for this set of aggregations. */
+  const void **data; /* One data set for every aggregation in the view. */
+} census_view_aggregation_data;
+
+/** Census view data as returned by census_view_get_data(). */
+typedef struct {
+  size_t n_tag_sets; /* Number of unique tag sets that matched view. */
+  const census_view_aggregation_data *data; /* n_tag_sets entries */
+} census_view_data;
+
+/** Get data from aggregations associated with a view.
+  @param view View from which to get data.
+  @return Full set of data for all aggregations for the view.
+*/
+const census_view_data *census_view_get_data(const census_view *view);
 
-/* Record new stats against the given context. */
-void census_record_stat(census_context *context, census_stat *stats,
-                        size_t nstats);
+/** Reset all view data to zero for the specified view */
+void census_view_reset(census_view *view);
 
 #ifdef __cplusplus
 }
diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h
index 101fc88d8f..47f3df6605 100644
--- a/include/grpc/grpc.h
+++ b/include/grpc/grpc.h
@@ -47,11 +47,9 @@ extern "C" {
 
 /*! \mainpage GRPC Core
  *
- * \section intro_sec The GRPC Core library is a low-level library designed
- * to be wrapped by higher level libraries.
- *
- * The top-level API is provided in grpc.h.
- * Security related functionality lives in grpc_security.h.
+ * The GRPC Core library is a low-level library designed to be wrapped by higher
+ * level libraries. The top-level API is provided in grpc.h. Security related
+ * functionality lives in grpc_security.h.
  */
 
 /** Completion Queues enable notification of the completion of asynchronous
@@ -134,6 +132,14 @@ typedef struct {
 /** Secondary user agent: goes at the end of the user-agent metadata
     sent on each request */
 #define GRPC_ARG_SECONDARY_USER_AGENT_STRING "grpc.secondary_user_agent"
+/* The caller of the secure_channel_create functions may override the target
+   name used for SSL host name checking using this channel argument which is of
+   type GRPC_ARG_STRING. This *should* be used for testing only.
+   If this argument is not specified, the name used for SSL host name checking
+   will be the target parameter (assuming that the secure channel is an SSL
+   channel). If this parameter is specified and the underlying is not an SSL
+   channel, it will just be ignored. */
+#define GRPC_SSL_TARGET_NAME_OVERRIDE_ARG "grpc.ssl_target_name_override"
 
 /** Connectivity state of a channel. */
 typedef enum {
@@ -206,8 +212,7 @@ typedef struct grpc_metadata {
 
   /** The following fields are reserved for grpc internal use.
       There is no need to initialize them, and they will be set to garbage
-     during
-      calls to grpc. */
+      during calls to grpc. */
   struct {
     void *obfuscated[4];
   } internal_data;
diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h
index 7f8f4d4a05..87bc250429 100644
--- a/include/grpc/grpc_security.h
+++ b/include/grpc/grpc_security.h
@@ -89,72 +89,55 @@ typedef struct {
      key and certificate chain. This parameter can be NULL if the client does
      not have such a key/cert pair. */
 grpc_credentials *grpc_ssl_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair);
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+    void *reserved);
 
 /* Creates a composite credentials object. */
 grpc_credentials *grpc_composite_credentials_create(grpc_credentials *creds1,
-                                                    grpc_credentials *creds2);
+                                                    grpc_credentials *creds2,
+                                                    void *reserved);
 
-/* Creates a compute engine credentials object.
+/* Creates a compute engine credentials object for connecting to Google.
    WARNING: Do NOT use this credentials to connect to a non-google service as
    this could result in an oauth2 token leak. */
-grpc_credentials *grpc_compute_engine_credentials_create(void);
+grpc_credentials *grpc_google_compute_engine_credentials_create(void *reserved);
 
 extern const gpr_timespec grpc_max_auth_token_lifetime;
 
-/* Creates a service account credentials object. May return NULL if the input is
-   invalid.
-   WARNING: Do NOT use this credentials to connect to a non-google service as
-   this could result in an oauth2 token leak.
-   - json_key is the JSON key string containing the client's private key.
-   - scope is a space-delimited list of the requested permissions.
-   - token_lifetime is the lifetime of each token acquired through this service
-     account credentials.  It should not exceed grpc_max_auth_token_lifetime
-     or will be cropped to this value.  */
-grpc_credentials *grpc_service_account_credentials_create(
-    const char *json_key, const char *scope, gpr_timespec token_lifetime);
-
 /* Creates a JWT credentials object. May return NULL if the input is invalid.
    - json_key is the JSON key string containing the client's private key.
    - token_lifetime is the lifetime of each Json Web Token (JWT) created with
      this credentials.  It should not exceed grpc_max_auth_token_lifetime or
      will be cropped to this value.  */
 grpc_credentials *grpc_service_account_jwt_access_credentials_create(
-    const char *json_key, gpr_timespec token_lifetime);
+    const char *json_key, gpr_timespec token_lifetime, void *reserved);
 
-/* Creates an Oauth2 Refresh Token credentials object. May return NULL if the
-   input is invalid.
+/* Creates an Oauth2 Refresh Token credentials object for connecting to Google.
+   May return NULL if the input is invalid.
    WARNING: Do NOT use this credentials to connect to a non-google service as
    this could result in an oauth2 token leak.
    - json_refresh_token is the JSON string containing the refresh token itself
      along with a client_id and client_secret. */
-grpc_credentials *grpc_refresh_token_credentials_create(
-    const char *json_refresh_token);
+grpc_credentials *grpc_google_refresh_token_credentials_create(
+    const char *json_refresh_token, void *reserved);
 
 /* Creates an Oauth2 Access Token credentials with an access token that was
    aquired by an out of band mechanism. */
 grpc_credentials *grpc_access_token_credentials_create(
-    const char *access_token);
+    const char *access_token, void *reserved);
 
-/* Creates an IAM credentials object. */
-grpc_credentials *grpc_iam_credentials_create(const char *authorization_token,
-                                              const char *authority_selector);
+/* Creates an IAM credentials object for connecting to Google. */
+grpc_credentials *grpc_google_iam_credentials_create(
+    const char *authorization_token, const char *authority_selector,
+    void *reserved);
 
 /* --- Secure channel creation. --- */
 
-/* The caller of the secure_channel_create functions may override the target
-   name used for SSL host name checking using this channel argument which is of
-   type GRPC_ARG_STRING. This *should* be used for testing only.
-   If this argument is not specified, the name used for SSL host name checking
-   will be the target parameter (assuming that the secure channel is an SSL
-   channel). If this parameter is specified and the underlying is not an SSL
-   channel, it will just be ignored. */
-#define GRPC_SSL_TARGET_NAME_OVERRIDE_ARG "grpc.ssl_target_name_override"
-
 /* Creates a secure channel using the passed-in credentials. */
 grpc_channel *grpc_secure_channel_create(grpc_credentials *creds,
                                          const char *target,
-                                         const grpc_channel_args *args);
+                                         const grpc_channel_args *args,
+                                         void *reserved);
 
 /* --- grpc_server_credentials object. ---
 
@@ -180,7 +163,7 @@ void grpc_server_credentials_release(grpc_server_credentials *creds);
      NULL. */
 grpc_server_credentials *grpc_ssl_server_credentials_create(
     const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs, int force_client_auth);
+    size_t num_key_cert_pairs, int force_client_auth, void *reserved);
 
 /* --- Server-side secure ports. --- */
 
@@ -292,10 +275,12 @@ typedef void (*grpc_process_auth_metadata_done_cb)(
 typedef struct {
   /* The context object is read/write: it contains the properties of the
      channel peer and it is the job of the process function to augment it with
-     properties derived from the passed-in metadata. */
+     properties derived from the passed-in metadata.
+     The lifetime of these objects is guaranteed until cb is invoked. */
   void (*process)(void *state, grpc_auth_context *context,
-                  const grpc_metadata *md, size_t md_count,
+                  const grpc_metadata *md, size_t num_md,
                   grpc_process_auth_metadata_done_cb cb, void *user_data);
+  void (*destroy)(void *state);
   void *state;
 } grpc_auth_metadata_processor;
 
diff --git a/include/grpc/support/slice_buffer.h b/include/grpc/support/slice_buffer.h
index ec048e8c91..04db003ac5 100644
--- a/include/grpc/support/slice_buffer.h
+++ b/include/grpc/support/slice_buffer.h
@@ -86,6 +86,8 @@ void gpr_slice_buffer_reset_and_unref(gpr_slice_buffer *sb);
 void gpr_slice_buffer_swap(gpr_slice_buffer *a, gpr_slice_buffer *b);
 /* move all of the elements of src into dst */
 void gpr_slice_buffer_move_into(gpr_slice_buffer *src, gpr_slice_buffer *dst);
+/* remove n bytes from the end of a slice buffer */
+void gpr_slice_buffer_trim_end(gpr_slice_buffer *src, size_t n);
 
 #ifdef __cplusplus
 }
diff --git a/include/grpc/support/time.h b/include/grpc/support/time.h
index 4ef9c76459..f8ed893bc0 100644
--- a/include/grpc/support/time.h
+++ b/include/grpc/support/time.h
@@ -52,6 +52,9 @@ typedef enum {
   /* Realtime clock. May jump forwards or backwards. Settable by
      the system administrator. Has its epoch at 0:00:00 UTC 1 Jan 1970. */
   GPR_CLOCK_REALTIME,
+  /* CPU cycle time obtained by rdtsc instruction on x86 platforms. Epoch
+     undefined. Degrades to GPR_CLOCK_REALTIME on other platforms. */
+  GPR_CLOCK_PRECISE,
   /* Unmeasurable clock type: no base, created by taking the difference
      between two times */
   GPR_TIMESPAN