Merge branch 'master' of github.com:grpc/grpc into lb_pollset_propagation

168 files changed, 4278 insertions, 4843 deletions

diff --git a/src/core/ext/client_config/subchannel.c b/src/core/ext/client_config/subchannel.c
index 08795d1358..a67ec72bd1 100644
--- a/src/core/ext/client_config/subchannel.c
+++ b/src/core/ext/client_config/subchannel.c
@@ -320,7 +320,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
     c->filters = NULL;
   }
   c->addr = gpr_malloc(args->addr_len);
-  memcpy(c->addr, args->addr, args->addr_len);
+  if (args->addr_len) memcpy(c->addr, args->addr, args->addr_len);
   c->pollset_set = grpc_pollset_set_create();
   c->addr_len = args->addr_len;
   grpc_set_initial_connect_string(&c->addr, &c->addr_len,
diff --git a/src/core/ext/client_config/subchannel_index.c b/src/core/ext/client_config/subchannel_index.c
index ab8d9bd91d..690cb16b96 100644
--- a/src/core/ext/client_config/subchannel_index.c
+++ b/src/core/ext/client_config/subchannel_index.c
@@ -77,12 +77,19 @@ static grpc_subchannel_key *create_key(
   grpc_subchannel_key *k = gpr_malloc(sizeof(*k));
   k->connector = grpc_connector_ref(connector);
   k->args.filter_count = args->filter_count;
-  k->args.filters = gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
-  memcpy((grpc_channel_filter *)k->args.filters, args->filters,
-         sizeof(*k->args.filters) * k->args.filter_count);
+  if (k->args.filter_count > 0) {
+    k->args.filters =
+        gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
+    memcpy((grpc_channel_filter *)k->args.filters, args->filters,
+           sizeof(*k->args.filters) * k->args.filter_count);
+  } else {
+    k->args.filters = NULL;
+  }
   k->args.addr_len = args->addr_len;
   k->args.addr = gpr_malloc(args->addr_len);
-  memcpy(k->args.addr, args->addr, k->args.addr_len);
+  if (k->args.addr_len > 0) {
+    memcpy(k->args.addr, args->addr, k->args.addr_len);
+  }
   k->args.args = copy_channel_args(args->args);
   return k;
 }
@@ -104,11 +111,15 @@ static int subchannel_key_compare(grpc_subchannel_key *a,
   if (c != 0) return c;
   c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
   if (c != 0) return c;
-  c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
-  if (c != 0) return c;
-  c = memcmp(a->args.filters, b->args.filters,
-             a->args.filter_count * sizeof(*a->args.filters));
-  if (c != 0) return c;
+  if (a->args.addr_len) {
+    c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
+    if (c != 0) return c;
+  }
+  if (a->args.filter_count > 0) {
+    c = memcmp(a->args.filters, b->args.filters,
+               a->args.filter_count * sizeof(*a->args.filters));
+    if (c != 0) return c;
+  }
   return grpc_channel_args_compare(a->args.args, b->args.args);
 }
 
diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
index 58af6f995a..a262306085 100644
--- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
+++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
@@ -45,9 +45,9 @@
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/tcp_client.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
index e21fa2a072..0428bb1e3d 100644
--- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -43,14 +43,8 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-static void setup_transport(grpc_exec_ctx *exec_ctx, void *server,
-                            grpc_transport *transport) {
-  grpc_server_setup_transport(exec_ctx, server, transport,
-                              grpc_server_get_channel_args(server));
-}
-
 static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
-                          grpc_endpoint *tcp,
+                          grpc_endpoint *tcp, grpc_pollset *accepting_pollset,
                           grpc_tcp_server_acceptor *acceptor) {
   /*
    * Beware that the call to grpc_create_chttp2_transport() has to happen before
@@ -61,7 +55,8 @@ static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
    */
   grpc_transport *transport = grpc_create_chttp2_transport(
       exec_ctx, grpc_server_get_channel_args(server), tcp, 0);
-  setup_transport(exec_ctx, server, transport);
+  grpc_server_setup_transport(exec_ctx, server, transport, accepting_pollset,
+                              grpc_server_get_channel_args(server));
   grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
 }
 
diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index 698b2bef61..ebbefbcd89 100644
--- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -45,14 +45,14 @@
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/tcp_server.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-typedef struct grpc_server_secure_state {
+typedef struct server_secure_state {
   grpc_server *server;
   grpc_tcp_server *tcp;
   grpc_server_security_connector *sc;
@@ -62,13 +62,16 @@ typedef struct grpc_server_secure_state {
   gpr_refcount refcount;
   grpc_closure destroy_closure;
   grpc_closure *destroy_callback;
-} grpc_server_secure_state;
+} server_secure_state;
 
-static void state_ref(grpc_server_secure_state *state) {
-  gpr_ref(&state->refcount);
-}
+typedef struct server_secure_connect {
+  server_secure_state *state;
+  grpc_pollset *accepting_pollset;
+} server_secure_connect;
+
+static void state_ref(server_secure_state *state) { gpr_ref(&state->refcount); }
 
-static void state_unref(grpc_server_secure_state *state) {
+static void state_unref(server_secure_state *state) {
   if (gpr_unref(&state->refcount)) {
     /* ensure all threads have unlocked */
     gpr_mu_lock(&state->mu);
@@ -80,67 +83,66 @@ static void state_unref(grpc_server_secure_state *state) {
   }
 }
 
-static void setup_transport(grpc_exec_ctx *exec_ctx, void *statep,
-                            grpc_transport *transport,
-                            grpc_auth_context *auth_context) {
-  grpc_server_secure_state *state = statep;
-  grpc_channel_args *args_copy;
-  grpc_arg args_to_add[2];
-  args_to_add[0] = grpc_server_credentials_to_arg(state->creds);
-  args_to_add[1] = grpc_auth_context_to_arg(auth_context);
-  args_copy = grpc_channel_args_copy_and_add(
-      grpc_server_get_channel_args(state->server), args_to_add,
-      GPR_ARRAY_SIZE(args_to_add));
-  grpc_server_setup_transport(exec_ctx, state->server, transport, args_copy);
-  grpc_channel_args_destroy(args_copy);
-}
-
 static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
                                      grpc_security_status status,
                                      grpc_endpoint *secure_endpoint,
                                      grpc_auth_context *auth_context) {
-  grpc_server_secure_state *state = statep;
+  server_secure_connect *state = statep;
   grpc_transport *transport;
   if (status == GRPC_SECURITY_OK) {
     if (secure_endpoint) {
-      gpr_mu_lock(&state->mu);
-      if (!state->is_shutdown) {
+      gpr_mu_lock(&state->state->mu);
+      if (!state->state->is_shutdown) {
         transport = grpc_create_chttp2_transport(
-            exec_ctx, grpc_server_get_channel_args(state->server),
+            exec_ctx, grpc_server_get_channel_args(state->state->server),
             secure_endpoint, 0);
-        setup_transport(exec_ctx, state, transport, auth_context);
+        grpc_channel_args *args_copy;
+        grpc_arg args_to_add[2];
+        args_to_add[0] = grpc_server_credentials_to_arg(state->state->creds);
+        args_to_add[1] = grpc_auth_context_to_arg(auth_context);
+        args_copy = grpc_channel_args_copy_and_add(
+            grpc_server_get_channel_args(state->state->server), args_to_add,
+            GPR_ARRAY_SIZE(args_to_add));
+        grpc_server_setup_transport(exec_ctx, state->state->server, transport,
+                                    state->accepting_pollset, args_copy);
+        grpc_channel_args_destroy(args_copy);
         grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
       } else {
         /* We need to consume this here, because the server may already have
          * gone away. */
         grpc_endpoint_destroy(exec_ctx, secure_endpoint);
       }
-      gpr_mu_unlock(&state->mu);
+      gpr_mu_unlock(&state->state->mu);
     }
   } else {
     gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
   }
-  state_unref(state);
+  state_unref(state->state);
+  gpr_free(state);
 }
 
 static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
+                      grpc_pollset *accepting_pollset,
                       grpc_tcp_server_acceptor *acceptor) {
-  grpc_server_secure_state *state = statep;
-  state_ref(state);
-  grpc_server_security_connector_do_handshake(
-      exec_ctx, state->sc, acceptor, tcp, on_secure_handshake_done, state);
+  server_secure_connect *state = gpr_malloc(sizeof(*state));
+  state->state = statep;
+  state_ref(state->state);
+  state->accepting_pollset = accepting_pollset;
+  grpc_server_security_connector_do_handshake(exec_ctx, state->state->sc,
+                                              acceptor, tcp,
+                                              on_secure_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
 static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
                   grpc_pollset **pollsets, size_t pollset_count) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   grpc_tcp_server_start(exec_ctx, state->tcp, pollsets, pollset_count,
                         on_accept, state);
 }
 
 static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   if (state->destroy_callback != NULL) {
     state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg,
                                 success);
@@ -153,7 +155,7 @@ static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
    callbacks) */
 static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
                     grpc_closure *callback) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   grpc_tcp_server *tcp;
   gpr_mu_lock(&state->mu);
   state->is_shutdown = 1;
@@ -167,7 +169,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
                                       grpc_server_credentials *creds) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
-  grpc_server_secure_state *state = NULL;
+  server_secure_state *state = NULL;
   size_t i;
   unsigned count = 0;
   int port_num = -1;
diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.c b/src/core/ext/transport/chttp2/transport/frame_goaway.c
index 69accb7696..827e7a6977 100644
--- a/src/core/ext/transport/chttp2/transport/frame_goaway.c
+++ b/src/core/ext/transport/chttp2/transport/frame_goaway.c
@@ -137,7 +137,8 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
       ++cur;
     /* fallthrough */
     case GRPC_CHTTP2_GOAWAY_DEBUG:
-      memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
+      if (end != cur)
+        memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
       GPR_ASSERT((size_t)(end - cur) < UINT32_MAX - p->debug_pos);
       p->debug_pos += (uint32_t)(end - cur);
       p->state = GRPC_CHTTP2_GOAWAY_DEBUG;
diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c
index 687936bfd3..ed45bc9cb3 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_parser.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c
@@ -1138,6 +1138,7 @@ static int parse_string_prefix(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 /* append some bytes to a string */
 static void append_bytes(grpc_chttp2_hpack_parser_string *str,
                          const uint8_t *data, size_t length) {
+  if (length == 0) return;
   if (length + str->length > str->capacity) {
     GPR_ASSERT(str->length + length <= UINT32_MAX);
     str->capacity = (uint32_t)(str->length + length);
@@ -1445,6 +1446,11 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
        stream id on a header */
     if (stream_parsing != NULL) {
       if (parser->is_boundary) {
+        if (stream_parsing->header_frames_received ==
+            GPR_ARRAY_SIZE(stream_parsing->got_metadata_on_parse)) {
+          gpr_log(GPR_ERROR, "too many trailer frames");
+          return GRPC_CHTTP2_CONNECTION_ERROR;
+        }
         stream_parsing
             ->got_metadata_on_parse[stream_parsing->header_frames_received] = 1;
         stream_parsing->header_frames_received++;
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c
index bebc6448e5..1bb53b756c 100644
--- a/src/core/ext/transport/cronet/transport/cronet_transport.c
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.c
@@ -222,8 +222,11 @@ static void on_write_completed(cronet_bidirectional_stream *stream,
 static void process_recv_message(stream_obj *s, const uint8_t *recv_data) {
   gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes);
   uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice);
-  memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
-  gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  if (s->total_read_bytes > 0) {
+    // Only copy if there is non-zero number of bytes
+    memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
+    gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  }
   grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0);
   *s->recv_message = (grpc_byte_buffer *)&s->sbs;
 }
@@ -351,8 +354,17 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) {
           if (grpc_cronet_trace) {
             gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");
           }
-          cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
-                                           s->remaining_read_bytes);
+          if (s->remaining_read_bytes > 0) {
+            cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
+                                             s->remaining_read_bytes);
+          } else {
+            // Calling the closing callback directly since this is a 0 byte read
+            // for an empty message.
+            process_recv_message(s, NULL);
+            enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]);
+            invoke_closing_callback(s);
+            set_recv_state(s, CRONET_RECV_CLOSED);
+          }
         }
       }
       break;
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index 893cf0700e..569be4dc28 100644
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -132,7 +132,8 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
   for (size_t i = 0; i < a->num_args; i++) {
     args[i] = &a->args[i];
   }
-  qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
+  if (a->num_args > 1)
+    qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
 
   grpc_channel_args *b = gpr_malloc(sizeof(grpc_channel_args));
   b->num_args = a->num_args;
diff --git a/src/core/lib/compression/compression_algorithm.c b/src/core/lib/compression/compression_algorithm.c
index 7039364b7b..820871d579 100644
--- a/src/core/lib/compression/compression_algorithm.c
+++ b/src/core/lib/compression/compression_algorithm.c
@@ -199,5 +199,6 @@ void grpc_compression_options_disable_algorithm(
 int grpc_compression_options_is_algorithm_enabled(
     const grpc_compression_options *opts,
     grpc_compression_algorithm algorithm) {
+  if (algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) return 0;
   return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
 }
diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c
index ea4bff30d4..5590928968 100644
--- a/src/core/lib/http/httpcli_security_connector.c
+++ b/src/core/lib/http/httpcli_security_connector.c
@@ -38,7 +38,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
 
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
deleted file mode 100644
index aeb6e28665..0000000000
--- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
+++ /dev/null
@@ -1,1934 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/* This file will be removed shortly: it's here to keep refactoring
- * steps simple and auditable.
- * It's the combination of the old files:
- *  - fd_posix.{h,c}
- *  - pollset_posix.{h,c}
- *  - pullset_multipoller_with_{poll,epoll}.{h,c}
- * The new version will be split into:
- *  - ev_poll_posix.{h,c}
- *  - ev_epoll_posix.{h,c}
- */
-
-#include <grpc/support/port_platform.h>
-
-#ifdef GPR_POSIX_SOCKET
-
-#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <poll.h>
-#include <string.h>
-#include <sys/socket.h>
-#include <unistd.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/tls.h>
-#include <grpc/support/useful.h>
-
-#include "src/core/lib/iomgr/iomgr_internal.h"
-#include "src/core/lib/iomgr/wakeup_fd_posix.h"
-#include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/block_annotate.h"
-
-/*******************************************************************************
- * FD declarations
- */
-
-typedef struct grpc_fd_watcher {
-  struct grpc_fd_watcher *next;
-  struct grpc_fd_watcher *prev;
-  grpc_pollset *pollset;
-  grpc_pollset_worker *worker;
-  grpc_fd *fd;
-} grpc_fd_watcher;
-
-struct grpc_fd {
-  int fd;
-  /* refst format:
-     bit0:   1=active/0=orphaned
-     bit1-n: refcount
-     meaning that mostly we ref by two to avoid altering the orphaned bit,
-     and just unref by 1 when we're ready to flag the object as orphaned */
-  gpr_atm refst;
-
-  gpr_mu mu;
-  int shutdown;
-  int closed;
-  int released;
-
-  /* The watcher list.
-
-     The following watcher related fields are protected by watcher_mu.
-
-     An fd_watcher is an ephemeral object created when an fd wants to
-     begin polling, and destroyed after the poll.
-
-     It denotes the fd's interest in whether to read poll or write poll
-     or both or neither on this fd.
-
-     If a watcher is asked to poll for reads or writes, the read_watcher
-     or write_watcher fields are set respectively. A watcher may be asked
-     to poll for both, in which case both fields will be set.
-
-     read_watcher and write_watcher may be NULL if no watcher has been
-     asked to poll for reads or writes.
-
-     If an fd_watcher is not asked to poll for reads or writes, it's added
-     to a linked list of inactive watchers, rooted at inactive_watcher_root.
-     If at a later time there becomes need of a poller to poll, one of
-     the inactive pollers may be kicked out of their poll loops to take
-     that responsibility. */
-  grpc_fd_watcher inactive_watcher_root;
-  grpc_fd_watcher *read_watcher;
-  grpc_fd_watcher *write_watcher;
-
-  grpc_closure *read_closure;
-  grpc_closure *write_closure;
-
-  struct grpc_fd *freelist_next;
-
-  grpc_closure *on_done_closure;
-
-  grpc_iomgr_object iomgr_object;
-};
-
-/* Begin polling on an fd.
-   Registers that the given pollset is interested in this fd - so that if read
-   or writability interest changes, the pollset can be kicked to pick up that
-   new interest.
-   Return value is:
-     (fd_needs_read? read_mask : 0) | (fd_needs_write? write_mask : 0)
-   i.e. a combination of read_mask and write_mask determined by the fd's current
-   interest in said events.
-   Polling strategies that do not need to alter their behavior depending on the
-   fd's current interest (such as epoll) do not need to call this function.
-   MUST NOT be called with a pollset lock taken */
-static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
-                              grpc_pollset_worker *worker, uint32_t read_mask,
-                              uint32_t write_mask, grpc_fd_watcher *rec);
-/* Complete polling previously started with fd_begin_poll
-   MUST NOT be called with a pollset lock taken
-   if got_read or got_write are 1, also does the become_{readable,writable} as
-   appropriate. */
-static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
-
-/* Return 1 if this fd is orphaned, 0 otherwise */
-static bool fd_is_orphaned(grpc_fd *fd);
-
-/* Reference counting for fds */
-/*#define GRPC_FD_REF_COUNT_DEBUG*/
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line);
-static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
-                     int line);
-#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__)
-#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__)
-#else
-static void fd_ref(grpc_fd *fd);
-static void fd_unref(grpc_fd *fd);
-#define GRPC_FD_REF(fd, reason) fd_ref(fd)
-#define GRPC_FD_UNREF(fd, reason) fd_unref(fd)
-#endif
-
-static void fd_global_init(void);
-static void fd_global_shutdown(void);
-
-#define CLOSURE_NOT_READY ((grpc_closure *)0)
-#define CLOSURE_READY ((grpc_closure *)1)
-
-/*******************************************************************************
- * pollset declarations
- */
-
-typedef struct grpc_pollset_vtable grpc_pollset_vtable;
-
-typedef struct grpc_cached_wakeup_fd {
-  grpc_wakeup_fd fd;
-  struct grpc_cached_wakeup_fd *next;
-} grpc_cached_wakeup_fd;
-
-struct grpc_pollset_worker {
-  grpc_cached_wakeup_fd *wakeup_fd;
-  int reevaluate_polling_on_wakeup;
-  int kicked_specifically;
-  struct grpc_pollset_worker *next;
-  struct grpc_pollset_worker *prev;
-};
-
-struct grpc_pollset {
-  /* pollsets under posix can mutate representation as fds are added and
-     removed.
-     For example, we may choose a poll() based implementation on linux for
-     few fds, and an epoll() based implementation for many fds */
-  const grpc_pollset_vtable *vtable;
-  gpr_mu mu;
-  grpc_pollset_worker root_worker;
-  int in_flight_cbs;
-  int shutting_down;
-  int called_shutdown;
-  int kicked_without_pollers;
-  grpc_closure *shutdown_done;
-  grpc_closure_list idle_jobs;
-  union {
-    int fd;
-    void *ptr;
-  } data;
-  /* Local cache of eventfds for workers */
-  grpc_cached_wakeup_fd *local_wakeup_cache;
-};
-
-struct grpc_pollset_vtable {
-  void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                 struct grpc_fd *fd, int and_unlock_pollset);
-  void (*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                grpc_pollset_worker *worker,
-                                gpr_timespec deadline, gpr_timespec now);
-  void (*finish_shutdown)(grpc_pollset *pollset);
-  void (*destroy)(grpc_pollset *pollset);
-};
-
-/* Add an fd to a pollset */
-static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           struct grpc_fd *fd);
-
-static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd);
-
-/* Convert a timespec to milliseconds:
-   - very small or negative poll times are clamped to zero to do a
-     non-blocking poll (which becomes spin polling)
-   - other small values are rounded up to one millisecond
-   - longer than a millisecond polls are rounded up to the next nearest
-     millisecond to avoid spinning
-   - infinite timeouts are converted to -1 */
-static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
-                                           gpr_timespec now);
-
-/* Allow kick to wakeup the currently polling worker */
-#define GRPC_POLLSET_CAN_KICK_SELF 1
-/* Force the wakee to repoll when awoken */
-#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2
-/* As per pollset_kick, with an extended set of flags (defined above)
-   -- mostly for fd_posix's use. */
-static void pollset_kick_ext(grpc_pollset *p,
-                             grpc_pollset_worker *specific_worker,
-                             uint32_t flags);
-
-/* turn a pollset into a multipoller: platform specific */
-typedef void (*platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx,
-                                                 grpc_pollset *pollset,
-                                                 struct grpc_fd **fds,
-                                                 size_t fd_count);
-static platform_become_multipoller_type platform_become_multipoller;
-
-/* Return 1 if the pollset has active threads in pollset_work (pollset must
- * be locked) */
-static int pollset_has_workers(grpc_pollset *pollset);
-
-static void remove_fd_from_all_epoll_sets(int fd);
-
-/*******************************************************************************
- * pollset_set definitions
- */
-
-struct grpc_pollset_set {
-  gpr_mu mu;
-
-  size_t pollset_count;
-  size_t pollset_capacity;
-  grpc_pollset **pollsets;
-
-  size_t pollset_set_count;
-  size_t pollset_set_capacity;
-  struct grpc_pollset_set **pollset_sets;
-
-  size_t fd_count;
-  size_t fd_capacity;
-  grpc_fd **fds;
-};
-
-/*******************************************************************************
- * fd_posix.c
- */
-
-/* We need to keep a freelist not because of any concerns of malloc performance
- * but instead so that implementations with multiple threads in (for example)
- * epoll_wait deal with the race between pollset removal and incoming poll
- * notifications.
- *
- * The problem is that the poller ultimately holds a reference to this
- * object, so it is very difficult to know when is safe to free it, at least
- * without some expensive synchronization.
- *
- * If we keep the object freelisted, in the worst case losing this race just
- * becomes a spurious read notification on a reused fd.
- */
-/* TODO(klempner): We could use some form of polling generation count to know
- * when these are safe to free. */
-/* TODO(klempner): Consider disabling freelisting if we don't have multiple
- * threads in poll on the same fd */
-/* TODO(klempner): Batch these allocations to reduce fragmentation */
-static grpc_fd *fd_freelist = NULL;
-static gpr_mu fd_freelist_mu;
-
-static void freelist_fd(grpc_fd *fd) {
-  gpr_mu_lock(&fd_freelist_mu);
-  fd->freelist_next = fd_freelist;
-  fd_freelist = fd;
-  grpc_iomgr_unregister_object(&fd->iomgr_object);
-  gpr_mu_unlock(&fd_freelist_mu);
-}
-
-static grpc_fd *alloc_fd(int fd) {
-  grpc_fd *r = NULL;
-  gpr_mu_lock(&fd_freelist_mu);
-  if (fd_freelist != NULL) {
-    r = fd_freelist;
-    fd_freelist = fd_freelist->freelist_next;
-  }
-  gpr_mu_unlock(&fd_freelist_mu);
-  if (r == NULL) {
-    r = gpr_malloc(sizeof(grpc_fd));
-    gpr_mu_init(&r->mu);
-  }
-
-  gpr_mu_lock(&r->mu);
-  gpr_atm_rel_store(&r->refst, 1);
-  r->shutdown = 0;
-  r->read_closure = CLOSURE_NOT_READY;
-  r->write_closure = CLOSURE_NOT_READY;
-  r->fd = fd;
-  r->inactive_watcher_root.next = r->inactive_watcher_root.prev =
-      &r->inactive_watcher_root;
-  r->freelist_next = NULL;
-  r->read_watcher = r->write_watcher = NULL;
-  r->on_done_closure = NULL;
-  r->closed = 0;
-  r->released = 0;
-  gpr_mu_unlock(&r->mu);
-  return r;
-}
-
-static void destroy(grpc_fd *fd) {
-  gpr_mu_destroy(&fd->mu);
-  gpr_free(fd);
-}
-
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__)
-#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__)
-static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file,
-                   int line) {
-  gpr_log(GPR_DEBUG, "FD %d %p   ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
-          gpr_atm_no_barrier_load(&fd->refst),
-          gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line);
-#else
-#define REF_BY(fd, n, reason) ref_by(fd, n)
-#define UNREF_BY(fd, n, reason) unref_by(fd, n)
-static void ref_by(grpc_fd *fd, int n) {
-#endif
-  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
-}
-
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file,
-                     int line) {
-  gpr_atm old;
-  gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
-          gpr_atm_no_barrier_load(&fd->refst),
-          gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line);
-#else
-static void unref_by(grpc_fd *fd, int n) {
-  gpr_atm old;
-#endif
-  old = gpr_atm_full_fetch_add(&fd->refst, -n);
-  if (old == n) {
-    freelist_fd(fd);
-  } else {
-    GPR_ASSERT(old > n);
-  }
-}
-
-static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); }
-
-static void fd_global_shutdown(void) {
-  gpr_mu_lock(&fd_freelist_mu);
-  gpr_mu_unlock(&fd_freelist_mu);
-  while (fd_freelist != NULL) {
-    grpc_fd *fd = fd_freelist;
-    fd_freelist = fd_freelist->freelist_next;
-    destroy(fd);
-  }
-  gpr_mu_destroy(&fd_freelist_mu);
-}
-
-static grpc_fd *fd_create(int fd, const char *name) {
-  grpc_fd *r = alloc_fd(fd);
-  char *name2;
-  gpr_asprintf(&name2, "%s fd=%d", name, fd);
-  grpc_iomgr_register_object(&r->iomgr_object, name2);
-  gpr_free(name2);
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-  gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name);
-#endif
-  return r;
-}
-
-static bool fd_is_orphaned(grpc_fd *fd) {
-  return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
-}
-
-static void pollset_kick_locked(grpc_fd_watcher *watcher) {
-  gpr_mu_lock(&watcher->pollset->mu);
-  GPR_ASSERT(watcher->worker);
-  pollset_kick_ext(watcher->pollset, watcher->worker,
-                   GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
-  gpr_mu_unlock(&watcher->pollset->mu);
-}
-
-static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
-  if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) {
-    pollset_kick_locked(fd->inactive_watcher_root.next);
-  } else if (fd->read_watcher) {
-    pollset_kick_locked(fd->read_watcher);
-  } else if (fd->write_watcher) {
-    pollset_kick_locked(fd->write_watcher);
-  }
-}
-
-static void wake_all_watchers_locked(grpc_fd *fd) {
-  grpc_fd_watcher *watcher;
-  for (watcher = fd->inactive_watcher_root.next;
-       watcher != &fd->inactive_watcher_root; watcher = watcher->next) {
-    pollset_kick_locked(watcher);
-  }
-  if (fd->read_watcher) {
-    pollset_kick_locked(fd->read_watcher);
-  }
-  if (fd->write_watcher && fd->write_watcher != fd->read_watcher) {
-    pollset_kick_locked(fd->write_watcher);
-  }
-}
-
-static int has_watchers(grpc_fd *fd) {
-  return fd->read_watcher != NULL || fd->write_watcher != NULL ||
-         fd->inactive_watcher_root.next != &fd->inactive_watcher_root;
-}
-
-static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  fd->closed = 1;
-  if (!fd->released) {
-    close(fd->fd);
-  } else {
-    remove_fd_from_all_epoll_sets(fd->fd);
-  }
-  grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, true, NULL);
-}
-
-static int fd_wrapped_fd(grpc_fd *fd) {
-  if (fd->released || fd->closed) {
-    return -1;
-  } else {
-    return fd->fd;
-  }
-}
-
-static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                      grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  fd->on_done_closure = on_done;
-  fd->released = release_fd != NULL;
-  if (!fd->released) {
-    shutdown(fd->fd, SHUT_RDWR);
-  } else {
-    *release_fd = fd->fd;
-  }
-  gpr_mu_lock(&fd->mu);
-  REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
-  if (!has_watchers(fd)) {
-    close_fd_locked(exec_ctx, fd);
-  } else {
-    wake_all_watchers_locked(fd);
-  }
-  gpr_mu_unlock(&fd->mu);
-  UNREF_BY(fd, 2, reason); /* drop the reference */
-}
-
-/* increment refcount by two to avoid changing the orphan bit */
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-static void fd_ref(grpc_fd *fd, const char *reason, const char *file,
-                   int line) {
-  ref_by(fd, 2, reason, file, line);
-}
-
-static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
-                     int line) {
-  unref_by(fd, 2, reason, file, line);
-}
-#else
-static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
-
-static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
-#endif
-
-static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                             grpc_closure **st, grpc_closure *closure) {
-  if (*st == CLOSURE_NOT_READY) {
-    /* not ready ==> switch to a waiting state by setting the closure */
-    *st = closure;
-  } else if (*st == CLOSURE_READY) {
-    /* already ready ==> queue the closure to run immediately */
-    *st = CLOSURE_NOT_READY;
-    grpc_exec_ctx_enqueue(exec_ctx, closure, !fd->shutdown, NULL);
-    maybe_wake_one_watcher_locked(fd);
-  } else {
-    /* upcallptr was set to a different closure.  This is an error! */
-    gpr_log(GPR_ERROR,
-            "User called a notify_on function with a previous callback still "
-            "pending");
-    abort();
-  }
-}
-
-/* returns 1 if state becomes not ready */
-static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                            grpc_closure **st) {
-  if (*st == CLOSURE_READY) {
-    /* duplicate ready ==> ignore */
-    return 0;
-  } else if (*st == CLOSURE_NOT_READY) {
-    /* not ready, and not waiting ==> flag ready */
-    *st = CLOSURE_READY;
-    return 0;
-  } else {
-    /* waiting ==> queue closure */
-    grpc_exec_ctx_enqueue(exec_ctx, *st, !fd->shutdown, NULL);
-    *st = CLOSURE_NOT_READY;
-    return 1;
-  }
-}
-
-static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  gpr_mu_lock(&fd->mu);
-  GPR_ASSERT(!fd->shutdown);
-  fd->shutdown = 1;
-  set_ready_locked(exec_ctx, fd, &fd->read_closure);
-  set_ready_locked(exec_ctx, fd, &fd->write_closure);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                              grpc_closure *closure) {
-  gpr_mu_lock(&fd->mu);
-  notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                               grpc_closure *closure) {
-  gpr_mu_lock(&fd->mu);
-  notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
-                              grpc_pollset_worker *worker, uint32_t read_mask,
-                              uint32_t write_mask, grpc_fd_watcher *watcher) {
-  uint32_t mask = 0;
-  grpc_closure *cur;
-  int requested;
-  /* keep track of pollers that have requested our events, in case they change
-   */
-  GRPC_FD_REF(fd, "poll");
-
-  gpr_mu_lock(&fd->mu);
-
-  /* if we are shutdown, then don't add to the watcher set */
-  if (fd->shutdown) {
-    watcher->fd = NULL;
-    watcher->pollset = NULL;
-    watcher->worker = NULL;
-    gpr_mu_unlock(&fd->mu);
-    GRPC_FD_UNREF(fd, "poll");
-    return 0;
-  }
-
-  /* if there is nobody polling for read, but we need to, then start doing so */
-  cur = fd->read_closure;
-  requested = cur != CLOSURE_READY;
-  if (read_mask && fd->read_watcher == NULL && requested) {
-    fd->read_watcher = watcher;
-    mask |= read_mask;
-  }
-  /* if there is nobody polling for write, but we need to, then start doing so
-   */
-  cur = fd->write_closure;
-  requested = cur != CLOSURE_READY;
-  if (write_mask && fd->write_watcher == NULL && requested) {
-    fd->write_watcher = watcher;
-    mask |= write_mask;
-  }
-  /* if not polling, remember this watcher in case we need someone to later */
-  if (mask == 0 && worker != NULL) {
-    watcher->next = &fd->inactive_watcher_root;
-    watcher->prev = watcher->next->prev;
-    watcher->next->prev = watcher->prev->next = watcher;
-  }
-  watcher->pollset = pollset;
-  watcher->worker = worker;
-  watcher->fd = fd;
-  gpr_mu_unlock(&fd->mu);
-
-  return mask;
-}
-
-static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
-  int was_polling = 0;
-  int kick = 0;
-  grpc_fd *fd = watcher->fd;
-
-  if (fd == NULL) {
-    return;
-  }
-
-  gpr_mu_lock(&fd->mu);
-
-  if (watcher == fd->read_watcher) {
-    /* remove read watcher, kick if we still need a read */
-    was_polling = 1;
-    if (!got_read) {
-      kick = 1;
-    }
-    fd->read_watcher = NULL;
-  }
-  if (watcher == fd->write_watcher) {
-    /* remove write watcher, kick if we still need a write */
-    was_polling = 1;
-    if (!got_write) {
-      kick = 1;
-    }
-    fd->write_watcher = NULL;
-  }
-  if (!was_polling && watcher->worker != NULL) {
-    /* remove from inactive list */
-    watcher->next->prev = watcher->prev;
-    watcher->prev->next = watcher->next;
-  }
-  if (got_read) {
-    if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
-      kick = 1;
-    }
-  }
-  if (got_write) {
-    if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
-      kick = 1;
-    }
-  }
-  if (kick) {
-    maybe_wake_one_watcher_locked(fd);
-  }
-  if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) {
-    close_fd_locked(exec_ctx, fd);
-  }
-  gpr_mu_unlock(&fd->mu);
-
-  GRPC_FD_UNREF(fd, "poll");
-}
-
-/*******************************************************************************
- * pollset_posix.c
- */
-
-GPR_TLS_DECL(g_current_thread_poller);
-GPR_TLS_DECL(g_current_thread_worker);
-
-/** The alarm system needs to be able to wakeup 'some poller' sometimes
- *  (specifically when a new alarm needs to be triggered earlier than the next
- *  alarm 'epoch').
- *  This wakeup_fd gives us something to alert on when such a case occurs. */
-grpc_wakeup_fd grpc_global_wakeup_fd;
-
-static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
-  worker->prev->next = worker->next;
-  worker->next->prev = worker->prev;
-}
-
-static int pollset_has_workers(grpc_pollset *p) {
-  return p->root_worker.next != &p->root_worker;
-}
-
-static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
-  if (pollset_has_workers(p)) {
-    grpc_pollset_worker *w = p->root_worker.next;
-    remove_worker(p, w);
-    return w;
-  } else {
-    return NULL;
-  }
-}
-
-static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
-  worker->next = &p->root_worker;
-  worker->prev = worker->next->prev;
-  worker->prev->next = worker->next->prev = worker;
-}
-
-static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
-  worker->prev = &p->root_worker;
-  worker->next = worker->prev->next;
-  worker->prev->next = worker->next->prev = worker;
-}
-
-static void pollset_kick_ext(grpc_pollset *p,
-                             grpc_pollset_worker *specific_worker,
-                             uint32_t flags) {
-  GPR_TIMER_BEGIN("pollset_kick_ext", 0);
-
-  /* pollset->mu already held */
-  if (specific_worker != NULL) {
-    if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
-      GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0);
-      GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
-      for (specific_worker = p->root_worker.next;
-           specific_worker != &p->root_worker;
-           specific_worker = specific_worker->next) {
-        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-      }
-      p->kicked_without_pollers = 1;
-      GPR_TIMER_END("pollset_kick_ext.broadcast", 0);
-    } else if (gpr_tls_get(&g_current_thread_worker) !=
-               (intptr_t)specific_worker) {
-      GPR_TIMER_MARK("different_thread_worker", 0);
-      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
-        specific_worker->reevaluate_polling_on_wakeup = 1;
-      }
-      specific_worker->kicked_specifically = 1;
-      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-    } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) {
-      GPR_TIMER_MARK("kick_yoself", 0);
-      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
-        specific_worker->reevaluate_polling_on_wakeup = 1;
-      }
-      specific_worker->kicked_specifically = 1;
-      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-    }
-  } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) {
-    GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
-    GPR_TIMER_MARK("kick_anonymous", 0);
-    specific_worker = pop_front_worker(p);
-    if (specific_worker != NULL) {
-      if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) {
-        GPR_TIMER_MARK("kick_anonymous_not_self", 0);
-        push_back_worker(p, specific_worker);
-        specific_worker = pop_front_worker(p);
-        if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 &&
-            gpr_tls_get(&g_current_thread_worker) ==
-                (intptr_t)specific_worker) {
-          push_back_worker(p, specific_worker);
-          specific_worker = NULL;
-        }
-      }
-      if (specific_worker != NULL) {
-        GPR_TIMER_MARK("finally_kick", 0);
-        push_back_worker(p, specific_worker);
-        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-      }
-    } else {
-      GPR_TIMER_MARK("kicked_no_pollers", 0);
-      p->kicked_without_pollers = 1;
-    }
-  }
-
-  GPR_TIMER_END("pollset_kick_ext", 0);
-}
-
-static void pollset_kick(grpc_pollset *p,
-                         grpc_pollset_worker *specific_worker) {
-  pollset_kick_ext(p, specific_worker, 0);
-}
-
-/* global state management */
-
-static void pollset_global_init(void) {
-  gpr_tls_init(&g_current_thread_poller);
-  gpr_tls_init(&g_current_thread_worker);
-  grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
-}
-
-static void pollset_global_shutdown(void) {
-  grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd);
-  gpr_tls_destroy(&g_current_thread_poller);
-  gpr_tls_destroy(&g_current_thread_worker);
-}
-
-static void kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); }
-
-/* main interface */
-
-static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null);
-
-static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
-  gpr_mu_init(&pollset->mu);
-  *mu = &pollset->mu;
-  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
-  pollset->in_flight_cbs = 0;
-  pollset->shutting_down = 0;
-  pollset->called_shutdown = 0;
-  pollset->kicked_without_pollers = 0;
-  pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL;
-  pollset->local_wakeup_cache = NULL;
-  pollset->kicked_without_pollers = 0;
-  become_basic_pollset(pollset, NULL);
-}
-
-static void pollset_destroy(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
-  GPR_ASSERT(!pollset_has_workers(pollset));
-  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
-  pollset->vtable->destroy(pollset);
-  while (pollset->local_wakeup_cache) {
-    grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next;
-    grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd);
-    gpr_free(pollset->local_wakeup_cache);
-    pollset->local_wakeup_cache = next;
-  }
-  gpr_mu_destroy(&pollset->mu);
-}
-
-static void pollset_reset(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->shutting_down);
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
-  GPR_ASSERT(!pollset_has_workers(pollset));
-  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
-  pollset->vtable->destroy(pollset);
-  pollset->shutting_down = 0;
-  pollset->called_shutdown = 0;
-  pollset->kicked_without_pollers = 0;
-  become_basic_pollset(pollset, NULL);
-}
-
-static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           grpc_fd *fd) {
-  gpr_mu_lock(&pollset->mu);
-  pollset->vtable->add_fd(exec_ctx, pollset, fd, 1);
-/* the following (enabled only in debug) will reacquire and then release
-   our lock - meaning that if the unlocking flag passed to add_fd above is
-   not respected, the code will deadlock (in a way that we have a chance of
-   debugging) */
-#ifndef NDEBUG
-  gpr_mu_lock(&pollset->mu);
-  gpr_mu_unlock(&pollset->mu);
-#endif
-}
-
-static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
-  GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs));
-  pollset->vtable->finish_shutdown(pollset);
-  grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL);
-}
-
-static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                         grpc_pollset_worker **worker_hdl, gpr_timespec now,
-                         gpr_timespec deadline) {
-  grpc_pollset_worker worker;
-  *worker_hdl = &worker;
-
-  /* pollset->mu already held */
-  int added_worker = 0;
-  int locked = 1;
-  int queued_work = 0;
-  int keep_polling = 0;
-  GPR_TIMER_BEGIN("pollset_work", 0);
-  /* this must happen before we (potentially) drop pollset->mu */
-  worker.next = worker.prev = NULL;
-  worker.reevaluate_polling_on_wakeup = 0;
-  if (pollset->local_wakeup_cache != NULL) {
-    worker.wakeup_fd = pollset->local_wakeup_cache;
-    pollset->local_wakeup_cache = worker.wakeup_fd->next;
-  } else {
-    worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
-    grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
-  }
-  worker.kicked_specifically = 0;
-  /* If there's work waiting for the pollset to be idle, and the
-     pollset is idle, then do that work */
-  if (!pollset_has_workers(pollset) &&
-      !grpc_closure_list_empty(pollset->idle_jobs)) {
-    GPR_TIMER_MARK("pollset_work.idle_jobs", 0);
-    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
-    goto done;
-  }
-  /* If we're shutting down then we don't execute any extended work */
-  if (pollset->shutting_down) {
-    GPR_TIMER_MARK("pollset_work.shutting_down", 0);
-    goto done;
-  }
-  /* Give do_promote priority so we don't starve it out */
-  if (pollset->in_flight_cbs) {
-    GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0);
-    gpr_mu_unlock(&pollset->mu);
-    locked = 0;
-    goto done;
-  }
-  /* Start polling, and keep doing so while we're being asked to
-     re-evaluate our pollers (this allows poll() based pollers to
-     ensure they don't miss wakeups) */
-  keep_polling = 1;
-  while (keep_polling) {
-    keep_polling = 0;
-    if (!pollset->kicked_without_pollers) {
-      if (!added_worker) {
-        push_front_worker(pollset, &worker);
-        added_worker = 1;
-        gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
-      }
-      gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
-      GPR_TIMER_BEGIN("maybe_work_and_unlock", 0);
-      pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, &worker,
-                                             deadline, now);
-      GPR_TIMER_END("maybe_work_and_unlock", 0);
-      locked = 0;
-      gpr_tls_set(&g_current_thread_poller, 0);
-    } else {
-      GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0);
-      pollset->kicked_without_pollers = 0;
-    }
-  /* Finished execution - start cleaning up.
-     Note that we may arrive here from outside the enclosing while() loop.
-     In that case we won't loop though as we haven't added worker to the
-     worker list, which means nobody could ask us to re-evaluate polling). */
-  done:
-    if (!locked) {
-      queued_work |= grpc_exec_ctx_flush(exec_ctx);
-      gpr_mu_lock(&pollset->mu);
-      locked = 1;
-    }
-    /* If we're forced to re-evaluate polling (via pollset_kick with
-       GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force
-       a loop */
-    if (worker.reevaluate_polling_on_wakeup) {
-      worker.reevaluate_polling_on_wakeup = 0;
-      pollset->kicked_without_pollers = 0;
-      if (queued_work || worker.kicked_specifically) {
-        /* If there's queued work on the list, then set the deadline to be
-           immediate so we get back out of the polling loop quickly */
-        deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
-      }
-      keep_polling = 1;
-    }
-  }
-  if (added_worker) {
-    remove_worker(pollset, &worker);
-    gpr_tls_set(&g_current_thread_worker, 0);
-  }
-  /* release wakeup fd to the local pool */
-  worker.wakeup_fd->next = pollset->local_wakeup_cache;
-  pollset->local_wakeup_cache = worker.wakeup_fd;
-  /* check shutdown conditions */
-  if (pollset->shutting_down) {
-    if (pollset_has_workers(pollset)) {
-      pollset_kick(pollset, NULL);
-    } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
-      pollset->called_shutdown = 1;
-      gpr_mu_unlock(&pollset->mu);
-      finish_shutdown(exec_ctx, pollset);
-      grpc_exec_ctx_flush(exec_ctx);
-      /* Continuing to access pollset here is safe -- it is the caller's
-       * responsibility to not destroy when it has outstanding calls to
-       * pollset_work.
-       * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */
-      gpr_mu_lock(&pollset->mu);
-    } else if (!grpc_closure_list_empty(pollset->idle_jobs)) {
-      grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
-      gpr_mu_unlock(&pollset->mu);
-      grpc_exec_ctx_flush(exec_ctx);
-      gpr_mu_lock(&pollset->mu);
-    }
-  }
-  *worker_hdl = NULL;
-  GPR_TIMER_END("pollset_work", 0);
-}
-
-static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                             grpc_closure *closure) {
-  GPR_ASSERT(!pollset->shutting_down);
-  pollset->shutting_down = 1;
-  pollset->shutdown_done = closure;
-  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
-  if (!pollset_has_workers(pollset)) {
-    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
-  }
-  if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
-      !pollset_has_workers(pollset)) {
-    pollset->called_shutdown = 1;
-    finish_shutdown(exec_ctx, pollset);
-  }
-}
-
-static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
-                                           gpr_timespec now) {
-  gpr_timespec timeout;
-  static const int64_t max_spin_polling_us = 10;
-  if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) {
-    return -1;
-  }
-  if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros(
-                                                   max_spin_polling_us,
-                                                   GPR_TIMESPAN))) <= 0) {
-    return 0;
-  }
-  timeout = gpr_time_sub(deadline, now);
-  return gpr_time_to_millis(gpr_time_add(
-      timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN)));
-}
-
-/*
- * basic_pollset - a vtable that provides polling for zero or one file
- *                 descriptor via poll()
- */
-
-typedef struct grpc_unary_promote_args {
-  const grpc_pollset_vtable *original_vtable;
-  grpc_pollset *pollset;
-  grpc_fd *fd;
-  grpc_closure promotion_closure;
-} grpc_unary_promote_args;
-
-static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args,
-                             bool success) {
-  grpc_unary_promote_args *up_args = args;
-  const grpc_pollset_vtable *original_vtable = up_args->original_vtable;
-  grpc_pollset *pollset = up_args->pollset;
-  grpc_fd *fd = up_args->fd;
-
-  /*
-   * This is quite tricky. There are a number of cases to keep in mind here:
-   * 1. fd may have been orphaned
-   * 2. The pollset may no longer be a unary poller (and we can't let case #1
-   * leak to other pollset types!)
-   * 3. pollset's fd (which may have changed) may have been orphaned
-   * 4. The pollset may be shutting down.
-   */
-
-  gpr_mu_lock(&pollset->mu);
-  /* First we need to ensure that nobody is polling concurrently */
-  GPR_ASSERT(!pollset_has_workers(pollset));
-
-  gpr_free(up_args);
-  /* At this point the pollset may no longer be a unary poller. In that case
-   * we should just call the right add function and be done. */
-  /* TODO(klempner): If we're not careful this could cause infinite recursion.
-   * That's not a problem for now because empty_pollset has a trivial poller
-   * and we don't have any mechanism to unbecome multipoller. */
-  pollset->in_flight_cbs--;
-  if (pollset->shutting_down) {
-    /* We don't care about this pollset anymore. */
-    if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) {
-      pollset->called_shutdown = 1;
-      finish_shutdown(exec_ctx, pollset);
-    }
-  } else if (fd_is_orphaned(fd)) {
-    /* Don't try to add it to anything, we'll drop our ref on it below */
-  } else if (pollset->vtable != original_vtable) {
-    pollset->vtable->add_fd(exec_ctx, pollset, fd, 0);
-  } else if (fd != pollset->data.ptr) {
-    grpc_fd *fds[2];
-    fds[0] = pollset->data.ptr;
-    fds[1] = fd;
-
-    if (fds[0] && !fd_is_orphaned(fds[0])) {
-      platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds));
-      GRPC_FD_UNREF(fds[0], "basicpoll");
-    } else {
-      /* old fd is orphaned and we haven't cleaned it up until now, so remain a
-       * unary poller */
-      /* Note that it is possible that fds[1] is also orphaned at this point.
-       * That's okay, we'll correct it at the next add or poll. */
-      if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll");
-      pollset->data.ptr = fd;
-      GRPC_FD_REF(fd, "basicpoll");
-    }
-  }
-
-  gpr_mu_unlock(&pollset->mu);
-
-  /* Matching ref in basic_pollset_add_fd */
-  GRPC_FD_UNREF(fd, "basicpoll_add");
-}
-
-static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                 grpc_fd *fd, int and_unlock_pollset) {
-  grpc_unary_promote_args *up_args;
-  GPR_ASSERT(fd);
-  if (fd == pollset->data.ptr) goto exit;
-
-  if (!pollset_has_workers(pollset)) {
-    /* Fast path -- no in flight cbs */
-    /* TODO(klempner): Comment this out and fix any test failures or establish
-     * they are due to timing issues */
-    grpc_fd *fds[2];
-    fds[0] = pollset->data.ptr;
-    fds[1] = fd;
-
-    if (fds[0] == NULL) {
-      pollset->data.ptr = fd;
-      GRPC_FD_REF(fd, "basicpoll");
-    } else if (!fd_is_orphaned(fds[0])) {
-      platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds));
-      GRPC_FD_UNREF(fds[0], "basicpoll");
-    } else {
-      /* old fd is orphaned and we haven't cleaned it up until now, so remain a
-       * unary poller */
-      GRPC_FD_UNREF(fds[0], "basicpoll");
-      pollset->data.ptr = fd;
-      GRPC_FD_REF(fd, "basicpoll");
-    }
-    goto exit;
-  }
-
-  /* Now we need to promote. This needs to happen when we're not polling. Since
-   * this may be called from poll, the wait needs to happen asynchronously. */
-  GRPC_FD_REF(fd, "basicpoll_add");
-  pollset->in_flight_cbs++;
-  up_args = gpr_malloc(sizeof(*up_args));
-  up_args->fd = fd;
-  up_args->original_vtable = pollset->vtable;
-  up_args->pollset = pollset;
-  up_args->promotion_closure.cb = basic_do_promote;
-  up_args->promotion_closure.cb_arg = up_args;
-
-  grpc_closure_list_add(&pollset->idle_jobs, &up_args->promotion_closure, 1);
-  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
-
-exit:
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-}
-
-static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
-                                                grpc_pollset *pollset,
-                                                grpc_pollset_worker *worker,
-                                                gpr_timespec deadline,
-                                                gpr_timespec now) {
-#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
-#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
-
-  struct pollfd pfd[3];
-  grpc_fd *fd;
-  grpc_fd_watcher fd_watcher;
-  int timeout;
-  int r;
-  nfds_t nfds;
-
-  fd = pollset->data.ptr;
-  if (fd && fd_is_orphaned(fd)) {
-    GRPC_FD_UNREF(fd, "basicpoll");
-    fd = pollset->data.ptr = NULL;
-  }
-  timeout = poll_deadline_to_millis_timeout(deadline, now);
-  pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
-  pfd[0].events = POLLIN;
-  pfd[0].revents = 0;
-  pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
-  pfd[1].events = POLLIN;
-  pfd[1].revents = 0;
-  nfds = 2;
-  if (fd) {
-    pfd[2].fd = fd->fd;
-    pfd[2].revents = 0;
-    GRPC_FD_REF(fd, "basicpoll_begin");
-    gpr_mu_unlock(&pollset->mu);
-    pfd[2].events =
-        (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher);
-    if (pfd[2].events != 0) {
-      nfds++;
-    }
-  } else {
-    gpr_mu_unlock(&pollset->mu);
-  }
-
-  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
-     even going into the blocking annotation if possible */
-  /* poll fd count (argument 2) is shortened by one if we have no events
-     to poll on - such that it only includes the kicker */
-  GPR_TIMER_BEGIN("poll", 0);
-  GRPC_SCHEDULING_START_BLOCKING_REGION;
-  r = grpc_poll_function(pfd, nfds, timeout);
-  GRPC_SCHEDULING_END_BLOCKING_REGION;
-  GPR_TIMER_END("poll", 0);
-
-  if (r < 0) {
-    if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
-    }
-    if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
-    }
-  } else if (r == 0) {
-    if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
-    }
-  } else {
-    if (pfd[0].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
-    }
-    if (pfd[1].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
-    }
-    if (nfds > 2) {
-      fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK,
-                  pfd[2].revents & POLLOUT_CHECK);
-    } else if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
-    }
-  }
-
-  if (fd) {
-    GRPC_FD_UNREF(fd, "basicpoll_begin");
-  }
-}
-
-static void basic_pollset_destroy(grpc_pollset *pollset) {
-  if (pollset->data.ptr != NULL) {
-    GRPC_FD_UNREF(pollset->data.ptr, "basicpoll");
-    pollset->data.ptr = NULL;
-  }
-}
-
-static const grpc_pollset_vtable basic_pollset = {
-    basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock,
-    basic_pollset_destroy, basic_pollset_destroy};
-
-static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) {
-  pollset->vtable = &basic_pollset;
-  pollset->data.ptr = fd_or_null;
-  if (fd_or_null != NULL) {
-    GRPC_FD_REF(fd_or_null, "basicpoll");
-  }
-}
-
-/*******************************************************************************
- * pollset_multipoller_with_poll_posix.c
- */
-
-#ifndef GPR_LINUX_MULTIPOLL_WITH_EPOLL
-
-typedef struct {
-  /* all polled fds */
-  size_t fd_count;
-  size_t fd_capacity;
-  grpc_fd **fds;
-  /* fds that have been removed from the pollset explicitly */
-  size_t del_count;
-  size_t del_capacity;
-  grpc_fd **dels;
-} poll_hdr;
-
-static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
-                                               grpc_pollset *pollset,
-                                               grpc_fd *fd,
-                                               int and_unlock_pollset) {
-  size_t i;
-  poll_hdr *h = pollset->data.ptr;
-  /* TODO(ctiller): this is O(num_fds^2); maybe switch to a hash set here */
-  for (i = 0; i < h->fd_count; i++) {
-    if (h->fds[i] == fd) goto exit;
-  }
-  if (h->fd_count == h->fd_capacity) {
-    h->fd_capacity = GPR_MAX(h->fd_capacity + 8, h->fd_count * 3 / 2);
-    h->fds = gpr_realloc(h->fds, sizeof(grpc_fd *) * h->fd_capacity);
-  }
-  h->fds[h->fd_count++] = fd;
-  GRPC_FD_REF(fd, "multipoller");
-exit:
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-}
-
-static void multipoll_with_poll_pollset_maybe_work_and_unlock(
-    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
-    gpr_timespec deadline, gpr_timespec now) {
-#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
-#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
-
-  int timeout;
-  int r;
-  size_t i, j, fd_count;
-  nfds_t pfd_count;
-  poll_hdr *h;
-  /* TODO(ctiller): inline some elements to avoid an allocation */
-  grpc_fd_watcher *watchers;
-  struct pollfd *pfds;
-
-  h = pollset->data.ptr;
-  timeout = poll_deadline_to_millis_timeout(deadline, now);
-  /* TODO(ctiller): perform just one malloc here if we exceed the inline case */
-  pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 2));
-  watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 2));
-  fd_count = 0;
-  pfd_count = 2;
-  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
-  pfds[0].events = POLLIN;
-  pfds[0].revents = 0;
-  pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
-  pfds[1].events = POLLIN;
-  pfds[1].revents = 0;
-  for (i = 0; i < h->fd_count; i++) {
-    int remove = fd_is_orphaned(h->fds[i]);
-    for (j = 0; !remove && j < h->del_count; j++) {
-      if (h->fds[i] == h->dels[j]) remove = 1;
-    }
-    if (remove) {
-      GRPC_FD_UNREF(h->fds[i], "multipoller");
-    } else {
-      h->fds[fd_count++] = h->fds[i];
-      watchers[pfd_count].fd = h->fds[i];
-      GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start");
-      pfds[pfd_count].fd = h->fds[i]->fd;
-      pfds[pfd_count].revents = 0;
-      pfd_count++;
-    }
-  }
-  for (j = 0; j < h->del_count; j++) {
-    GRPC_FD_UNREF(h->dels[j], "multipoller_del");
-  }
-  h->del_count = 0;
-  h->fd_count = fd_count;
-  gpr_mu_unlock(&pollset->mu);
-
-  for (i = 2; i < pfd_count; i++) {
-    grpc_fd *fd = watchers[i].fd;
-    pfds[i].events = (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT,
-                                          &watchers[i]);
-    GRPC_FD_UNREF(fd, "multipoller_start");
-  }
-
-  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
-     even going into the blocking annotation if possible */
-  GRPC_SCHEDULING_START_BLOCKING_REGION;
-  r = grpc_poll_function(pfds, pfd_count, timeout);
-  GRPC_SCHEDULING_END_BLOCKING_REGION;
-
-  if (r < 0) {
-    if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
-    }
-    for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
-    }
-  } else if (r == 0) {
-    for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
-    }
-  } else {
-    if (pfds[0].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
-    }
-    if (pfds[1].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
-    }
-    for (i = 2; i < pfd_count; i++) {
-      if (watchers[i].fd == NULL) {
-        fd_end_poll(exec_ctx, &watchers[i], 0, 0);
-        continue;
-      }
-      fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                  pfds[i].revents & POLLOUT_CHECK);
-    }
-  }
-
-  gpr_free(pfds);
-  gpr_free(watchers);
-}
-
-static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) {
-  size_t i;
-  poll_hdr *h = pollset->data.ptr;
-  for (i = 0; i < h->fd_count; i++) {
-    GRPC_FD_UNREF(h->fds[i], "multipoller");
-  }
-  for (i = 0; i < h->del_count; i++) {
-    GRPC_FD_UNREF(h->dels[i], "multipoller_del");
-  }
-  h->fd_count = 0;
-  h->del_count = 0;
-}
-
-static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
-  poll_hdr *h = pollset->data.ptr;
-  multipoll_with_poll_pollset_finish_shutdown(pollset);
-  gpr_free(h->fds);
-  gpr_free(h->dels);
-  gpr_free(h);
-}
-
-static const grpc_pollset_vtable multipoll_with_poll_pollset = {
-    multipoll_with_poll_pollset_add_fd,
-    multipoll_with_poll_pollset_maybe_work_and_unlock,
-    multipoll_with_poll_pollset_finish_shutdown,
-    multipoll_with_poll_pollset_destroy};
-
-static void poll_become_multipoller(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset *pollset, grpc_fd **fds,
-                                    size_t nfds) {
-  size_t i;
-  poll_hdr *h = gpr_malloc(sizeof(poll_hdr));
-  pollset->vtable = &multipoll_with_poll_pollset;
-  pollset->data.ptr = h;
-  h->fd_count = nfds;
-  h->fd_capacity = nfds;
-  h->fds = gpr_malloc(nfds * sizeof(grpc_fd *));
-  h->del_count = 0;
-  h->del_capacity = 0;
-  h->dels = NULL;
-  for (i = 0; i < nfds; i++) {
-    h->fds[i] = fds[i];
-    GRPC_FD_REF(fds[i], "multipoller");
-  }
-}
-
-#endif /* !GPR_LINUX_MULTIPOLL_WITH_EPOLL */
-
-/*******************************************************************************
- * pollset_multipoller_with_epoll_posix.c
- */
-
-#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
-
-#include <errno.h>
-#include <poll.h>
-#include <string.h>
-#include <sys/epoll.h>
-#include <unistd.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/useful.h>
-
-#include "src/core/lib/iomgr/ev_posix.h"
-#include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/block_annotate.h"
-
-static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st) {
-  /* only one set_ready can be active at once (but there may be a racing
-     notify_on) */
-  gpr_mu_lock(&fd->mu);
-  set_ready_locked(exec_ctx, fd, st);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->read_closure);
-}
-
-static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->write_closure);
-}
-
-struct epoll_fd_list {
-  int *epoll_fds;
-  size_t count;
-  size_t capacity;
-};
-
-static struct epoll_fd_list epoll_fd_global_list;
-static gpr_once init_epoll_fd_list_mu = GPR_ONCE_INIT;
-static gpr_mu epoll_fd_list_mu;
-
-static void init_mu(void) { gpr_mu_init(&epoll_fd_list_mu); }
-
-static void add_epoll_fd_to_global_list(int epoll_fd) {
-  gpr_once_init(&init_epoll_fd_list_mu, init_mu);
-
-  gpr_mu_lock(&epoll_fd_list_mu);
-  if (epoll_fd_global_list.count == epoll_fd_global_list.capacity) {
-    epoll_fd_global_list.capacity =
-        GPR_MAX((size_t)8, epoll_fd_global_list.capacity * 2);
-    epoll_fd_global_list.epoll_fds =
-        gpr_realloc(epoll_fd_global_list.epoll_fds,
-                    epoll_fd_global_list.capacity * sizeof(int));
-  }
-  epoll_fd_global_list.epoll_fds[epoll_fd_global_list.count++] = epoll_fd;
-  gpr_mu_unlock(&epoll_fd_list_mu);
-}
-
-static void remove_epoll_fd_from_global_list(int epoll_fd) {
-  gpr_mu_lock(&epoll_fd_list_mu);
-  GPR_ASSERT(epoll_fd_global_list.count > 0);
-  for (size_t i = 0; i < epoll_fd_global_list.count; i++) {
-    if (epoll_fd == epoll_fd_global_list.epoll_fds[i]) {
-      epoll_fd_global_list.epoll_fds[i] =
-          epoll_fd_global_list.epoll_fds[--(epoll_fd_global_list.count)];
-      break;
-    }
-  }
-  gpr_mu_unlock(&epoll_fd_list_mu);
-}
-
-static void remove_fd_from_all_epoll_sets(int fd) {
-  int err;
-  gpr_once_init(&init_epoll_fd_list_mu, init_mu);
-  gpr_mu_lock(&epoll_fd_list_mu);
-  if (epoll_fd_global_list.count == 0) {
-    gpr_mu_unlock(&epoll_fd_list_mu);
-    return;
-  }
-  for (size_t i = 0; i < epoll_fd_global_list.count; i++) {
-    err = epoll_ctl(epoll_fd_global_list.epoll_fds[i], EPOLL_CTL_DEL, fd, NULL);
-    if (err < 0 && errno != ENOENT) {
-      gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd,
-              strerror(errno));
-    }
-  }
-  gpr_mu_unlock(&epoll_fd_list_mu);
-}
-
-typedef struct {
-  grpc_pollset *pollset;
-  grpc_fd *fd;
-  grpc_closure closure;
-} delayed_add;
-
-typedef struct { int epoll_fd; } epoll_hdr;
-
-static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           grpc_fd *fd) {
-  epoll_hdr *h = pollset->data.ptr;
-  struct epoll_event ev;
-  int err;
-  grpc_fd_watcher watcher;
-
-  /* We pretend to be polling whilst adding an fd to keep the fd from being
-     closed during the add. This may result in a spurious wakeup being assigned
-     to this pollset whilst adding, but that should be benign. */
-  GPR_ASSERT(fd_begin_poll(fd, pollset, NULL, 0, 0, &watcher) == 0);
-  if (watcher.fd != NULL) {
-    ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET);
-    ev.data.ptr = fd;
-    err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev);
-    if (err < 0) {
-      /* FDs may be added to a pollset multiple times, so EEXIST is normal. */
-      if (errno != EEXIST) {
-        gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd,
-                strerror(errno));
-      }
-    }
-  }
-  fd_end_poll(exec_ctx, &watcher, 0, 0);
-}
-
-static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg,
-                                bool iomgr_status) {
-  delayed_add *da = arg;
-
-  if (!fd_is_orphaned(da->fd)) {
-    finally_add_fd(exec_ctx, da->pollset, da->fd);
-  }
-
-  gpr_mu_lock(&da->pollset->mu);
-  da->pollset->in_flight_cbs--;
-  if (da->pollset->shutting_down) {
-    /* We don't care about this pollset anymore. */
-    if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) {
-      da->pollset->called_shutdown = 1;
-      grpc_exec_ctx_enqueue(exec_ctx, da->pollset->shutdown_done, true, NULL);
-    }
-  }
-  gpr_mu_unlock(&da->pollset->mu);
-
-  GRPC_FD_UNREF(da->fd, "delayed_add");
-
-  gpr_free(da);
-}
-
-static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
-                                                grpc_pollset *pollset,
-                                                grpc_fd *fd,
-                                                int and_unlock_pollset) {
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-    finally_add_fd(exec_ctx, pollset, fd);
-  } else {
-    delayed_add *da = gpr_malloc(sizeof(*da));
-    da->pollset = pollset;
-    da->fd = fd;
-    GRPC_FD_REF(fd, "delayed_add");
-    grpc_closure_init(&da->closure, perform_delayed_add, da);
-    pollset->in_flight_cbs++;
-    grpc_exec_ctx_enqueue(exec_ctx, &da->closure, true, NULL);
-  }
-}
-
-/* TODO(klempner): We probably want to turn this down a bit */
-#define GRPC_EPOLL_MAX_EVENTS 1000
-
-static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
-    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
-    gpr_timespec deadline, gpr_timespec now) {
-  struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
-  int ep_rv;
-  int poll_rv;
-  epoll_hdr *h = pollset->data.ptr;
-  int timeout_ms;
-  struct pollfd pfds[2];
-
-  /* If you want to ignore epoll's ability to sanely handle parallel pollers,
-   * for a more apples-to-apples performance comparison with poll, add a
-   * if (pollset->counter != 0) { return 0; }
-   * here.
-   */
-
-  gpr_mu_unlock(&pollset->mu);
-
-  timeout_ms = poll_deadline_to_millis_timeout(deadline, now);
-
-  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
-  pfds[0].events = POLLIN;
-  pfds[0].revents = 0;
-  pfds[1].fd = h->epoll_fd;
-  pfds[1].events = POLLIN;
-  pfds[1].revents = 0;
-
-  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
-     even going into the blocking annotation if possible */
-  GPR_TIMER_BEGIN("poll", 0);
-  GRPC_SCHEDULING_START_BLOCKING_REGION;
-  poll_rv = grpc_poll_function(pfds, 2, timeout_ms);
-  GRPC_SCHEDULING_END_BLOCKING_REGION;
-  GPR_TIMER_END("poll", 0);
-
-  if (poll_rv < 0) {
-    if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
-    }
-  } else if (poll_rv == 0) {
-    /* do nothing */
-  } else {
-    if (pfds[0].revents) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
-    }
-    if (pfds[1].revents) {
-      do {
-        /* The following epoll_wait never blocks; it has a timeout of 0 */
-        ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
-        if (ep_rv < 0) {
-          if (errno != EINTR) {
-            gpr_log(GPR_ERROR, "epoll_wait() failed: %s", strerror(errno));
-          }
-        } else {
-          int i;
-          for (i = 0; i < ep_rv; ++i) {
-            grpc_fd *fd = ep_ev[i].data.ptr;
-            /* TODO(klempner): We might want to consider making err and pri
-             * separate events */
-            int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
-            int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
-            int write_ev = ep_ev[i].events & EPOLLOUT;
-            if (fd == NULL) {
-              grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
-            } else {
-              if (read_ev || cancel) {
-                fd_become_readable(exec_ctx, fd);
-              }
-              if (write_ev || cancel) {
-                fd_become_writable(exec_ctx, fd);
-              }
-            }
-          }
-        }
-      } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
-    }
-  }
-}
-
-static void multipoll_with_epoll_pollset_finish_shutdown(
-    grpc_pollset *pollset) {}
-
-static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) {
-  epoll_hdr *h = pollset->data.ptr;
-  close(h->epoll_fd);
-  remove_epoll_fd_from_global_list(h->epoll_fd);
-  gpr_free(h);
-}
-
-static const grpc_pollset_vtable multipoll_with_epoll_pollset = {
-    multipoll_with_epoll_pollset_add_fd,
-    multipoll_with_epoll_pollset_maybe_work_and_unlock,
-    multipoll_with_epoll_pollset_finish_shutdown,
-    multipoll_with_epoll_pollset_destroy};
-
-static void epoll_become_multipoller(grpc_exec_ctx *exec_ctx,
-                                     grpc_pollset *pollset, grpc_fd **fds,
-                                     size_t nfds) {
-  size_t i;
-  epoll_hdr *h = gpr_malloc(sizeof(epoll_hdr));
-  struct epoll_event ev;
-  int err;
-
-  pollset->vtable = &multipoll_with_epoll_pollset;
-  pollset->data.ptr = h;
-  h->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
-  if (h->epoll_fd < 0) {
-    /* TODO(klempner): Fall back to poll here, especially on ENOSYS */
-    gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno));
-    abort();
-  }
-  add_epoll_fd_to_global_list(h->epoll_fd);
-
-  ev.events = (uint32_t)(EPOLLIN | EPOLLET);
-  ev.data.ptr = NULL;
-  err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD,
-                  GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), &ev);
-  if (err < 0) {
-    gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s",
-            GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd),
-            strerror(errno));
-  }
-
-  for (i = 0; i < nfds; i++) {
-    multipoll_with_epoll_pollset_add_fd(exec_ctx, pollset, fds[i], 0);
-  }
-}
-
-#else /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
-
-static void remove_fd_from_all_epoll_sets(int fd) {}
-
-#endif /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
-
-/*******************************************************************************
- * pollset_set_posix.c
- */
-
-static grpc_pollset_set *pollset_set_create(void) {
-  grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set));
-  memset(pollset_set, 0, sizeof(*pollset_set));
-  gpr_mu_init(&pollset_set->mu);
-  return pollset_set;
-}
-
-static void pollset_set_destroy(grpc_pollset_set *pollset_set) {
-  size_t i;
-  gpr_mu_destroy(&pollset_set->mu);
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
-  }
-  gpr_free(pollset_set->pollsets);
-  gpr_free(pollset_set->pollset_sets);
-  gpr_free(pollset_set->fds);
-  gpr_free(pollset_set);
-}
-
-static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset_set *pollset_set,
-                                    grpc_pollset *pollset) {
-  size_t i, j;
-  gpr_mu_lock(&pollset_set->mu);
-  if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
-    pollset_set->pollset_capacity =
-        GPR_MAX(8, 2 * pollset_set->pollset_capacity);
-    pollset_set->pollsets =
-        gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity *
-                                               sizeof(*pollset_set->pollsets));
-  }
-  pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
-  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
-    if (fd_is_orphaned(pollset_set->fds[i])) {
-      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
-    } else {
-      pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
-      pollset_set->fds[j++] = pollset_set->fds[i];
-    }
-  }
-  pollset_set->fd_count = j;
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset_set *pollset_set,
-                                    grpc_pollset *pollset) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  for (i = 0; i < pollset_set->pollset_count; i++) {
-    if (pollset_set->pollsets[i] == pollset) {
-      pollset_set->pollset_count--;
-      GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i],
-               pollset_set->pollsets[pollset_set->pollset_count]);
-      break;
-    }
-  }
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
-                                        grpc_pollset_set *bag,
-                                        grpc_pollset_set *item) {
-  size_t i, j;
-  gpr_mu_lock(&bag->mu);
-  if (bag->pollset_set_count == bag->pollset_set_capacity) {
-    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
-    bag->pollset_sets =
-        gpr_realloc(bag->pollset_sets,
-                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
-  }
-  bag->pollset_sets[bag->pollset_set_count++] = item;
-  for (i = 0, j = 0; i < bag->fd_count; i++) {
-    if (fd_is_orphaned(bag->fds[i])) {
-      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
-    } else {
-      pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
-      bag->fds[j++] = bag->fds[i];
-    }
-  }
-  bag->fd_count = j;
-  gpr_mu_unlock(&bag->mu);
-}
-
-static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
-                                        grpc_pollset_set *bag,
-                                        grpc_pollset_set *item) {
-  size_t i;
-  gpr_mu_lock(&bag->mu);
-  for (i = 0; i < bag->pollset_set_count; i++) {
-    if (bag->pollset_sets[i] == item) {
-      bag->pollset_set_count--;
-      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
-               bag->pollset_sets[bag->pollset_set_count]);
-      break;
-    }
-  }
-  gpr_mu_unlock(&bag->mu);
-}
-
-static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  if (pollset_set->fd_count == pollset_set->fd_capacity) {
-    pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity);
-    pollset_set->fds = gpr_realloc(
-        pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds));
-  }
-  GRPC_FD_REF(fd, "pollset_set");
-  pollset_set->fds[pollset_set->fd_count++] = fd;
-  for (i = 0; i < pollset_set->pollset_count; i++) {
-    pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
-  }
-  for (i = 0; i < pollset_set->pollset_set_count; i++) {
-    pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
-  }
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    if (pollset_set->fds[i] == fd) {
-      pollset_set->fd_count--;
-      GPR_SWAP(grpc_fd *, pollset_set->fds[i],
-               pollset_set->fds[pollset_set->fd_count]);
-      GRPC_FD_UNREF(fd, "pollset_set");
-      break;
-    }
-  }
-  for (i = 0; i < pollset_set->pollset_set_count; i++) {
-    pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
-  }
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-/*******************************************************************************
- * event engine binding
- */
-
-static void shutdown_engine(void) {
-  fd_global_shutdown();
-  pollset_global_shutdown();
-}
-
-static const grpc_event_engine_vtable vtable = {
-    .pollset_size = sizeof(grpc_pollset),
-
-    .fd_create = fd_create,
-    .fd_wrapped_fd = fd_wrapped_fd,
-    .fd_orphan = fd_orphan,
-    .fd_shutdown = fd_shutdown,
-    .fd_notify_on_read = fd_notify_on_read,
-    .fd_notify_on_write = fd_notify_on_write,
-
-    .pollset_init = pollset_init,
-    .pollset_shutdown = pollset_shutdown,
-    .pollset_reset = pollset_reset,
-    .pollset_destroy = pollset_destroy,
-    .pollset_work = pollset_work,
-    .pollset_kick = pollset_kick,
-    .pollset_add_fd = pollset_add_fd,
-
-    .pollset_set_create = pollset_set_create,
-    .pollset_set_destroy = pollset_set_destroy,
-    .pollset_set_add_pollset = pollset_set_add_pollset,
-    .pollset_set_del_pollset = pollset_set_del_pollset,
-    .pollset_set_add_pollset_set = pollset_set_add_pollset_set,
-    .pollset_set_del_pollset_set = pollset_set_del_pollset_set,
-    .pollset_set_add_fd = pollset_set_add_fd,
-    .pollset_set_del_fd = pollset_set_del_fd,
-
-    .kick_poller = kick_poller,
-
-    .shutdown_engine = shutdown_engine,
-};
-
-const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void) {
-#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
-  platform_become_multipoller = epoll_become_multipoller;
-#else
-  platform_become_multipoller = poll_become_multipoller;
-#endif
-  fd_global_init();
-  pollset_global_init();
-  return &vtable;
-}
-
-#endif
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index e91ae40212..e2a21230b9 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -59,6 +59,8 @@
  * FD declarations
  */
 
+grpc_wakeup_fd grpc_global_wakeup_fd;
+
 typedef struct grpc_fd_watcher {
   struct grpc_fd_watcher *next;
   struct grpc_fd_watcher *prev;
@@ -113,6 +115,9 @@ struct grpc_fd {
   grpc_closure *on_done_closure;
 
   grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
 };
 
 /* Begin polling on an fd.
@@ -134,7 +139,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
    if got_read or got_write are 1, also does the become_{readable,writable} as
    appropriate. */
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
 
 /* Return 1 if this fd is orphaned, 0 otherwise */
 static bool fd_is_orphaned(grpc_fd *fd);
@@ -177,7 +183,6 @@ struct grpc_pollset_worker {
 struct grpc_pollset {
   gpr_mu mu;
   grpc_pollset_worker root_worker;
-  int in_flight_cbs;
   int shutting_down;
   int called_shutdown;
   int kicked_without_pollers;
@@ -187,10 +192,6 @@ struct grpc_pollset {
   size_t fd_count;
   size_t fd_capacity;
   grpc_fd **fds;
-  /* fds that have been removed from the pollset explicitly */
-  size_t del_count;
-  size_t del_capacity;
-  grpc_fd **dels;
   /* Local cache of eventfds for workers */
   grpc_cached_wakeup_fd *local_wakeup_cache;
 };
@@ -301,6 +302,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
   r->on_done_closure = NULL;
   r->closed = 0;
   r->released = 0;
+  r->read_notifier_pollset = NULL;
 
   char *name2;
   gpr_asprintf(&name2, "%s fd=%d", name, fd);
@@ -316,6 +318,18 @@ static bool fd_is_orphaned(grpc_fd *fd) {
   return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
 }
 
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
 static void pollset_kick_locked(grpc_fd_watcher *watcher) {
   gpr_mu_lock(&watcher->pollset->mu);
   GPR_ASSERT(watcher->worker);
@@ -444,6 +458,11 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   }
 }
 
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_lock(&fd->mu);
   GPR_ASSERT(!fd->shutdown);
@@ -519,7 +538,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
 }
 
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
   int was_polling = 0;
   int kick = 0;
   grpc_fd *fd = watcher->fd;
@@ -555,6 +575,9 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
     if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
       kick = 1;
     }
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
   }
   if (got_write) {
     if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
@@ -700,7 +723,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->mu);
   *mu = &pollset->mu;
   pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
-  pollset->in_flight_cbs = 0;
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
@@ -709,14 +731,10 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   pollset->kicked_without_pollers = 0;
   pollset->fd_count = 0;
   pollset->fd_capacity = 0;
-  pollset->del_count = 0;
-  pollset->del_capacity = 0;
   pollset->fds = NULL;
-  pollset->dels = NULL;
 }
 
 static void pollset_destroy(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
   GPR_ASSERT(!pollset_has_workers(pollset));
   GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   while (pollset->local_wakeup_cache) {
@@ -726,17 +744,14 @@ static void pollset_destroy(grpc_pollset *pollset) {
     pollset->local_wakeup_cache = next;
   }
   gpr_free(pollset->fds);
-  gpr_free(pollset->dels);
   gpr_mu_destroy(&pollset->mu);
 }
 
 static void pollset_reset(grpc_pollset *pollset) {
   GPR_ASSERT(pollset->shutting_down);
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
   GPR_ASSERT(!pollset_has_workers(pollset));
   GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   GPR_ASSERT(pollset->fd_count == 0);
-  GPR_ASSERT(pollset->del_count == 0);
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
@@ -769,11 +784,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
   for (i = 0; i < pollset->fd_count; i++) {
     GRPC_FD_UNREF(pollset->fds[i], "multipoller");
   }
-  for (i = 0; i < pollset->del_count; i++) {
-    GRPC_FD_UNREF(pollset->dels[i], "multipoller_del");
-  }
   pollset->fd_count = 0;
-  pollset->del_count = 0;
   grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL);
 }
 
@@ -813,13 +824,6 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     GPR_TIMER_MARK("pollset_work.shutting_down", 0);
     goto done;
   }
-  /* Give do_promote priority so we don't starve it out */
-  if (pollset->in_flight_cbs) {
-    GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0);
-    gpr_mu_unlock(&pollset->mu);
-    locked = 0;
-    goto done;
-  }
   /* Start polling, and keep doing so while we're being asked to
      re-evaluate our pollers (this allows poll() based pollers to
      ensure they don't miss wakeups) */
@@ -839,7 +843,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
       int timeout;
       int r;
-      size_t i, j, fd_count;
+      size_t i, fd_count;
       nfds_t pfd_count;
       /* TODO(ctiller): inline some elements to avoid an allocation */
       grpc_fd_watcher *watchers;
@@ -859,11 +863,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       pfds[1].events = POLLIN;
       pfds[1].revents = 0;
       for (i = 0; i < pollset->fd_count; i++) {
-        int remove = fd_is_orphaned(pollset->fds[i]);
-        for (j = 0; !remove && j < pollset->del_count; j++) {
-          if (pollset->fds[i] == pollset->dels[j]) remove = 1;
-        }
-        if (remove) {
+        if (fd_is_orphaned(pollset->fds[i])) {
           GRPC_FD_UNREF(pollset->fds[i], "multipoller");
         } else {
           pollset->fds[fd_count++] = pollset->fds[i];
@@ -874,10 +874,6 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           pfd_count++;
         }
       }
-      for (j = 0; j < pollset->del_count; j++) {
-        GRPC_FD_UNREF(pollset->dels[j], "multipoller_del");
-      }
-      pollset->del_count = 0;
       pollset->fd_count = fd_count;
       gpr_mu_unlock(&pollset->mu);
 
@@ -899,11 +895,11 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
         }
         for (i = 2; i < pfd_count; i++) {
-          fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else if (r == 0) {
         for (i = 2; i < pfd_count; i++) {
-          fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else {
         if (pfds[0].revents & POLLIN_CHECK) {
@@ -914,10 +910,10 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
         }
         for (i = 2; i < pfd_count; i++) {
           if (watchers[i].fd == NULL) {
-            fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+            fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
           } else {
             fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                        pfds[i].revents & POLLOUT_CHECK);
+                        pfds[i].revents & POLLOUT_CHECK, pollset);
           }
         }
       }
@@ -969,7 +965,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (pollset->shutting_down) {
     if (pollset_has_workers(pollset)) {
       pollset_kick(pollset, NULL);
-    } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
+    } else if (!pollset->called_shutdown) {
       pollset->called_shutdown = 1;
       gpr_mu_unlock(&pollset->mu);
       finish_shutdown(exec_ctx, pollset);
@@ -999,8 +995,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (!pollset_has_workers(pollset)) {
     grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
   }
-  if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
-      !pollset_has_workers(pollset)) {
+  if (!pollset->called_shutdown && !pollset_has_workers(pollset)) {
     pollset->called_shutdown = 1;
     finish_shutdown(exec_ctx, pollset);
   }
@@ -1181,6 +1176,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_shutdown = fd_shutdown,
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index a7dfc9552d..95520b01d3 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -44,7 +44,6 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
 #include "src/core/lib/iomgr/ev_poll_posix.h"
 #include "src/core/lib/support/env.h"
 
@@ -62,7 +61,7 @@ typedef struct {
 } event_engine_factory;
 
 static const event_engine_factory g_factories[] = {
-    {"poll", grpc_init_poll_posix}, {"legacy", grpc_init_poll_and_epoll_posix},
+    {"poll", grpc_init_poll_posix},
 };
 
 static void add(const char *beg, const char *end, char ***ss, size_t *ns) {
@@ -163,6 +162,11 @@ void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   g_event_engine->fd_notify_on_write(exec_ctx, fd, closure);
 }
 
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd) {
+  return g_event_engine->fd_get_read_notifier_pollset(exec_ctx, fd);
+}
+
 size_t grpc_pollset_size(void) { return g_event_engine->pollset_size; }
 
 void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 1fa9f5ef2d..344bf63438 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -55,6 +55,8 @@ typedef struct grpc_event_engine_vtable {
                             grpc_closure *closure);
   void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
+  grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
 
   void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu);
   void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -137,6 +139,10 @@ void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
 
+/* Return the read notifier pollset from the fd */
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
+
 /* pollset_posix functions */
 
 /* Add an fd to a pollset */
diff --git a/src/core/lib/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h
index 99b9f29729..fee14ae661 100644
--- a/src/core/lib/iomgr/tcp_server.h
+++ b/src/core/lib/iomgr/tcp_server.h
@@ -52,6 +52,7 @@ typedef struct grpc_tcp_server_acceptor {
 /* Called for newly connected TCP connections. */
 typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_endpoint *ep,
+                                   grpc_pollset *accepting_pollset,
                                    grpc_tcp_server_acceptor *acceptor);
 
 /* Create a server, initially not bound to any ports. The caller owns one ref.
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index aaeb384f6e..909e34abc7 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -128,6 +128,9 @@ struct grpc_tcp_server {
   grpc_pollset **pollsets;
   /* number of pollsets in the pollsets array */
   size_t pollset_count;
+
+  /* next pollset to assign a channel to */
+  size_t next_pollset_to_assign;
 };
 
 grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
@@ -145,6 +148,7 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
   s->head = NULL;
   s->tail = NULL;
   s->nports = 0;
+  s->next_pollset_to_assign = 0;
   return s;
 }
 
@@ -310,13 +314,17 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_tcp_listener *sp = arg;
   grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index,
                                        sp->fd_index};
+  grpc_pollset *read_notifier_pollset = NULL;
   grpc_fd *fdobj;
-  size_t i;
 
   if (!success) {
     goto error;
   }
 
+  read_notifier_pollset =
+      sp->server->pollsets[(sp->server->next_pollset_to_assign++) %
+                           sp->server->pollset_count];
+
   /* loop until accept4 returns EAGAIN, and then re-arm notification */
   for (;;) {
     struct sockaddr_storage addr;
@@ -349,16 +357,18 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     }
 
     fdobj = grpc_fd_create(fd, name);
-    /* TODO(ctiller): revise this when we have server-side sharding
-       of channels -- we certainly should not be automatically adding every
-       incoming channel to every pollset owned by the server */
-    for (i = 0; i < sp->server->pollset_count; i++) {
-      grpc_pollset_add_fd(exec_ctx, sp->server->pollsets[i], fdobj);
+
+    if (read_notifier_pollset == NULL) {
+      gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd");
+      goto error;
     }
+
+    grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj);
+
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
         grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
-        &acceptor);
+        read_notifier_pollset, &acceptor);
 
     gpr_free(name);
     gpr_free(addr_str);
diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 125f521d87..e15f8b0cdf 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -379,9 +379,10 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, bool from_iocp) {
 
   /* The only time we should call our callback, is where we successfully
      managed to accept a connection, and created an endpoint. */
-  if (ep)
-    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep,
+  if (ep) {
+    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
                              &acceptor);
+  }
   /* As we were notified from the IOCP of one and exactly one accept,
      the former socked we created has now either been destroy or assigned
      to the new connection. We need to create a new one for the next
diff --git a/src/core/lib/security/security_context.c b/src/core/lib/security/context/security_context.c
index 343e0b5b8b..127b13ee50 100644
--- a/src/core/lib/security/security_context.c
+++ b/src/core/lib/security/context/security_context.c
@@ -33,7 +33,7 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
diff --git a/src/core/lib/security/security_context.h b/src/core/lib/security/context/security_context.h
index 81161ec47d..ef0c06b1fb 100644
--- a/src/core/lib/security/security_context.h
+++ b/src/core/lib/security/context/security_context.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
+#ifndef GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
+#define GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
 
 #include "src/core/lib/iomgr/pollset.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 /* --- grpc_auth_context ---
 
@@ -111,4 +111,4 @@ grpc_auth_context *grpc_auth_context_from_arg(const grpc_arg *arg);
 grpc_auth_context *grpc_find_auth_context_in_args(
     const grpc_channel_args *args);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H */
diff --git a/src/core/lib/security/credentials.c b/src/core/lib/security/credentials.c
deleted file mode 100644
index 43c061a265..0000000000
--- a/src/core/lib/security/credentials.c
+++ /dev/null
@@ -1,1296 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/lib/security/credentials.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/http_client_filter.h"
-#include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/http/parser.h"
-#include "src/core/lib/iomgr/executor.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/surface/api_trace.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/time.h>
-
-/* -- Common. -- */
-
-struct grpc_credentials_metadata_request {
-  grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
-  void *user_data;
-};
-
-static grpc_credentials_metadata_request *
-grpc_credentials_metadata_request_create(grpc_call_credentials *creds,
-                                         grpc_credentials_metadata_cb cb,
-                                         void *user_data) {
-  grpc_credentials_metadata_request *r =
-      gpr_malloc(sizeof(grpc_credentials_metadata_request));
-  r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
-  return r;
-}
-
-static void grpc_credentials_metadata_request_destroy(
-    grpc_credentials_metadata_request *r) {
-  grpc_call_credentials_unref(r->creds);
-  gpr_free(r);
-}
-
-grpc_channel_credentials *grpc_channel_credentials_ref(
-    grpc_channel_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
-  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
-  grpc_channel_credentials_unref(creds);
-}
-
-grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_call_credentials_unref(grpc_call_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_call_credentials_release(grpc_call_credentials *creds) {
-  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
-  grpc_call_credentials_unref(creds);
-}
-
-void grpc_call_credentials_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-    }
-    return;
-  }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pops, context, cb,
-                                      user_data);
-}
-
-grpc_security_status grpc_channel_credentials_create_security_connector(
-    grpc_channel_credentials *channel_creds, const char *target,
-    const grpc_channel_args *args, grpc_channel_security_connector **sc,
-    grpc_channel_args **new_args) {
-  *new_args = NULL;
-  if (channel_creds == NULL) {
-    return GRPC_SECURITY_ERROR;
-  }
-  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
-  return channel_creds->vtable->create_security_connector(
-      channel_creds, NULL, target, args, sc, new_args);
-}
-
-grpc_server_credentials *grpc_server_credentials_ref(
-    grpc_server_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_server_credentials_unref(grpc_server_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-      creds->processor.destroy(creds->processor.state);
-    }
-    gpr_free(creds);
-  }
-}
-
-void grpc_server_credentials_release(grpc_server_credentials *creds) {
-  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
-  grpc_server_credentials_unref(creds);
-}
-
-grpc_security_status grpc_server_credentials_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
-    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
-    return GRPC_SECURITY_ERROR;
-  }
-  return creds->vtable->create_security_connector(creds, sc);
-}
-
-void grpc_server_credentials_set_auth_metadata_processor(
-    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
-  GRPC_API_TRACE(
-      "grpc_server_credentials_set_auth_metadata_processor("
-      "creds=%p, "
-      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
-      3, (creds, (void *)(intptr_t)processor.process, processor.state));
-  if (creds == NULL) return;
-  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-    creds->processor.destroy(creds->processor.state);
-  }
-  creds->processor = processor;
-}
-
-static void server_credentials_pointer_arg_destroy(void *p) {
-  grpc_server_credentials_unref(p);
-}
-
-static void *server_credentials_pointer_arg_copy(void *p) {
-  return grpc_server_credentials_ref(p);
-}
-
-static int server_credentials_pointer_cmp(void *a, void *b) {
-  return GPR_ICMP(a, b);
-}
-
-static const grpc_arg_pointer_vtable cred_ptr_vtable = {
-    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
-    server_credentials_pointer_cmp};
-
-grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
-  grpc_arg arg;
-  memset(&arg, 0, sizeof(grpc_arg));
-  arg.type = GRPC_ARG_POINTER;
-  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
-  arg.value.pointer.p = p;
-  arg.value.pointer.vtable = &cred_ptr_vtable;
-  return arg;
-}
-
-grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
-  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
-  if (arg->type != GRPC_ARG_POINTER) {
-    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
-            GRPC_SERVER_CREDENTIALS_ARG);
-    return NULL;
-  }
-  return arg->value.pointer.p;
-}
-
-grpc_server_credentials *grpc_find_server_credentials_in_args(
-    const grpc_channel_args *args) {
-  size_t i;
-  if (args == NULL) return NULL;
-  for (i = 0; i < args->num_args; i++) {
-    grpc_server_credentials *p =
-        grpc_server_credentials_from_arg(&args->args[i]);
-    if (p != NULL) return p;
-  }
-  return NULL;
-}
-
-/* -- Ssl credentials. -- */
-
-static void ssl_destruct(grpc_channel_credentials *creds) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
-  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
-}
-
-static void ssl_server_destruct(grpc_server_credentials *creds) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
-    if (c->config.pem_private_keys[i] != NULL) {
-      gpr_free(c->config.pem_private_keys[i]);
-    }
-    if (c->config.pem_cert_chains[i] != NULL) {
-      gpr_free(c->config.pem_cert_chains[i]);
-    }
-  }
-  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
-  if (c->config.pem_private_keys_sizes != NULL) {
-    gpr_free(c->config.pem_private_keys_sizes);
-  }
-  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
-  if (c->config.pem_cert_chains_sizes != NULL) {
-    gpr_free(c->config.pem_cert_chains_sizes);
-  }
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-}
-
-static grpc_security_status ssl_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_OK;
-  size_t i = 0;
-  const char *overridden_target_name = NULL;
-  grpc_arg new_arg;
-
-  for (i = 0; args && i < args->num_args; i++) {
-    grpc_arg *arg = &args->args[i];
-    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
-        arg->type == GRPC_ARG_STRING) {
-      overridden_target_name = arg->value.string;
-      break;
-    }
-  }
-  status = grpc_ssl_channel_security_connector_create(
-      call_creds, &c->config, target, overridden_target_name, sc);
-  if (status != GRPC_SECURITY_OK) {
-    return status;
-  }
-  new_arg.type = GRPC_ARG_STRING;
-  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
-  new_arg.value.string = "https";
-  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
-  return status;
-}
-
-static grpc_security_status ssl_server_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  return grpc_ssl_server_security_connector_create(&c->config, sc);
-}
-
-static grpc_channel_credentials_vtable ssl_vtable = {
-    ssl_destruct, ssl_create_security_connector};
-
-static grpc_server_credentials_vtable ssl_server_vtable = {
-    ssl_server_destruct, ssl_server_create_security_connector};
-
-static void ssl_copy_key_material(const char *input, unsigned char **output,
-                                  size_t *output_size) {
-  *output_size = strlen(input);
-  *output = gpr_malloc(*output_size);
-  memcpy(*output, input, *output_size);
-}
-
-static void ssl_build_config(const char *pem_root_certs,
-                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-                             grpc_ssl_config *config) {
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (pem_key_cert_pair != NULL) {
-    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pair->private_key,
-                          &config->pem_private_key,
-                          &config->pem_private_key_size);
-    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
-                          &config->pem_cert_chain,
-                          &config->pem_cert_chain_size);
-  }
-}
-
-static void ssl_build_server_config(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    grpc_ssl_server_config *config) {
-  size_t i;
-  config->client_certificate_request = client_certificate_request;
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (num_key_cert_pairs > 0) {
-    GPR_ASSERT(pem_key_cert_pairs != NULL);
-    config->pem_private_keys =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_cert_chains =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_private_keys_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-    config->pem_cert_chains_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-  }
-  config->num_key_cert_pairs = num_key_cert_pairs;
-  for (i = 0; i < num_key_cert_pairs; i++) {
-    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
-                          &config->pem_private_keys[i],
-                          &config->pem_private_keys_sizes[i]);
-    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
-                          &config->pem_cert_chains[i],
-                          &config->pem_cert_chains_sizes[i]);
-  }
-}
-
-grpc_channel_credentials *grpc_ssl_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-    void *reserved) {
-  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_credentials_create(pem_root_certs=%s, "
-      "pem_key_cert_pair=%p, "
-      "reserved=%p)",
-      3, (pem_root_certs, pem_key_cert_pair, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  c->base.vtable = &ssl_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
-  return &c->base;
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
-  return grpc_ssl_server_credentials_create_ex(
-      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
-      force_client_auth
-          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
-          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
-      reserved);
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    void *reserved) {
-  grpc_ssl_server_credentials *c =
-      gpr_malloc(sizeof(grpc_ssl_server_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_server_credentials_create_ex("
-      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
-      "client_certificate_request=%d, reserved=%p)",
-      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
-          client_certificate_request, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_server_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &ssl_server_vtable;
-  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
-                          num_key_cert_pairs, client_certificate_request,
-                          &c->config);
-  return &c->base;
-}
-
-/* -- Jwt credentials -- */
-
-static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
-  if (c->cached.service_url != NULL) {
-    gpr_free(c->cached.service_url);
-    c->cached.service_url = NULL;
-  }
-  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-}
-
-static void jwt_destruct(grpc_call_credentials *creds) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  grpc_auth_json_key_destruct(&c->key);
-  jwt_reset_cache(c);
-  gpr_mu_destroy(&c->cache_mu);
-}
-
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pops *pops,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-
-  /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
-  {
-    gpr_mu_lock(&c->cache_mu);
-    if (c->cached.service_url != NULL &&
-        strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
-        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
-                                   gpr_now(GPR_CLOCK_REALTIME)),
-                      refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md == NULL) {
-    char *jwt = NULL;
-    /* Generate a new jwt. */
-    gpr_mu_lock(&c->cache_mu);
-    jwt_reset_cache(c);
-    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
-                                   c->jwt_lifetime, NULL);
-    if (jwt != NULL) {
-      char *md_value;
-      gpr_asprintf(&md_value, "Bearer %s", jwt);
-      gpr_free(jwt);
-      c->cached.jwt_expiration =
-          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
-      c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
-      gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(jwt_md);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  }
-}
-
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
-
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime) {
-  grpc_service_account_jwt_access_credentials *c;
-  if (!grpc_auth_json_key_is_valid(&key)) {
-    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
-  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &jwt_vtable;
-  c->key = key;
-  c->jwt_lifetime = token_lifetime;
-  gpr_mu_init(&c->cache_mu);
-  jwt_reset_cache(c);
-  return &c->base;
-}
-
-grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
-    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_service_account_jwt_access_credentials_create("
-      "json_key=%s, "
-      "token_lifetime="
-      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
-      "reserved=%p)",
-      5,
-      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
-       (int)token_lifetime.clock_type, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
-}
-
-/* -- Oauth2TokenFetcher credentials -- */
-
-static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-  gpr_mu_destroy(&c->mu);
-  grpc_httpcli_context_destroy(&c->httpcli_context);
-}
-
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const grpc_http_response *response, grpc_credentials_md_store **token_md,
-    gpr_timespec *token_lifetime) {
-  char *null_terminated_body = NULL;
-  char *new_access_token = NULL;
-  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
-  grpc_json *json = NULL;
-
-  if (response == NULL) {
-    gpr_log(GPR_ERROR, "Received NULL response.");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  }
-
-  if (response->body_length > 0) {
-    null_terminated_body = gpr_malloc(response->body_length + 1);
-    null_terminated_body[response->body_length] = '\0';
-    memcpy(null_terminated_body, response->body, response->body_length);
-  }
-
-  if (response->status != 200) {
-    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
-            response->status,
-            null_terminated_body != NULL ? null_terminated_body : "");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  } else {
-    grpc_json *access_token = NULL;
-    grpc_json *token_type = NULL;
-    grpc_json *expires_in = NULL;
-    grpc_json *ptr;
-    json = grpc_json_parse_string(null_terminated_body);
-    if (json == NULL) {
-      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (json->type != GRPC_JSON_OBJECT) {
-      gpr_log(GPR_ERROR, "Response should be a JSON object");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    for (ptr = json->child; ptr; ptr = ptr->next) {
-      if (strcmp(ptr->key, "access_token") == 0) {
-        access_token = ptr;
-      } else if (strcmp(ptr->key, "token_type") == 0) {
-        token_type = ptr;
-      } else if (strcmp(ptr->key, "expires_in") == 0) {
-        expires_in = ptr;
-      }
-    }
-    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
-      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
-                 access_token->value);
-    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
-    token_lifetime->tv_nsec = 0;
-    token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
-    status = GRPC_CREDENTIALS_OK;
-  }
-
-end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(*token_md);
-    *token_md = NULL;
-  }
-  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
-  if (new_access_token != NULL) gpr_free(new_access_token);
-  if (json != NULL) grpc_json_destroy(json);
-  return status;
-}
-
-static void on_oauth2_token_fetcher_http_response(
-    grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_http_response *response) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)r->creds;
-  gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, status);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status);
-  }
-  gpr_mu_unlock(&c->mu);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void oauth2_token_fetcher_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
-    gpr_mu_unlock(&c->mu);
-  }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(cached_access_token_md);
-  } else {
-    c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pops, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
-  }
-}
-
-static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
-                                      grpc_fetch_oauth2_func fetch_func) {
-  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  gpr_ref_init(&c->base.refcount, 1);
-  gpr_mu_init(&c->mu);
-  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-  c->fetch_func = fetch_func;
-  grpc_httpcli_context_init(&c->httpcli_context);
-}
-
-/* -- GoogleComputeEngine credentials. -- */
-
-static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void compute_engine_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pops *pops,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_http_header header = {"Metadata-Flavor", "Google"};
-  grpc_httpcli_request request;
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
-  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  grpc_httpcli_get(exec_ctx, httpcli_context, pops, &request, deadline,
-                   response_cb, metadata_req);
-}
-
-grpc_call_credentials *grpc_google_compute_engine_credentials_create(
-    void *reserved) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
-  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
-  c->base.vtable = &compute_engine_vtable;
-  return &c->base;
-}
-
-/* -- GoogleRefreshToken credentials. -- */
-
-static void refresh_token_destruct(grpc_call_credentials *creds) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)creds;
-  grpc_auth_refresh_token_destruct(&c->refresh_token);
-  oauth2_token_fetcher_destruct(&c->base.base);
-}
-
-static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void refresh_token_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pops *pops,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)metadata_req->creds;
-  grpc_http_header header = {"Content-Type",
-                             "application/x-www-form-urlencoded"};
-  grpc_httpcli_request request;
-  char *body = NULL;
-  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
-               c->refresh_token.client_id, c->refresh_token.client_secret,
-               c->refresh_token.refresh_token);
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
-  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  request.handshaker = &grpc_httpcli_ssl;
-  grpc_httpcli_post(exec_ctx, httpcli_context, pops, &request, body,
-                    strlen(body), deadline, response_cb, metadata_req);
-  gpr_free(body);
-}
-
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token refresh_token) {
-  grpc_google_refresh_token_credentials *c;
-  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
-    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
-  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
-  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
-  c->base.base.vtable = &refresh_token_vtable;
-  c->refresh_token = refresh_token;
-  return &c->base.base;
-}
-
-grpc_call_credentials *grpc_google_refresh_token_credentials_create(
-    const char *json_refresh_token, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
-      "reserved=%p)",
-      2, (json_refresh_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
-      grpc_auth_refresh_token_create_from_string(json_refresh_token));
-}
-
-/* -- Metadata-only credentials. -- */
-
-static void md_only_test_destruct(grpc_call_credentials *creds) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(c->md_store);
-}
-
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, bool success) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void md_only_test_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
-  if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    grpc_executor_enqueue(
-        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
-
-grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
-  grpc_md_only_test_credentials *c =
-      gpr_malloc(sizeof(grpc_md_only_test_credentials));
-  memset(c, 0, sizeof(grpc_md_only_test_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &md_only_test_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
-  c->is_async = is_async;
-  return &c->base;
-}
-
-/* -- Oauth2 Access Token credentials. -- */
-
-static void access_token_destruct(grpc_call_credentials *creds) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-}
-
-static void access_token_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
-
-grpc_call_credentials *grpc_access_token_credentials_create(
-    const char *access_token, void *reserved) {
-  grpc_access_token_credentials *c =
-      gpr_malloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
-  GRPC_API_TRACE(
-      "grpc_access_token_credentials_create(access_token=%s, "
-      "reserved=%p)",
-      2, (access_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_access_token_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &access_token_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
-  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
-  gpr_free(token_md_value);
-  return &c->base;
-}
-
-/* -- Fake transport security credentials. -- */
-
-static grpc_security_status fake_transport_security_create_security_connector(
-    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  *sc = grpc_fake_channel_security_connector_create(call_creds);
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_security_status
-fake_transport_security_server_create_security_connector(
-    grpc_server_credentials *c, grpc_server_security_connector **sc) {
-  *sc = grpc_fake_server_security_connector_create();
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_channel_credentials_vtable
-    fake_transport_security_credentials_vtable = {
-        NULL, fake_transport_security_create_security_connector};
-
-static grpc_server_credentials_vtable
-    fake_transport_security_server_credentials_vtable = {
-        NULL, fake_transport_security_server_create_security_connector};
-
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
-    void) {
-  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
-  memset(c, 0, sizeof(grpc_channel_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  c->vtable = &fake_transport_security_credentials_vtable;
-  gpr_ref_init(&c->refcount, 1);
-  return c;
-}
-
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void) {
-  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
-  memset(c, 0, sizeof(grpc_server_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  gpr_ref_init(&c->refcount, 1);
-  c->vtable = &fake_transport_security_server_credentials_vtable;
-  return c;
-}
-
-/* -- Composite call credentials. -- */
-
-typedef struct {
-  grpc_composite_call_credentials *composite_creds;
-  size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
-  grpc_pops *pops;
-  grpc_credentials_metadata_cb cb;
-} grpc_composite_call_credentials_metadata_context;
-
-static void composite_call_destruct(grpc_call_credentials *creds) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
-    grpc_call_credentials_unref(c->inner.creds_array[i]);
-  }
-  gpr_free(c->inner.creds_array);
-}
-
-static void composite_call_md_context_destroy(
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status) {
-  grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
-    }
-  }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(exec_ctx, inner_creds, ctx->pops,
-                                               ctx->auth_md_context,
-                                               composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
-  composite_call_md_context_destroy(ctx);
-}
-
-static void composite_call_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context auth_md_context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  grpc_composite_call_credentials_metadata_context *ctx;
-
-  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
-  ctx->composite_creds = c;
-  ctx->pops = pops;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], pops, auth_md_context,
-      composite_call_metadata_cb, ctx);
-}
-
-static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
-
-static grpc_call_credentials_array get_creds_array(
-    grpc_call_credentials **creds_addr) {
-  grpc_call_credentials_array result;
-  grpc_call_credentials *creds = *creds_addr;
-  result.creds_array = creds_addr;
-  result.num_creds = 1;
-  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    result = *grpc_composite_call_credentials_get_credentials(creds);
-  }
-  return result;
-}
-
-grpc_call_credentials *grpc_composite_call_credentials_create(
-    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
-    void *reserved) {
-  size_t i;
-  size_t creds_array_byte_size;
-  grpc_call_credentials_array creds1_array;
-  grpc_call_credentials_array creds2_array;
-  grpc_composite_call_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
-      "reserved=%p)",
-      3, (creds1, creds2, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(creds1 != NULL);
-  GPR_ASSERT(creds2 != NULL);
-  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
-  memset(c, 0, sizeof(grpc_composite_call_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
-  c->base.vtable = &composite_call_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  creds1_array = get_creds_array(&creds1);
-  creds2_array = get_creds_array(&creds2);
-  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
-  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
-  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
-  memset(c->inner.creds_array, 0, creds_array_byte_size);
-  for (i = 0; i < creds1_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
-    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
-  }
-  for (i = 0; i < creds2_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
-    c->inner.creds_array[i + creds1_array.num_creds] =
-        grpc_call_credentials_ref(cur_creds);
-  }
-  return &c->base;
-}
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
-  const grpc_composite_call_credentials *c =
-      (const grpc_composite_call_credentials *)creds;
-  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
-  return &c->inner;
-}
-
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds) {
-  size_t i;
-  if (strcmp(creds->type, type) == 0) {
-    if (composite_creds != NULL) *composite_creds = NULL;
-    return creds;
-  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    const grpc_call_credentials_array *inner_creds_array =
-        grpc_composite_call_credentials_get_credentials(creds);
-    for (i = 0; i < inner_creds_array->num_creds; i++) {
-      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
-        if (composite_creds != NULL) *composite_creds = creds;
-        return inner_creds_array->creds_array[i];
-      }
-    }
-  }
-  return NULL;
-}
-
-/* -- IAM credentials. -- */
-
-static void iam_destruct(grpc_call_credentials *creds) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(c->iam_md);
-}
-
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pops *pops,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
-
-grpc_call_credentials *grpc_google_iam_credentials_create(
-    const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
-      "reserved=%p)",
-      3, (token, authority_selector, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(token != NULL);
-  GPR_ASSERT(authority_selector != NULL);
-  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
-  memset(c, 0, sizeof(grpc_google_iam_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
-  c->base.vtable = &iam_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
-  return &c->base;
-}
-
-/* -- Plugin credentials. -- */
-
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
-static void plugin_destruct(grpc_call_credentials *creds) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
-    c->plugin.destroy(c->plugin.state);
-  }
-}
-
-static void plugin_md_request_metadata_ready(void *request,
-                                             const grpc_metadata *md,
-                                             size_t num_md,
-                                             grpc_status_code status,
-                                             const char *error_details) {
-  /* called from application code */
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  } else {
-    size_t i;
-    grpc_credentials_md *md_array = NULL;
-    if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
-        md_array[i].value =
-            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
-      }
-    }
-    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
-    if (md_array != NULL) {
-      for (i = 0; i < num_md; i++) {
-        gpr_slice_unref(md_array[i].key);
-        gpr_slice_unref(md_array[i].value);
-      }
-      gpr_free(md_array);
-    }
-  }
-  gpr_free(r);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_credentials *creds,
-                                        grpc_pops *pops,
-                                        grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
-    memset(request, 0, sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
-    c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
-
-grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
-    grpc_metadata_credentials_plugin plugin, void *reserved) {
-  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
-  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(*c));
-  c->base.type = plugin.type;
-  c->base.vtable = &plugin_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->plugin = plugin;
-  return &c->base;
-}
-
-/* -- Composite channel credentials. -- */
-
-static void composite_channel_destruct(grpc_channel_credentials *creds) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_channel_credentials_unref(c->inner_creds);
-  grpc_call_credentials_unref(c->call_creds);
-}
-
-static grpc_security_status composite_channel_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_ERROR;
-
-  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
-             c->inner_creds->vtable != NULL &&
-             c->inner_creds->vtable->create_security_connector != NULL);
-  /* If we are passed a call_creds, create a call composite to pass it
-     downstream. */
-  if (call_creds != NULL) {
-    grpc_call_credentials *composite_call_creds =
-        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, composite_call_creds, target, args, sc, new_args);
-    grpc_call_credentials_unref(composite_call_creds);
-  } else {
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, c->call_creds, target, args, sc, new_args);
-  }
-  return status;
-}
-
-static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
-    composite_channel_destruct, composite_channel_create_security_connector};
-
-grpc_channel_credentials *grpc_composite_channel_credentials_create(
-    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
-    void *reserved) {
-  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
-  memset(c, 0, sizeof(*c));
-  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
-  GRPC_API_TRACE(
-      "grpc_composite_channel_credentials_create(channel_creds=%p, "
-      "call_creds=%p, reserved=%p)",
-      3, (channel_creds, call_creds, reserved));
-  c->base.type = channel_creds->type;
-  c->base.vtable = &composite_channel_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
-  c->call_creds = grpc_call_credentials_ref(call_creds);
-  return &c->base;
-}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
new file mode 100644
index 0000000000..38abd84b04
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -0,0 +1,263 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/composite/composite_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/iomgr/pops.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Composite call credentials. -- */
+
+typedef struct {
+  grpc_composite_call_credentials *composite_creds;
+  size_t creds_index;
+  grpc_credentials_md_store *md_elems;
+  grpc_auth_metadata_context auth_md_context;
+  void *user_data;
+  grpc_pops *pops;
+  grpc_credentials_metadata_cb cb;
+} grpc_composite_call_credentials_metadata_context;
+
+static void composite_call_destruct(grpc_call_credentials *creds) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->inner.num_creds; i++) {
+    grpc_call_credentials_unref(c->inner.creds_array[i]);
+  }
+  gpr_free(c->inner.creds_array);
+}
+
+static void composite_call_md_context_destroy(
+    grpc_composite_call_credentials_metadata_context *ctx) {
+  grpc_credentials_md_store_unref(ctx->md_elems);
+  gpr_free(ctx);
+}
+
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
+                                       grpc_credentials_md *md_elems,
+                                       size_t num_md,
+                                       grpc_credentials_status status) {
+  grpc_composite_call_credentials_metadata_context *ctx =
+      (grpc_composite_call_credentials_metadata_context *)user_data;
+  if (status != GRPC_CREDENTIALS_OK) {
+    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
+    return;
+  }
+
+  /* Copy the metadata in the context. */
+  if (num_md > 0) {
+    size_t i;
+    for (i = 0; i < num_md; i++) {
+      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
+                                    md_elems[i].value);
+    }
+  }
+
+  /* See if we need to get some more metadata. */
+  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    grpc_call_credentials_get_request_metadata(exec_ctx, inner_creds, ctx->pops,
+                                               ctx->auth_md_context,
+                                               composite_call_metadata_cb, ctx);
+    return;
+  }
+
+  /* We're done!. */
+  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
+          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
+  composite_call_md_context_destroy(ctx);
+}
+
+static void composite_call_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
+    grpc_auth_metadata_context auth_md_context, grpc_credentials_metadata_cb cb,
+    void *user_data) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  grpc_composite_call_credentials_metadata_context *ctx;
+
+  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
+  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
+  ctx->auth_md_context = auth_md_context;
+  ctx->user_data = user_data;
+  ctx->cb = cb;
+  ctx->composite_creds = c;
+  ctx->pops = pops;
+  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
+  grpc_call_credentials_get_request_metadata(
+      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pops,
+      auth_md_context, composite_call_metadata_cb, ctx);
+}
+
+static grpc_call_credentials_vtable composite_call_credentials_vtable = {
+    composite_call_destruct, composite_call_get_request_metadata};
+
+static grpc_call_credentials_array get_creds_array(
+    grpc_call_credentials **creds_addr) {
+  grpc_call_credentials_array result;
+  grpc_call_credentials *creds = *creds_addr;
+  result.creds_array = creds_addr;
+  result.num_creds = 1;
+  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    result = *grpc_composite_call_credentials_get_credentials(creds);
+  }
+  return result;
+}
+
+grpc_call_credentials *grpc_composite_call_credentials_create(
+    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
+    void *reserved) {
+  size_t i;
+  size_t creds_array_byte_size;
+  grpc_call_credentials_array creds1_array;
+  grpc_call_credentials_array creds2_array;
+  grpc_composite_call_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
+      "reserved=%p)",
+      3, (creds1, creds2, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(creds1 != NULL);
+  GPR_ASSERT(creds2 != NULL);
+  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
+  memset(c, 0, sizeof(grpc_composite_call_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
+  c->base.vtable = &composite_call_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  creds1_array = get_creds_array(&creds1);
+  creds2_array = get_creds_array(&creds2);
+  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
+  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
+  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
+  memset(c->inner.creds_array, 0, creds_array_byte_size);
+  for (i = 0; i < creds1_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
+    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
+  }
+  for (i = 0; i < creds2_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
+    c->inner.creds_array[i + creds1_array.num_creds] =
+        grpc_call_credentials_ref(cur_creds);
+  }
+  return &c->base;
+}
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
+  const grpc_composite_call_credentials *c =
+      (const grpc_composite_call_credentials *)creds;
+  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
+  return &c->inner;
+}
+
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds) {
+  size_t i;
+  if (strcmp(creds->type, type) == 0) {
+    if (composite_creds != NULL) *composite_creds = NULL;
+    return creds;
+  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    const grpc_call_credentials_array *inner_creds_array =
+        grpc_composite_call_credentials_get_credentials(creds);
+    for (i = 0; i < inner_creds_array->num_creds; i++) {
+      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
+        if (composite_creds != NULL) *composite_creds = creds;
+        return inner_creds_array->creds_array[i];
+      }
+    }
+  }
+  return NULL;
+}
+
+/* -- Composite channel credentials. -- */
+
+static void composite_channel_destruct(grpc_channel_credentials *creds) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_channel_credentials_unref(c->inner_creds);
+  grpc_call_credentials_unref(c->call_creds);
+}
+
+static grpc_security_status composite_channel_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_ERROR;
+
+  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
+             c->inner_creds->vtable != NULL &&
+             c->inner_creds->vtable->create_security_connector != NULL);
+  /* If we are passed a call_creds, create a call composite to pass it
+     downstream. */
+  if (call_creds != NULL) {
+    grpc_call_credentials *composite_call_creds =
+        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, composite_call_creds, target, args, sc, new_args);
+    grpc_call_credentials_unref(composite_call_creds);
+  } else {
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, c->call_creds, target, args, sc, new_args);
+  }
+  return status;
+}
+
+static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
+    composite_channel_destruct, composite_channel_create_security_connector};
+
+grpc_channel_credentials *grpc_composite_channel_credentials_create(
+    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
+    void *reserved) {
+  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
+  memset(c, 0, sizeof(*c));
+  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
+  GRPC_API_TRACE(
+      "grpc_composite_channel_credentials_create(channel_creds=%p, "
+      "call_creds=%p, reserved=%p)",
+      3, (channel_creds, call_creds, reserved));
+  c->base.type = channel_creds->type;
+  c->base.vtable = &composite_channel_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
+  c->call_creds = grpc_call_credentials_ref(call_creds);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h
new file mode 100644
index 0000000000..0d8966f464
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.h
@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials **creds_array;
+  size_t num_creds;
+} grpc_call_credentials_array;
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(
+    grpc_call_credentials *composite_creds);
+
+/* Returns creds if creds is of the specified type or the inner creds of the
+   specified type (if found), if the creds is of type COMPOSITE.
+   If composite_creds is not NULL, *composite_creds will point to creds if of
+   type COMPOSITE in case of success. */
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds);
+
+/* -- Channel composite credentials. -- */
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_channel_credentials *inner_creds;
+  grpc_call_credentials *call_creds;
+} grpc_composite_channel_credentials;
+
+/* -- Composite credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_call_credentials_array inner;
+} grpc_composite_call_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
new file mode 100644
index 0000000000..b33deefb3d
--- /dev/null
+++ b/src/core/lib/security/credentials/credentials.c
@@ -0,0 +1,231 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+
+/* -- Common. -- */
+
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data) {
+  grpc_credentials_metadata_request *r =
+      gpr_malloc(sizeof(grpc_credentials_metadata_request));
+  r->creds = grpc_call_credentials_ref(creds);
+  r->cb = cb;
+  r->user_data = user_data;
+  return r;
+}
+
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r) {
+  grpc_call_credentials_unref(r->creds);
+  gpr_free(r);
+}
+
+grpc_channel_credentials *grpc_channel_credentials_ref(
+    grpc_channel_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
+  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
+  grpc_channel_credentials_unref(creds);
+}
+
+grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_call_credentials_unref(grpc_call_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_call_credentials_release(grpc_call_credentials *creds) {
+  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
+  grpc_call_credentials_unref(creds);
+}
+
+void grpc_call_credentials_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops *pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
+    if (cb != NULL) {
+      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+    }
+    return;
+  }
+  creds->vtable->get_request_metadata(exec_ctx, creds, pops, context, cb,
+                                      user_data);
+}
+
+grpc_security_status grpc_channel_credentials_create_security_connector(
+    grpc_channel_credentials *channel_creds, const char *target,
+    const grpc_channel_args *args, grpc_channel_security_connector **sc,
+    grpc_channel_args **new_args) {
+  *new_args = NULL;
+  if (channel_creds == NULL) {
+    return GRPC_SECURITY_ERROR;
+  }
+  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
+  return channel_creds->vtable->create_security_connector(
+      channel_creds, NULL, target, args, sc, new_args);
+}
+
+grpc_server_credentials *grpc_server_credentials_ref(
+    grpc_server_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_server_credentials_unref(grpc_server_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+      creds->processor.destroy(creds->processor.state);
+    }
+    gpr_free(creds);
+  }
+}
+
+void grpc_server_credentials_release(grpc_server_credentials *creds) {
+  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
+  grpc_server_credentials_unref(creds);
+}
+
+grpc_security_status grpc_server_credentials_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
+    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
+    return GRPC_SECURITY_ERROR;
+  }
+  return creds->vtable->create_security_connector(creds, sc);
+}
+
+void grpc_server_credentials_set_auth_metadata_processor(
+    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
+  GRPC_API_TRACE(
+      "grpc_server_credentials_set_auth_metadata_processor("
+      "creds=%p, "
+      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
+      3, (creds, (void *)(intptr_t)processor.process, processor.state));
+  if (creds == NULL) return;
+  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+    creds->processor.destroy(creds->processor.state);
+  }
+  creds->processor = processor;
+}
+
+static void server_credentials_pointer_arg_destroy(void *p) {
+  grpc_server_credentials_unref(p);
+}
+
+static void *server_credentials_pointer_arg_copy(void *p) {
+  return grpc_server_credentials_ref(p);
+}
+
+static int server_credentials_pointer_cmp(void *a, void *b) {
+  return GPR_ICMP(a, b);
+}
+
+static const grpc_arg_pointer_vtable cred_ptr_vtable = {
+    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
+    server_credentials_pointer_cmp};
+
+grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
+  grpc_arg arg;
+  memset(&arg, 0, sizeof(grpc_arg));
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
+  arg.value.pointer.p = p;
+  arg.value.pointer.vtable = &cred_ptr_vtable;
+  return arg;
+}
+
+grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
+  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
+  if (arg->type != GRPC_ARG_POINTER) {
+    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
+            GRPC_SERVER_CREDENTIALS_ARG);
+    return NULL;
+  }
+  return arg->value.pointer.p;
+}
+
+grpc_server_credentials *grpc_find_server_credentials_in_args(
+    const grpc_channel_args *args) {
+  size_t i;
+  if (args == NULL) return NULL;
+  for (i = 0; i < args->num_args; i++) {
+    grpc_server_credentials *p =
+        grpc_server_credentials_from_arg(&args->args[i]);
+    if (p != NULL) return p;
+  }
+  return NULL;
+}
diff --git a/src/core/lib/security/credentials.h b/src/core/lib/security/credentials/credentials.h
index 3d374949d5..d3531bcfc0 100644
--- a/src/core/lib/security/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
-#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
@@ -42,8 +42,7 @@
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/pops.h"
-#include "src/core/lib/security/json_token.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 struct grpc_http_response;
 
@@ -70,10 +69,6 @@ typedef enum {
   "x-goog-iam-authorization-token"
 #define GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY "x-goog-iam-authority-selector"
 
-#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
-#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
-  "application_default_credentials.json"
-
 #define GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS 60
 
 #define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata"
@@ -189,48 +184,11 @@ void grpc_call_credentials_get_request_metadata(
     grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
     void *user_data);
 
-typedef struct {
-  grpc_call_credentials **creds_array;
-  size_t num_creds;
-} grpc_call_credentials_array;
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(
-    grpc_call_credentials *composite_creds);
-
-/* Returns creds if creds is of the specified type or the inner creds of the
-   specified type (if found), if the creds is of type COMPOSITE.
-   If composite_creds is not NULL, *composite_creds will point to creds if of
-   type COMPOSITE in case of success. */
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds);
-
-/* Exposed for testing only. */
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
-
-void grpc_flush_cached_google_default_credentials(void);
-
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
     const char *md_key, const char *md_value, int is_async);
 
-/* Private constructor for jwt credentials from an already parsed json key.
-   Takes ownership of the key. */
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime);
-
-/* Private constructor for refresh token credentials from an already parsed
-   refresh token. Takes ownership of the refresh token. */
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token token);
-
 /* --- grpc_server_credentials. --- */
 
 typedef struct {
@@ -261,118 +219,19 @@ grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg);
 grpc_server_credentials *grpc_find_server_credentials_in_args(
     const grpc_channel_args *args);
 
-/* -- Fake transport security credentials. -- */
-
-/* Creates a fake transport security credentials object for testing. */
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
-/* Creates a fake server transport security credentials object for testing. */
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void);
-
-/* -- Ssl credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_ssl_config config;
-} grpc_ssl_credentials;
-
-typedef struct {
-  grpc_server_credentials base;
-  grpc_ssl_server_config config;
-} grpc_ssl_server_credentials;
-
-/* -- Channel composite credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_channel_credentials *inner_creds;
-  grpc_call_credentials *call_creds;
-} grpc_composite_channel_credentials;
-
-/* -- Jwt credentials -- */
-
-typedef struct {
-  grpc_call_credentials base;
-
-  /* Have a simple cache for now with just 1 entry. We could have a map based on
-     the service_url for a more sophisticated one. */
-  gpr_mu cache_mu;
-  struct {
-    grpc_credentials_md_store *jwt_md;
-    char *service_url;
-    gpr_timespec jwt_expiration;
-  } cached;
-
-  grpc_auth_json_key key;
-  gpr_timespec jwt_lifetime;
-} grpc_service_account_jwt_access_credentials;
-
-/* -- Oauth2TokenFetcher credentials --
-
-   This object is a base for credentials that need to acquire an oauth2 token
-   from an http service. */
-
-typedef struct grpc_credentials_metadata_request
-    grpc_credentials_metadata_request;
-
-typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
-                                       grpc_credentials_metadata_request *req,
-                                       grpc_httpcli_context *http_context,
-                                       grpc_pops *pops,
-                                       grpc_httpcli_response_cb response_cb,
-                                       gpr_timespec deadline);
-
-typedef struct {
-  grpc_call_credentials base;
-  gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
-  gpr_timespec token_expiration;
-  grpc_httpcli_context httpcli_context;
-  grpc_fetch_oauth2_func fetch_func;
-} grpc_oauth2_token_fetcher_credentials;
-
-/* -- GoogleRefreshToken credentials. -- */
-
-typedef struct {
-  grpc_oauth2_token_fetcher_credentials base;
-  grpc_auth_refresh_token refresh_token;
-} grpc_google_refresh_token_credentials;
-
-/* -- Oauth2 Access Token credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
-} grpc_access_token_credentials;
-
-/* --  Metadata-only Test credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
-} grpc_md_only_test_credentials;
-
-/* -- GoogleIAM credentials. -- */
+/* -- Credentials Metadata Request. -- */
 
 typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
-} grpc_google_iam_credentials;
+  grpc_call_credentials *creds;
+  grpc_credentials_metadata_cb cb;
+  void *user_data;
+} grpc_credentials_metadata_request;
 
-/* -- Composite credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_call_credentials_array inner;
-} grpc_composite_call_credentials;
-
-/* -- Plugin credentials. -- */
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data);
 
-typedef struct {
-  grpc_call_credentials base;
-  grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
-} grpc_plugin_credentials;
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r);
 
-#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
index bd00194278..6a352aab3a 100644
--- a/src/core/lib/security/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <grpc/support/alloc.h>
 
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
new file mode 100644
index 0000000000..beb0e5ee1e
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -0,0 +1,138 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/executor.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Fake transport security credentials. -- */
+
+static grpc_security_status fake_transport_security_create_security_connector(
+    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  *sc = grpc_fake_channel_security_connector_create(call_creds);
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_security_status
+fake_transport_security_server_create_security_connector(
+    grpc_server_credentials *c, grpc_server_security_connector **sc) {
+  *sc = grpc_fake_server_security_connector_create();
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_channel_credentials_vtable
+    fake_transport_security_credentials_vtable = {
+        NULL, fake_transport_security_create_security_connector};
+
+static grpc_server_credentials_vtable
+    fake_transport_security_server_credentials_vtable = {
+        NULL, fake_transport_security_server_create_security_connector};
+
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
+    void) {
+  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
+  memset(c, 0, sizeof(grpc_channel_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  c->vtable = &fake_transport_security_credentials_vtable;
+  gpr_ref_init(&c->refcount, 1);
+  return c;
+}
+
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void) {
+  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
+  memset(c, 0, sizeof(grpc_server_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  gpr_ref_init(&c->refcount, 1);
+  c->vtable = &fake_transport_security_server_credentials_vtable;
+  return c;
+}
+
+/* -- Metadata-only test credentials. -- */
+
+static void md_only_test_destruct(grpc_call_credentials *creds) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+  grpc_credentials_md_store_unref(c->md_store);
+}
+
+static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
+                                          void *user_data, bool success) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
+  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
+        GRPC_CREDENTIALS_OK);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void md_only_test_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops *pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+
+  if (c->is_async) {
+    grpc_credentials_metadata_request *cb_arg =
+        grpc_credentials_metadata_request_create(creds, cb, user_data);
+    grpc_executor_enqueue(
+        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
+  } else {
+    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_call_credentials_vtable md_only_test_vtable = {
+    md_only_test_destruct, md_only_test_get_request_metadata};
+
+grpc_call_credentials *grpc_md_only_test_credentials_create(
+    const char *md_key, const char *md_value, int is_async) {
+  grpc_md_only_test_credentials *c =
+      gpr_malloc(sizeof(grpc_md_only_test_credentials));
+  memset(c, 0, sizeof(grpc_md_only_test_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &md_only_test_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->md_store = grpc_credentials_md_store_create(1);
+  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->is_async = is_async;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
new file mode 100644
index 0000000000..9cf38084a3
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+/* -- Fake transport security credentials. -- */
+
+/* Creates a fake transport security credentials object for testing. */
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
+
+/* Creates a fake server transport security credentials object for testing. */
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void);
+
+/* --  Metadata-only Test credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *md_store;
+  int is_async;
+} grpc_md_only_test_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_posix.c b/src/core/lib/security/credentials/google_default/credentials_posix.c
index a07de182a0..42c9d7f997 100644
--- a/src/core/lib/security/credentials_posix.c
+++ b/src/core/lib/security/credentials/google_default/credentials_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/credentials_win32.c b/src/core/lib/security/credentials/google_default/credentials_win32.c
index d29847af38..cd8b48080a 100644
--- a/src/core/lib/security/credentials_win32.c
+++ b/src/core/lib/security/credentials/google_default/credentials_win32.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WIN32
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c
index 395a23b1a7..ddea7a9ed2 100644
--- a/src/core/lib/security/google_default_credentials.c
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <string.h>
 
@@ -42,6 +42,8 @@
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/pops.h"
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/load_file.h"
 #include "src/core/lib/surface/api_trace.h"
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.h b/src/core/lib/security/credentials/google_default/google_default_credentials.h
new file mode 100644
index 0000000000..fac4377e2c
--- /dev/null
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
+#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
+  "application_default_credentials.json"
+
+void grpc_flush_cached_google_default_credentials(void);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
new file mode 100644
index 0000000000..90779cfc71
--- /dev/null
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/iam/iam_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void iam_destruct(grpc_call_credentials *creds) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  grpc_credentials_md_store_unref(c->iam_md);
+}
+
+static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_pops* pops,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
+     GRPC_CREDENTIALS_OK);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
+                                                  iam_get_request_metadata};
+
+grpc_call_credentials *grpc_google_iam_credentials_create(
+    const char *token, const char *authority_selector, void *reserved) {
+  grpc_google_iam_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
+      "reserved=%p)",
+      3, (token, authority_selector, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(token != NULL);
+  GPR_ASSERT(authority_selector != NULL);
+  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
+  memset(c, 0, sizeof(grpc_google_iam_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
+  c->base.vtable = &iam_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->iam_md = grpc_credentials_md_store_create(2);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  return &c->base;
+}
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h b/src/core/lib/security/credentials/iam/iam_credentials.h
index 06d6dbf29d..af54faa586 100644
--- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,14 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H
-#define GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
 
-#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
-const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void);
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *iam_md;
+} grpc_google_iam_credentials;
 
-#endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/json_token.c b/src/core/lib/security/credentials/jwt/json_token.c
index d5bc2c8d60..354c13133e 100644
--- a/src/core/lib/security/json_token.c
+++ b/src/core/lib/security/credentials/jwt/json_token.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
 
 #include <string.h>
 
@@ -39,7 +39,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
+#include "src/core/lib/security/util/json_util.h"
 #include "src/core/lib/support/string.h"
 
 #include <openssl/bio.h>
@@ -66,28 +67,6 @@ static grpc_jwt_encode_and_sign_override g_jwt_encode_and_sign_override = NULL;
 
 /* --- grpc_auth_json_key. --- */
 
-static const char *json_get_string_property(const grpc_json *json,
-                                            const char *prop_name) {
-  grpc_json *child;
-  for (child = json->child; child != NULL; child = child->next) {
-    if (strcmp(child->key, prop_name) == 0) break;
-  }
-  if (child == NULL || child->type != GRPC_JSON_STRING) {
-    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
-    return NULL;
-  }
-  return child->value;
-}
-
-static int set_json_key_string_property(const grpc_json *json,
-                                        const char *prop_name,
-                                        char **json_key_field) {
-  const char *prop_value = json_get_string_property(json, prop_name);
-  if (prop_value == NULL) return 0;
-  *json_key_field = gpr_strdup(prop_value);
-  return 1;
-}
-
 int grpc_auth_json_key_is_valid(const grpc_auth_json_key *json_key) {
   return (json_key != NULL) &&
          strcmp(json_key->type, GRPC_AUTH_JSON_TYPE_INVALID);
@@ -106,22 +85,22 @@ grpc_auth_json_key grpc_auth_json_key_create_from_json(const grpc_json *json) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "type");
+  prop_value = grpc_json_get_string_property(json, "type");
   if (prop_value == NULL ||
       strcmp(prop_value, GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT)) {
     goto end;
   }
   result.type = GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT;
 
-  if (!set_json_key_string_property(json, "private_key_id",
-                                    &result.private_key_id) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "client_email",
-                                    &result.client_email)) {
+  if (!grpc_copy_json_string_property(json, "private_key_id",
+                                      &result.private_key_id) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "client_email",
+                                      &result.client_email)) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "private_key");
+  prop_value = grpc_json_get_string_property(json, "private_key");
   if (prop_value == NULL) {
     goto end;
   }
@@ -339,73 +318,3 @@ void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func) {
   g_jwt_encode_and_sign_override = func;
 }
-
-/* --- grpc_auth_refresh_token --- */
-
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token) {
-  return (refresh_token != NULL) &&
-         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json) {
-  grpc_auth_refresh_token result;
-  const char *prop_value;
-  int success = 0;
-
-  memset(&result, 0, sizeof(grpc_auth_refresh_token));
-  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (json == NULL) {
-    gpr_log(GPR_ERROR, "Invalid json.");
-    goto end;
-  }
-
-  prop_value = json_get_string_property(json, "type");
-  if (prop_value == NULL ||
-      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
-    goto end;
-  }
-  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
-
-  if (!set_json_key_string_property(json, "client_secret",
-                                    &result.client_secret) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "refresh_token",
-                                    &result.refresh_token)) {
-    goto end;
-  }
-  success = 1;
-
-end:
-  if (!success) grpc_auth_refresh_token_destruct(&result);
-  return result;
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string) {
-  char *scratchpad = gpr_strdup(json_string);
-  grpc_json *json = grpc_json_parse_string(scratchpad);
-  grpc_auth_refresh_token result =
-      grpc_auth_refresh_token_create_from_json(json);
-  if (json != NULL) grpc_json_destroy(json);
-  gpr_free(scratchpad);
-  return result;
-}
-
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
-  if (refresh_token == NULL) return;
-  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (refresh_token->client_id != NULL) {
-    gpr_free(refresh_token->client_id);
-    refresh_token->client_id = NULL;
-  }
-  if (refresh_token->client_secret != NULL) {
-    gpr_free(refresh_token->client_secret);
-    refresh_token->client_secret = NULL;
-  }
-  if (refresh_token->refresh_token != NULL) {
-    gpr_free(refresh_token->refresh_token);
-    refresh_token->refresh_token = NULL;
-  }
-}
diff --git a/src/core/lib/security/json_token.h b/src/core/lib/security/credentials/jwt/json_token.h
index 123fa652fd..07fc5bf0e0 100644
--- a/src/core/lib/security/json_token.h
+++ b/src/core/lib/security/credentials/jwt/json_token.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
-#define GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
 
 #include <grpc/support/slice.h>
 #include <openssl/rsa.h>
@@ -43,10 +43,6 @@
 
 #define GRPC_JWT_OAUTH2_AUDIENCE "https://www.googleapis.com/oauth2/v3/token"
 
-#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
-#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
-#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
-
 /* --- auth_json_key parsing. --- */
 
 typedef struct {
@@ -89,30 +85,4 @@ typedef char *(*grpc_jwt_encode_and_sign_override)(
 void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func);
 
-/* --- auth_refresh_token parsing. --- */
-
-typedef struct {
-  const char *type;
-  char *client_id;
-  char *client_secret;
-  char *refresh_token;
-} grpc_auth_refresh_token;
-
-/* Returns 1 if the object is valid, 0 otherwise. */
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token);
-
-/* Creates a refresh token object from string. Returns an invalid object if a
-   parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string);
-
-/* Creates a refresh token object from parsed json. Returns an invalid object if
-   a parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json);
-
-/* Destructs the object. */
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
-
-#endif /* GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
new file mode 100644
index 0000000000..590f29b7b7
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -0,0 +1,160 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
+  if (c->cached.jwt_md != NULL) {
+    grpc_credentials_md_store_unref(c->cached.jwt_md);
+    c->cached.jwt_md = NULL;
+  }
+  if (c->cached.service_url != NULL) {
+    gpr_free(c->cached.service_url);
+    c->cached.service_url = NULL;
+  }
+  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+}
+
+static void jwt_destruct(grpc_call_credentials *creds) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  grpc_auth_json_key_destruct(&c->key);
+  jwt_reset_cache(c);
+  gpr_mu_destroy(&c->cache_mu);
+}
+
+static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_pops* pops,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+
+  /* See if we can return a cached jwt. */
+  grpc_credentials_md_store *jwt_md = NULL;
+  {
+    gpr_mu_lock(&c->cache_mu);
+    if (c->cached.service_url != NULL &&
+        strcmp(c->cached.service_url, context.service_url) == 0 &&
+        c->cached.jwt_md != NULL &&
+        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
+                                   gpr_now(GPR_CLOCK_REALTIME)),
+                      refresh_threshold) > 0)) {
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md == NULL) {
+    char *jwt = NULL;
+    /* Generate a new jwt. */
+    gpr_mu_lock(&c->cache_mu);
+    jwt_reset_cache(c);
+    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
+                                   c->jwt_lifetime, NULL);
+    if (jwt != NULL) {
+      char *md_value;
+      gpr_asprintf(&md_value, "Bearer %s", jwt);
+      gpr_free(jwt);
+      c->cached.jwt_expiration =
+          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
+      c->cached.service_url = gpr_strdup(context.service_url);
+      c->cached.jwt_md = grpc_credentials_md_store_create(1);
+      grpc_credentials_md_store_add_cstrings(
+          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      gpr_free(md_value);
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md != NULL) {
+    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
+       GRPC_CREDENTIALS_OK);
+    grpc_credentials_md_store_unref(jwt_md);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  }
+}
+
+static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
+                                                  jwt_get_request_metadata};
+
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime) {
+  grpc_service_account_jwt_access_credentials *c;
+  if (!grpc_auth_json_key_is_valid(&key)) {
+    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
+  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &jwt_vtable;
+  c->key = key;
+  c->jwt_lifetime = token_lifetime;
+  gpr_mu_init(&c->cache_mu);
+  jwt_reset_cache(c);
+  return &c->base;
+}
+
+grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
+    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_service_account_jwt_access_credentials_create("
+      "json_key=%s, "
+      "token_lifetime="
+      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "reserved=%p)",
+      5,
+      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
+       (int)token_lifetime.clock_type, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
+}
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
new file mode 100644
index 0000000000..d572606179
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
+
+typedef struct {
+  grpc_call_credentials base;
+
+  // Have a simple cache for now with just 1 entry. We could have a map based on
+  // the service_url for a more sophisticated one.
+  gpr_mu cache_mu;
+  struct {
+    grpc_credentials_md_store *jwt_md;
+    char *service_url;
+    gpr_timespec jwt_expiration;
+  } cached;
+
+  grpc_auth_json_key key;
+  gpr_timespec jwt_lifetime;
+} grpc_service_account_jwt_access_credentials;
+
+// Private constructor for jwt credentials from an already parsed json key.
+// Takes ownership of the key.
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H */
diff --git a/src/core/lib/security/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
index 1b6b7b175d..a3e0c2d78c 100644
--- a/src/core/lib/security/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/lib/security/jwt_verifier.h"
+#include "src/core/lib/security/credentials/jwt/jwt_verifier.h"
 
 #include <limits.h>
 #include <string.h>
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/iomgr/pops.h"
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 #include "src/core/lib/tsi/ssl_types.h"
 
 #include <grpc/support/alloc.h>
diff --git a/src/core/lib/security/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h
index 98a4f6b116..b0f6d1c240 100644
--- a/src/core/lib/security/jwt_verifier.h
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
-#define GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
 
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/json/json.h"
@@ -133,4 +133,4 @@ grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer);
 grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
                                                const char *audience);
 
-#endif /* GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H */
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
new file mode 100644
index 0000000000..131aa69fb8
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -0,0 +1,428 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/security/util/json_util.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+//
+// Auth Refresh Token.
+//
+
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token) {
+  return (refresh_token != NULL) &&
+         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json) {
+  grpc_auth_refresh_token result;
+  const char *prop_value;
+  int success = 0;
+
+  memset(&result, 0, sizeof(grpc_auth_refresh_token));
+  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (json == NULL) {
+    gpr_log(GPR_ERROR, "Invalid json.");
+    goto end;
+  }
+
+  prop_value = grpc_json_get_string_property(json, "type");
+  if (prop_value == NULL ||
+      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
+    goto end;
+  }
+  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
+
+  if (!grpc_copy_json_string_property(json, "client_secret",
+                                      &result.client_secret) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "refresh_token",
+                                      &result.refresh_token)) {
+    goto end;
+  }
+  success = 1;
+
+end:
+  if (!success) grpc_auth_refresh_token_destruct(&result);
+  return result;
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string) {
+  char *scratchpad = gpr_strdup(json_string);
+  grpc_json *json = grpc_json_parse_string(scratchpad);
+  grpc_auth_refresh_token result =
+      grpc_auth_refresh_token_create_from_json(json);
+  if (json != NULL) grpc_json_destroy(json);
+  gpr_free(scratchpad);
+  return result;
+}
+
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
+  if (refresh_token == NULL) return;
+  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (refresh_token->client_id != NULL) {
+    gpr_free(refresh_token->client_id);
+    refresh_token->client_id = NULL;
+  }
+  if (refresh_token->client_secret != NULL) {
+    gpr_free(refresh_token->client_secret);
+    refresh_token->client_secret = NULL;
+  }
+  if (refresh_token->refresh_token != NULL) {
+    gpr_free(refresh_token->refresh_token);
+    refresh_token->refresh_token = NULL;
+  }
+}
+
+//
+// Oauth2 Token Fetcher credentials.
+//
+
+static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+  gpr_mu_destroy(&c->mu);
+  grpc_httpcli_context_destroy(&c->httpcli_context);
+}
+
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const grpc_http_response *response, grpc_credentials_md_store **token_md,
+    gpr_timespec *token_lifetime) {
+  char *null_terminated_body = NULL;
+  char *new_access_token = NULL;
+  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
+  grpc_json *json = NULL;
+
+  if (response == NULL) {
+    gpr_log(GPR_ERROR, "Received NULL response.");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  }
+
+  if (response->body_length > 0) {
+    null_terminated_body = gpr_malloc(response->body_length + 1);
+    null_terminated_body[response->body_length] = '\0';
+    memcpy(null_terminated_body, response->body, response->body_length);
+  }
+
+  if (response->status != 200) {
+    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
+            response->status,
+            null_terminated_body != NULL ? null_terminated_body : "");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  } else {
+    grpc_json *access_token = NULL;
+    grpc_json *token_type = NULL;
+    grpc_json *expires_in = NULL;
+    grpc_json *ptr;
+    json = grpc_json_parse_string(null_terminated_body);
+    if (json == NULL) {
+      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (json->type != GRPC_JSON_OBJECT) {
+      gpr_log(GPR_ERROR, "Response should be a JSON object");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    for (ptr = json->child; ptr; ptr = ptr->next) {
+      if (strcmp(ptr->key, "access_token") == 0) {
+        access_token = ptr;
+      } else if (strcmp(ptr->key, "token_type") == 0) {
+        token_type = ptr;
+      } else if (strcmp(ptr->key, "expires_in") == 0) {
+        expires_in = ptr;
+      }
+    }
+    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
+      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
+                 access_token->value);
+    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
+    token_lifetime->tv_nsec = 0;
+    token_lifetime->clock_type = GPR_TIMESPAN;
+    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
+    *token_md = grpc_credentials_md_store_create(1);
+    grpc_credentials_md_store_add_cstrings(
+        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    status = GRPC_CREDENTIALS_OK;
+  }
+
+end:
+  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
+    grpc_credentials_md_store_unref(*token_md);
+    *token_md = NULL;
+  }
+  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
+  if (new_access_token != NULL) gpr_free(new_access_token);
+  if (json != NULL) grpc_json_destroy(json);
+  return status;
+}
+
+static void on_oauth2_token_fetcher_http_response(
+    grpc_exec_ctx *exec_ctx, void *user_data,
+    const grpc_http_response *response) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  gpr_timespec token_lifetime;
+  grpc_credentials_status status;
+
+  gpr_mu_lock(&c->mu);
+  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
+      response, &c->access_token_md, &token_lifetime);
+  if (status == GRPC_CREDENTIALS_OK) {
+    c->token_expiration =
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
+    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
+          c->access_token_md->num_entries, status);
+  } else {
+    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+    r->cb(exec_ctx, r->user_data, NULL, 0, status);
+  }
+  gpr_mu_unlock(&c->mu);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void oauth2_token_fetcher_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops *pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+  grpc_credentials_md_store *cached_access_token_md = NULL;
+  {
+    gpr_mu_lock(&c->mu);
+    if (c->access_token_md != NULL &&
+        (gpr_time_cmp(
+             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+             refresh_threshold) > 0)) {
+      cached_access_token_md =
+          grpc_credentials_md_store_ref(c->access_token_md);
+    }
+    gpr_mu_unlock(&c->mu);
+  }
+  if (cached_access_token_md != NULL) {
+    cb(exec_ctx, user_data, cached_access_token_md->entries,
+       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
+    grpc_credentials_md_store_unref(cached_access_token_md);
+  } else {
+    c->fetch_func(
+        exec_ctx,
+        grpc_credentials_metadata_request_create(creds, cb, user_data),
+        &c->httpcli_context, pops, on_oauth2_token_fetcher_http_response,
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+  }
+}
+
+static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
+                                      grpc_fetch_oauth2_func fetch_func) {
+  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  gpr_ref_init(&c->base.refcount, 1);
+  gpr_mu_init(&c->mu);
+  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+  c->fetch_func = fetch_func;
+  grpc_httpcli_context_init(&c->httpcli_context);
+}
+
+//
+//  Google Compute Engine credentials.
+//
+
+static grpc_call_credentials_vtable compute_engine_vtable = {
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void compute_engine_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_pops* pops,
+    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
+  grpc_http_header header = {"Metadata-Flavor", "Google"};
+  grpc_httpcli_request request;
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
+  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  grpc_httpcli_get(exec_ctx, httpcli_context, pops, &request, deadline,
+                   response_cb, metadata_req);
+}
+
+grpc_call_credentials *grpc_google_compute_engine_credentials_create(
+    void *reserved) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
+  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
+  c->base.vtable = &compute_engine_vtable;
+  return &c->base;
+}
+
+//
+// Google Refresh Token credentials.
+//
+
+static void refresh_token_destruct(grpc_call_credentials *creds) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)creds;
+  grpc_auth_refresh_token_destruct(&c->refresh_token);
+  oauth2_token_fetcher_destruct(&c->base.base);
+}
+
+static grpc_call_credentials_vtable refresh_token_vtable = {
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void refresh_token_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_pops* pops,
+    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)metadata_req->creds;
+  grpc_http_header header = {"Content-Type",
+                             "application/x-www-form-urlencoded"};
+  grpc_httpcli_request request;
+  char *body = NULL;
+  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
+               c->refresh_token.client_id, c->refresh_token.client_secret,
+               c->refresh_token.refresh_token);
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
+  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  request.handshaker = &grpc_httpcli_ssl;
+  grpc_httpcli_post(exec_ctx, httpcli_context, pops, &request, body,
+                    strlen(body), deadline, response_cb, metadata_req);
+  gpr_free(body);
+}
+
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token refresh_token) {
+  grpc_google_refresh_token_credentials *c;
+  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
+    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
+  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
+  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
+  c->base.base.vtable = &refresh_token_vtable;
+  c->refresh_token = refresh_token;
+  return &c->base.base;
+}
+
+grpc_call_credentials *grpc_google_refresh_token_credentials_create(
+    const char *json_refresh_token, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
+      "reserved=%p)",
+      2, (json_refresh_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
+      grpc_auth_refresh_token_create_from_string(json_refresh_token));
+}
+
+//
+// Oauth2 Access Token credentials.
+//
+
+static void access_token_destruct(grpc_call_credentials *creds) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+}
+
+static void access_token_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops* pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
+}
+
+static grpc_call_credentials_vtable access_token_vtable = {
+    access_token_destruct, access_token_get_request_metadata};
+
+grpc_call_credentials *grpc_access_token_credentials_create(
+    const char *access_token, void *reserved) {
+  grpc_access_token_credentials *c =
+      gpr_malloc(sizeof(grpc_access_token_credentials));
+  char *token_md_value;
+  GRPC_API_TRACE(
+      "grpc_access_token_credentials_create(access_token=%s, "
+      "reserved=%p)",
+      2, (access_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_access_token_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &access_token_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->access_token_md = grpc_credentials_md_store_create(1);
+  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
+  grpc_credentials_md_store_add_cstrings(
+      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  gpr_free(token_md_value);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
new file mode 100644
index 0000000000..2ae37f5c24
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/security/credentials/credentials.h"
+
+// auth_refresh_token parsing.
+typedef struct {
+  const char *type;
+  char *client_id;
+  char *client_secret;
+  char *refresh_token;
+} grpc_auth_refresh_token;
+
+/// Returns 1 if the object is valid, 0 otherwise.
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token);
+
+/// Creates a refresh token object from string. Returns an invalid object if a
+/// parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string);
+
+/// Creates a refresh token object from parsed json. Returns an invalid object
+/// if a parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json);
+
+/// Destructs the object.
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
+
+// -- Oauth2 Token Fetcher credentials --
+//
+//  This object is a base for credentials that need to acquire an oauth2 token
+//  from an http service.
+
+typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
+                                       grpc_credentials_metadata_request *req,
+                                       grpc_httpcli_context *http_context,
+                                       grpc_pops *pops,
+                                       grpc_httpcli_response_cb response_cb,
+                                       gpr_timespec deadline);
+typedef struct {
+  grpc_call_credentials base;
+  gpr_mu mu;
+  grpc_credentials_md_store *access_token_md;
+  gpr_timespec token_expiration;
+  grpc_httpcli_context httpcli_context;
+  grpc_fetch_oauth2_func fetch_func;
+} grpc_oauth2_token_fetcher_credentials;
+
+// Google refresh token credentials.
+typedef struct {
+  grpc_oauth2_token_fetcher_credentials base;
+  grpc_auth_refresh_token refresh_token;
+} grpc_google_refresh_token_credentials;
+
+// Access token credentials.
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *access_token_md;
+} grpc_access_token_credentials;
+
+// Private constructor for refresh token credentials from an already parsed
+// refresh token. Takes ownership of the refresh token.
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token token);
+
+// Exposed for testing only.
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const struct grpc_http_response *response,
+    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
new file mode 100644
index 0000000000..348f4c929f
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -0,0 +1,129 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/plugin/plugin_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+typedef struct {
+  void *user_data;
+  grpc_credentials_metadata_cb cb;
+} grpc_metadata_plugin_request;
+
+static void plugin_destruct(grpc_call_credentials *creds) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
+    c->plugin.destroy(c->plugin.state);
+  }
+}
+
+static void plugin_md_request_metadata_ready(void *request,
+                                             const grpc_metadata *md,
+                                             size_t num_md,
+                                             grpc_status_code status,
+                                             const char *error_details) {
+  /* called from application code */
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
+  if (status != GRPC_STATUS_OK) {
+    if (error_details != NULL) {
+      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
+              error_details);
+    }
+    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  } else {
+    size_t i;
+    grpc_credentials_md *md_array = NULL;
+    if (num_md > 0) {
+      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
+      for (i = 0; i < num_md; i++) {
+        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
+        md_array[i].value =
+            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
+      }
+    }
+    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
+    if (md_array != NULL) {
+      for (i = 0; i < num_md; i++) {
+        gpr_slice_unref(md_array[i].key);
+        gpr_slice_unref(md_array[i].value);
+      }
+      gpr_free(md_array);
+    }
+  }
+  gpr_free(r);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_credentials *creds,
+                                        grpc_pops *pops,
+                                        grpc_auth_metadata_context context,
+                                        grpc_credentials_metadata_cb cb,
+                                        void *user_data) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.get_metadata != NULL) {
+    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
+    memset(request, 0, sizeof(*request));
+    request->user_data = user_data;
+    request->cb = cb;
+    c->plugin.get_metadata(c->plugin.state, context,
+                           plugin_md_request_metadata_ready, request);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_call_credentials_vtable plugin_vtable = {
+    plugin_destruct, plugin_get_request_metadata};
+
+grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
+    grpc_metadata_credentials_plugin plugin, void *reserved) {
+  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
+  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(*c));
+  c->base.type = plugin.type;
+  c->base.vtable = &plugin_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->plugin = plugin;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
new file mode 100644
index 0000000000..89073cb3d1
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_credentials_md_store *plugin_md;
+} grpc_plugin_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c
new file mode 100644
index 0000000000..545bca9d98
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c
@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/ssl/ssl_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+//
+// Utils
+//
+
+static void ssl_copy_key_material(const char *input, unsigned char **output,
+                                  size_t *output_size) {
+  *output_size = strlen(input);
+  *output = gpr_malloc(*output_size);
+  memcpy(*output, input, *output_size);
+}
+
+//
+// SSL Channel Credentials.
+//
+
+static void ssl_destruct(grpc_channel_credentials *creds) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
+  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
+}
+
+static grpc_security_status ssl_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_OK;
+  size_t i = 0;
+  const char *overridden_target_name = NULL;
+  grpc_arg new_arg;
+
+  for (i = 0; args && i < args->num_args; i++) {
+    grpc_arg *arg = &args->args[i];
+    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
+        arg->type == GRPC_ARG_STRING) {
+      overridden_target_name = arg->value.string;
+      break;
+    }
+  }
+  status = grpc_ssl_channel_security_connector_create(
+      call_creds, &c->config, target, overridden_target_name, sc);
+  if (status != GRPC_SECURITY_OK) {
+    return status;
+  }
+  new_arg.type = GRPC_ARG_STRING;
+  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
+  new_arg.value.string = "https";
+  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
+  return status;
+}
+
+static grpc_channel_credentials_vtable ssl_vtable = {
+    ssl_destruct, ssl_create_security_connector};
+
+static void ssl_build_config(const char *pem_root_certs,
+                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+                             grpc_ssl_config *config) {
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (pem_key_cert_pair != NULL) {
+    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pair->private_key,
+                          &config->pem_private_key,
+                          &config->pem_private_key_size);
+    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
+                          &config->pem_cert_chain,
+                          &config->pem_cert_chain_size);
+  }
+}
+
+grpc_channel_credentials *grpc_ssl_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+    void *reserved) {
+  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_credentials_create(pem_root_certs=%s, "
+      "pem_key_cert_pair=%p, "
+      "reserved=%p)",
+      3, (pem_root_certs, pem_key_cert_pair, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  c->base.vtable = &ssl_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
+  return &c->base;
+}
+
+//
+// SSL Server Credentials.
+//
+
+static void ssl_server_destruct(grpc_server_credentials *creds) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
+    if (c->config.pem_private_keys[i] != NULL) {
+      gpr_free(c->config.pem_private_keys[i]);
+    }
+    if (c->config.pem_cert_chains[i] != NULL) {
+      gpr_free(c->config.pem_cert_chains[i]);
+    }
+  }
+  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
+  if (c->config.pem_private_keys_sizes != NULL) {
+    gpr_free(c->config.pem_private_keys_sizes);
+  }
+  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
+  if (c->config.pem_cert_chains_sizes != NULL) {
+    gpr_free(c->config.pem_cert_chains_sizes);
+  }
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+}
+
+static grpc_security_status ssl_server_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  return grpc_ssl_server_security_connector_create(&c->config, sc);
+}
+
+static grpc_server_credentials_vtable ssl_server_vtable = {
+    ssl_server_destruct, ssl_server_create_security_connector};
+
+static void ssl_build_server_config(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    grpc_ssl_server_config *config) {
+  size_t i;
+  config->client_certificate_request = client_certificate_request;
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (num_key_cert_pairs > 0) {
+    GPR_ASSERT(pem_key_cert_pairs != NULL);
+    config->pem_private_keys =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_cert_chains =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_private_keys_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+    config->pem_cert_chains_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+  }
+  config->num_key_cert_pairs = num_key_cert_pairs;
+  for (i = 0; i < num_key_cert_pairs; i++) {
+    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
+                          &config->pem_private_keys[i],
+                          &config->pem_private_keys_sizes[i]);
+    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
+                          &config->pem_cert_chains[i],
+                          &config->pem_cert_chains_sizes[i]);
+  }
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
+  return grpc_ssl_server_credentials_create_ex(
+      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
+      force_client_auth
+          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
+          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+      reserved);
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    void *reserved) {
+  grpc_ssl_server_credentials *c =
+      gpr_malloc(sizeof(grpc_ssl_server_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_server_credentials_create_ex("
+      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
+      "client_certificate_request=%d, reserved=%p)",
+      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
+          client_certificate_request, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_server_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &ssl_server_vtable;
+  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
+                          num_key_cert_pairs, client_certificate_request,
+                          &c->config);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.h b/src/core/lib/security/credentials/ssl/ssl_credentials.h
new file mode 100644
index 0000000000..f23dbdbe49
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.h
@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_ssl_config config;
+} grpc_ssl_credentials;
+
+typedef struct {
+  grpc_server_credentials base;
+  grpc_ssl_server_config config;
+} grpc_ssl_server_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H */
diff --git a/src/core/lib/security/auth_filters.h b/src/core/lib/security/transport/auth_filters.h
index 7fb56c3f3a..f688d4ed21 100644
--- a/src/core/lib/security/auth_filters.h
+++ b/src/core/lib/security/transport/auth_filters.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
-#define GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
 
 #include "src/core/lib/channel/channel_stack.h"
 
 extern const grpc_channel_filter grpc_client_auth_filter;
 extern const grpc_channel_filter grpc_server_auth_filter;
 
-#endif /* GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H */
diff --git a/src/core/lib/security/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index ea50c8513e..ac504ec95b 100644
--- a/src/core/lib/security/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <string.h>
 
@@ -40,9 +40,9 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
diff --git a/src/core/lib/security/handshake.c b/src/core/lib/security/transport/handshake.c
index d5fe0c7b7d..6561f4b47d 100644
--- a/src/core/lib/security/handshake.c
+++ b/src/core/lib/security/transport/handshake.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -39,8 +39,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
diff --git a/src/core/lib/security/handshake.h b/src/core/lib/security/transport/handshake.h
index f34476ed49..6ed850b315 100644
--- a/src/core/lib/security/handshake.h
+++ b/src/core/lib/security/transport/handshake.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
-#define GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
 
 #include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 /* Calls the callback upon completion. Takes owership of handshaker. */
 void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
@@ -48,4 +48,4 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
 
-#endif /* GRPC_CORE_LIB_SECURITY_HANDSHAKE_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H */
diff --git a/src/core/lib/security/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
index 27b0e98910..4438c8e559 100644
--- a/src/core/lib/security/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
diff --git a/src/core/lib/security/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h
index ff1c6639de..d00075b769 100644
--- a/src/core/lib/security/secure_endpoint.h
+++ b/src/core/lib/security/transport/secure_endpoint.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
-#define GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
 
 #include <grpc/support/slice.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -46,4 +46,4 @@ grpc_endpoint *grpc_secure_endpoint_create(
     struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
     gpr_slice *leftover_slices, size_t leftover_nslices);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */
diff --git a/src/core/lib/security/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 2d2023bdf5..72173e7c9d 100644
--- a/src/core/lib/security/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -43,10 +43,10 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/alpn/alpn.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/handshake.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/handshake.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/load_file.h"
 #include "src/core/lib/support/string.h"
diff --git a/src/core/lib/security/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 2c893cd5e9..84e586deaa 100644
--- a/src/core/lib/security/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -263,4 +263,4 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     const grpc_auth_context *auth_context);
 void tsi_shallow_peer_destruct(tsi_peer *peer);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H */
diff --git a/src/core/lib/security/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
index 845320975d..249f6af975 100644
--- a/src/core/lib/security/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -33,9 +33,9 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/b64.c b/src/core/lib/security/util/b64.c
index 87f0e05280..9da42e4e73 100644
--- a/src/core/lib/security/b64.c
+++ b/src/core/lib/security/util/b64.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 
 #include <stdint.h>
 #include <string.h>
diff --git a/src/core/lib/security/b64.h b/src/core/lib/security/util/b64.h
index c515e7af2c..6908095287 100644
--- a/src/core/lib/security/b64.h
+++ b/src/core/lib/security/util/b64.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_B64_H
-#define GRPC_CORE_LIB_SECURITY_B64_H
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_B64_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_B64_H
 
 #include <grpc/support/slice.h>
 
@@ -49,4 +49,4 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe);
 gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
                                       int url_safe);
 
-#endif /* GRPC_CORE_LIB_SECURITY_B64_H */
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_B64_H */
diff --git a/src/ruby/ext/grpc/rb_signal.c b/src/core/lib/security/util/json_util.c
index a9e512374b..7eed039baa 100644
--- a/src/ruby/ext/grpc/rb_signal.c
+++ b/src/core/lib/security/util/json_util.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2016, Google Inc.
+ * Copyright 2015, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,40 +31,31 @@
  *
  */
 
-#include <ruby/ruby.h>
-#include <signal.h>
-#include <stdbool.h>
+#include "src/core/lib/security/util/json_util.h"
 
-#include <grpc/support/log.h>
-
-#include "rb_grpc.h"
+#include <string.h>
 
-static void (*old_sigint_handler)(int);
-static void (*old_sigterm_handler)(int);
-
-static volatile bool signal_received = false;
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 
-/* This has to be handled at the C level instead of Ruby, because Ruby signal
- * handlers are constrained to run in the main interpreter thread. If that main
- * thread is blocked on grpc_completion_queue_pluck, the signal handlers will
- * never run */
-static void handle_signal(int signum) {
-  signal_received = true;
-  if (signum == SIGINT) {
-    old_sigint_handler(signum);
-  } else if (signum == SIGTERM) {
-    old_sigterm_handler(signum);
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name) {
+  grpc_json *child;
+  for (child = json->child; child != NULL; child = child->next) {
+    if (strcmp(child->key, prop_name) == 0) break;
   }
+  if (child == NULL || child->type != GRPC_JSON_STRING) {
+    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
+    return NULL;
+  }
+  return child->value;
 }
 
-static VALUE grpc_rb_signal_received(VALUE self) {
-  (void)self;
-  return signal_received ? Qtrue : Qfalse;
-}
-
-void Init_grpc_signals() {
-  old_sigint_handler = signal(SIGINT, handle_signal);
-  old_sigterm_handler = signal(SIGTERM, handle_signal);
-  rb_define_singleton_method(grpc_rb_mGrpcCore, "signal_received?",
-                             grpc_rb_signal_received, 0);
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name,
+                                    char **copied_value) {
+  const char *prop_value = grpc_json_get_string_property(json, prop_name);
+  if (prop_value == NULL) return false;
+  *copied_value = gpr_strdup(prop_value);
+  return true;
 }
diff --git a/src/core/lib/security/util/json_util.h b/src/core/lib/security/util/json_util.h
new file mode 100644
index 0000000000..137900593f
--- /dev/null
+++ b/src/core/lib/security/util/json_util.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+
+#include <stdbool.h>
+
+#include "src/core/lib/json/json.h"
+
+// Constants.
+#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
+#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
+#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
+
+// Gets a child property from a json node.
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name);
+
+// Copies the value of the json child property specified by prop_name.
+// Returns false if the property was not found.
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name, char **copied_value);
+
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H */
diff --git a/src/core/lib/support/murmur_hash.c b/src/core/lib/support/murmur_hash.c
index 5711fff0c0..7137c1f313 100644
--- a/src/core/lib/support/murmur_hash.c
+++ b/src/core/lib/support/murmur_hash.c
@@ -33,6 +33,8 @@
 
 #include "src/core/lib/support/murmur_hash.h"
 
+#include <string.h>
+
 #define ROTL32(x, r) ((x) << (r)) | ((x) >> (32 - (r)))
 
 #define FMIX32(h)    \
@@ -42,10 +44,6 @@
   (h) *= 0xc2b2ae35; \
   (h) ^= (h) >> 16;
 
-/* Block read - if your platform needs to do endian-swapping or can only
-   handle aligned reads, do the conversion here */
-#define GETBLOCK32(p, i) (p)[(i)]
-
 uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
   const uint8_t *data = (const uint8_t *)key;
   const size_t nblocks = len / 4;
@@ -62,7 +60,7 @@ uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
 
   /* body */
   for (i = -(int)nblocks; i; i++) {
-    k1 = GETBLOCK32(blocks, i);
+    memcpy(&k1, blocks + i, sizeof(uint32_t));
 
     k1 *= c1;
     k1 = ROTL32(k1, 15);
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index 1f82c3bad2..5eb7cf1bf4 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -70,6 +70,8 @@ struct grpc_completion_queue {
   int shutdown;
   int shutdown_called;
   int is_server_cq;
+  /** Can the server cq accept incoming channels */
+  int is_non_listening_server_cq;
   int num_pluckers;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
   grpc_closure pollset_shutdown_done;
@@ -84,6 +86,7 @@ struct grpc_completion_queue {
 };
 
 #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
+#define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
 
 static gpr_mu g_freelist_mu;
 static grpc_completion_queue *g_freelist;
@@ -149,6 +152,7 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
   cc->shutdown = 0;
   cc->shutdown_called = 0;
   cc->is_server_cq = 0;
+  cc->is_non_listening_server_cq = 0;
   cc->num_pluckers = 0;
 #ifndef NDEBUG
   cc->outstanding_tag_count = 0;
@@ -511,6 +515,18 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
   return POLLSET_FROM_CQ(cc);
 }
 
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) {
+  return CQ_FROM_POLLSET(ps);
+}
+
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc) {
+  cc->is_non_listening_server_cq = 1;
+}
+
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) {
+  return (cc->is_non_listening_server_cq == 1);
+}
+
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; }
 
 int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; }
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index eef82cf014..3d0dd13c53 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,7 +81,10 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
                     void *done_arg, grpc_cq_completion *storage);
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps);
 
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc);
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc);
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
 int grpc_cq_is_server_cq(grpc_completion_queue *cc);
 
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 3fda2c9e1e..7ee7b51568 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -37,10 +37,10 @@
 #include <string.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
 
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index aafbc988ce..14d82b515a 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,11 +69,6 @@ typedef struct call_data call_data;
 typedef struct channel_data channel_data;
 typedef struct registered_method registered_method;
 
-typedef struct {
-  call_data *next;
-  call_data *prev;
-} call_link;
-
 typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
 
 typedef struct requested_call {
@@ -81,7 +76,6 @@ typedef struct requested_call {
   void *tag;
   grpc_server *server;
   grpc_completion_queue *cq_bound_to_call;
-  grpc_completion_queue *cq_for_notification;
   grpc_call **call;
   grpc_cq_completion completion;
   grpc_metadata_array *initial_metadata;
@@ -108,6 +102,7 @@ struct channel_data {
   grpc_server *server;
   grpc_connectivity_state connectivity_state;
   grpc_channel *channel;
+  size_t cq_idx;
   /* linked list of all channels on a server */
   channel_data *next;
   channel_data *prev;
@@ -172,7 +167,7 @@ struct request_matcher {
   grpc_server *server;
   call_data *pending_head;
   call_data *pending_tail;
-  gpr_stack_lockfree *requests;
+  gpr_stack_lockfree **requests_per_cq;
 };
 
 struct registered_method {
@@ -180,6 +175,7 @@ struct registered_method {
   char *host;
   grpc_server_register_method_payload_handling payload_handling;
   uint32_t flags;
+  /* one request matcher per method */
   request_matcher request_matcher;
   registered_method *next;
 };
@@ -195,6 +191,7 @@ struct grpc_server {
   grpc_completion_queue **cqs;
   grpc_pollset **pollsets;
   size_t cq_count;
+  bool started;
 
   /* The two following mutexes control access to server-state
      mu_global controls access to non-call-related state (e.g., channel state)
@@ -207,6 +204,7 @@ struct grpc_server {
   gpr_mu mu_call;   /* mutex for call-specific state */
 
   registered_method *registered_methods;
+  /** one request matcher for unregistered methods */
   request_matcher unregistered_request_matcher;
   /** free list of available requested_calls indices */
   gpr_stack_lockfree *request_freelist;
@@ -234,7 +232,7 @@ struct grpc_server {
 
 static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld, bool success);
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc);
+                      size_t cq_idx, requested_call *rc);
 /* Before calling maybe_finish_shutdown, we must hold mu_global and not
    hold mu_call */
 static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_server *server);
@@ -312,12 +310,19 @@ static void request_matcher_init(request_matcher *rm, size_t entries,
                                  grpc_server *server) {
   memset(rm, 0, sizeof(*rm));
   rm->server = server;
-  rm->requests = gpr_stack_lockfree_create(entries);
+  rm->requests_per_cq =
+      gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    rm->requests_per_cq[i] = gpr_stack_lockfree_create(entries);
+  }
 }
 
 static void request_matcher_destroy(request_matcher *rm) {
-  GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests) == -1);
-  gpr_stack_lockfree_destroy(rm->requests);
+  for (size_t i = 0; i < rm->server->cq_count; i++) {
+    GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1);
+    gpr_stack_lockfree_destroy(rm->requests_per_cq[i]);
+  }
+  gpr_free(rm->requests_per_cq);
 }
 
 static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, bool success) {
@@ -343,8 +348,11 @@ static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx,
                                           grpc_server *server,
                                           request_matcher *rm) {
   int request_id;
-  while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
-    fail_call(exec_ctx, server, &server->requested_calls[request_id]);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    while ((request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[i])) !=
+           -1) {
+      fail_call(exec_ctx, server, i, &server->requested_calls[request_id]);
+    }
   }
 }
 
@@ -364,15 +372,19 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
   gpr_mu_destroy(&server->mu_call);
   while ((rm = server->registered_methods) != NULL) {
     server->registered_methods = rm->next;
-    request_matcher_destroy(&rm->request_matcher);
+    if (server->started) {
+      request_matcher_destroy(&rm->request_matcher);
+    }
     gpr_free(rm->method);
     gpr_free(rm->host);
     gpr_free(rm);
   }
+  if (server->started) {
+    request_matcher_destroy(&server->unregistered_request_matcher);
+  }
   for (i = 0; i < server->cq_count; i++) {
     GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server");
   }
-  request_matcher_destroy(&server->unregistered_request_matcher);
   gpr_stack_lockfree_destroy(server->request_freelist);
   gpr_free(server->cqs);
   gpr_free(server->pollsets);
@@ -453,11 +465,11 @@ static void done_request_event(grpc_exec_ctx *exec_ctx, void *req,
 }
 
 static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                         call_data *calld, requested_call *rc) {
+                         call_data *calld, size_t cq_idx, requested_call *rc) {
   grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
   grpc_call *call = calld->call;
   *rc->call = call;
-  calld->cq_new = rc->cq_for_notification;
+  calld->cq_new = server->cqs[cq_idx];
   GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
   switch (rc->type) {
     case BATCH_CALL:
@@ -492,7 +504,9 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
 }
 
 static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
-  call_data *calld = arg;
+  grpc_call_element *call_elem = arg;
+  call_data *calld = call_elem->call_data;
+  channel_data *chand = call_elem->channel_data;
   request_matcher *rm = calld->request_matcher;
   grpc_server *server = rm->server;
 
@@ -507,26 +521,34 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     return;
   }
 
-  int request_id = gpr_stack_lockfree_pop(rm->requests);
-  if (request_id == -1) {
-    gpr_mu_lock(&server->mu_call);
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = PENDING;
-    gpr_mu_unlock(&calld->mu_state);
-    if (rm->pending_head == NULL) {
-      rm->pending_tail = rm->pending_head = calld;
+  for (size_t i = 0; i < server->cq_count; i++) {
+    size_t cq_idx = (chand->cq_idx + i) % server->cq_count;
+    int request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+    if (request_id == -1) {
+      continue;
     } else {
-      rm->pending_tail->pending_next = calld;
-      rm->pending_tail = calld;
+      gpr_mu_lock(&calld->mu_state);
+      calld->state = ACTIVATED;
+      gpr_mu_unlock(&calld->mu_state);
+      publish_call(exec_ctx, server, calld, cq_idx,
+                   &server->requested_calls[request_id]);
+      return; /* early out */
     }
-    calld->pending_next = NULL;
-    gpr_mu_unlock(&server->mu_call);
+  }
+
+  /* no cq to take the request found: queue it on the slow list */
+  gpr_mu_lock(&server->mu_call);
+  gpr_mu_lock(&calld->mu_state);
+  calld->state = PENDING;
+  gpr_mu_unlock(&calld->mu_state);
+  if (rm->pending_head == NULL) {
+    rm->pending_tail = rm->pending_head = calld;
   } else {
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = ACTIVATED;
-    gpr_mu_unlock(&calld->mu_state);
-    publish_call(exec_ctx, server, calld, &server->requested_calls[request_id]);
+    rm->pending_tail->pending_next = calld;
+    rm->pending_tail = calld;
   }
+  calld->pending_next = NULL;
+  gpr_mu_unlock(&server->mu_call);
 }
 
 static void finish_start_new_rpc(
@@ -548,14 +570,14 @@ static void finish_start_new_rpc(
 
   switch (payload_handling) {
     case GRPC_SRM_PAYLOAD_NONE:
-      publish_new_rpc(exec_ctx, calld, true);
+      publish_new_rpc(exec_ctx, elem, true);
       break;
     case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: {
       grpc_op op;
       memset(&op, 0, sizeof(op));
       op.op = GRPC_OP_RECV_MESSAGE;
       op.data.recv_message = &calld->payload;
-      grpc_closure_init(&calld->publish, publish_new_rpc, calld);
+      grpc_closure_init(&calld->publish, publish_new_rpc, elem);
       grpc_call_start_batch_and_execute(exec_ctx, calld->call, &op, 1,
                                         &calld->publish);
       break;
@@ -637,14 +659,16 @@ static int num_channels(grpc_server *server) {
 
 static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx,
                                      grpc_server *server) {
-  registered_method *rm;
-  request_matcher_kill_requests(exec_ctx, server,
-                                &server->unregistered_request_matcher);
-  request_matcher_zombify_all_pending_calls(
-      exec_ctx, &server->unregistered_request_matcher);
-  for (rm = server->registered_methods; rm; rm = rm->next) {
-    request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
-    request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+  if (server->started) {
+    request_matcher_kill_requests(exec_ctx, server,
+                                  &server->unregistered_request_matcher);
+    request_matcher_zombify_all_pending_calls(
+        exec_ctx, &server->unregistered_request_matcher);
+    for (registered_method *rm = server->registered_methods; rm;
+         rm = rm->next) {
+      request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
+      request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+    }
   }
 }
 
@@ -895,25 +919,46 @@ const grpc_channel_filter grpc_server_top_filter = {
     "server",
 };
 
-void grpc_server_register_completion_queue(grpc_server *server,
-                                           grpc_completion_queue *cq,
-                                           void *reserved) {
+static void register_completion_queue(grpc_server *server,
+                                      grpc_completion_queue *cq,
+                                      bool is_non_listening, void *reserved) {
   size_t i, n;
-  GRPC_API_TRACE(
-      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
-      (server, cq, reserved));
   GPR_ASSERT(!reserved);
   for (i = 0; i < server->cq_count; i++) {
     if (server->cqs[i] == cq) return;
   }
-  GRPC_CQ_INTERNAL_REF(cq, "server");
+
   grpc_cq_mark_server_cq(cq);
+
+  if (is_non_listening) {
+    grpc_cq_mark_non_listening_server_cq(cq);
+  }
+
+  GRPC_CQ_INTERNAL_REF(cq, "server");
   n = server->cq_count++;
   server->cqs = gpr_realloc(server->cqs,
                             server->cq_count * sizeof(grpc_completion_queue *));
   server->cqs[n] = cq;
 }
 
+void grpc_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq,
+                                           void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
+      (server, cq, reserved));
+  register_completion_queue(server, cq, false, reserved);
+}
+
+void grpc_server_register_non_listening_completion_queue(
+    grpc_server *server, grpc_completion_queue *cq, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, "
+      "reserved=%p)",
+      3, (server, cq, reserved));
+  register_completion_queue(server, cq, true, reserved);
+}
+
 grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   size_t i;
 
@@ -940,8 +985,6 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   for (i = 0; i < (size_t)server->max_requested_calls; i++) {
     gpr_stack_lockfree_push(server->request_freelist, (int)i);
   }
-  request_matcher_init(&server->unregistered_request_matcher,
-                       server->max_requested_calls, server);
   server->requested_calls = gpr_malloc(server->max_requested_calls *
                                        sizeof(*server->requested_calls));
 
@@ -985,8 +1028,6 @@ void *grpc_server_register_method(
   }
   m = gpr_malloc(sizeof(registered_method));
   memset(m, 0, sizeof(*m));
-  request_matcher_init(&m->request_matcher, server->max_requested_calls,
-                       server);
   m->method = gpr_strdup(method);
   m->host = gpr_strdup(host);
   m->next = server->registered_methods;
@@ -1003,13 +1044,23 @@ void grpc_server_start(grpc_server *server) {
 
   GRPC_API_TRACE("grpc_server_start(server=%p)", 1, (server));
 
+  server->started = true;
+  size_t pollset_count = 0;
   server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
   for (i = 0; i < server->cq_count; i++) {
-    server->pollsets[i] = grpc_cq_pollset(server->cqs[i]);
+    if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) {
+      server->pollsets[pollset_count++] = grpc_cq_pollset(server->cqs[i]);
+    }
+  }
+  request_matcher_init(&server->unregistered_request_matcher,
+                       server->max_requested_calls, server);
+  for (registered_method *rm = server->registered_methods; rm; rm = rm->next) {
+    request_matcher_init(&rm->request_matcher, server->max_requested_calls,
+                         server);
   }
 
   for (l = server->listeners; l; l = l->next) {
-    l->start(&exec_ctx, server, l->arg, server->pollsets, server->cq_count);
+    l->start(&exec_ctx, server, l->arg, server->pollsets, pollset_count);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
@@ -1017,8 +1068,8 @@ void grpc_server_start(grpc_server *server) {
 
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args) {
-  size_t i;
   size_t num_registered_methods;
   size_t alloc;
   registered_method *rm;
@@ -1033,12 +1084,6 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   uint32_t max_probes = 0;
   grpc_transport_op op;
 
-  for (i = 0; i < s->cq_count; i++) {
-    memset(&op, 0, sizeof(op));
-    op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
-    grpc_transport_perform_op(exec_ctx, transport, &op);
-  }
-
   channel =
       grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
   chand = (channel_data *)grpc_channel_stack_element(
@@ -1048,6 +1093,17 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   server_ref(s);
   chand->channel = channel;
 
+  size_t cq_idx;
+  grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
+  for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
+    if (s->cqs[cq_idx] == accepting_cq) break;
+  }
+  if (cq_idx == s->cq_count) {
+    /* completion queue not found: pick a random one to publish new calls to */
+    cq_idx = (size_t)rand() % s->cq_count;
+  }
+  chand->cq_idx = cq_idx;
+
   num_registered_methods = 0;
   for (rm = s->registered_methods; rm; rm = rm->next) {
     num_registered_methods++;
@@ -1218,19 +1274,19 @@ void grpc_server_add_listener(
 }
 
 static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
-                                          grpc_server *server,
+                                          grpc_server *server, size_t cq_idx,
                                           requested_call *rc) {
   call_data *calld = NULL;
   request_matcher *rm = NULL;
   int request_id;
   if (gpr_atm_acq_load(&server->shutdown_flag)) {
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc);
     return GRPC_CALL_OK;
   }
   request_id = gpr_stack_lockfree_pop(server->request_freelist);
   if (request_id == -1) {
     /* out of request ids: just fail this one */
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc);
     return GRPC_CALL_OK;
   }
   switch (rc->type) {
@@ -1243,12 +1299,12 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
   }
   server->requested_calls[request_id] = *rc;
   gpr_free(rc);
-  if (gpr_stack_lockfree_push(rm->requests, request_id)) {
+  if (gpr_stack_lockfree_push(rm->requests_per_cq[cq_idx], request_id)) {
     /* this was the first queued request: we need to lock and start
        matching calls */
     gpr_mu_lock(&server->mu_call);
     while ((calld = rm->pending_head) != NULL) {
-      request_id = gpr_stack_lockfree_pop(rm->requests);
+      request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
       if (request_id == -1) break;
       rm->pending_head = calld->pending_next;
       gpr_mu_unlock(&server->mu_call);
@@ -1264,7 +1320,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
         GPR_ASSERT(calld->state == PENDING);
         calld->state = ACTIVATED;
         gpr_mu_unlock(&calld->mu_state);
-        publish_call(exec_ctx, server, calld,
+        publish_call(exec_ctx, server, calld, cq_idx,
                      &server->requested_calls[request_id]);
       }
       gpr_mu_lock(&server->mu_call);
@@ -1288,7 +1344,13 @@ grpc_call_error grpc_server_request_call(
       "cq_bound_to_call=%p, cq_for_notification=%p, tag=%p)",
       7, (server, call, details, initial_metadata, cq_bound_to_call,
           cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1299,11 +1361,10 @@ grpc_call_error grpc_server_request_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.batch.details = details;
   rc->initial_metadata = initial_metadata;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
@@ -1325,7 +1386,14 @@ grpc_call_error grpc_server_request_registered_call(
       "tag=%p)",
       9, (server, rmp, call, deadline, initial_metadata, optional_payload,
           cq_bound_to_call, cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1341,26 +1409,25 @@ grpc_call_error grpc_server_request_registered_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.registered.registered_method = rm;
   rc->data.registered.deadline = deadline;
   rc->initial_metadata = initial_metadata;
   rc->data.registered.optional_payload = optional_payload;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
 }
 
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc) {
+                      size_t cq_idx, requested_call *rc) {
   *rc->call = NULL;
   rc->initial_metadata->count = 0;
 
   server_ref(server);
-  grpc_cq_end_op(exec_ctx, rc->cq_for_notification, rc->tag, 0,
-                 done_request_event, rc, &rc->completion);
+  grpc_cq_end_op(exec_ctx, server->cqs[cq_idx], rc->tag, 0, done_request_event,
+                 rc, &rc->completion);
 }
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server) {
diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h
index 470ef23c69..fb6e4d60c5 100644
--- a/src/core/lib/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -53,6 +53,7 @@ void grpc_server_add_listener(
    server */
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *server,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args);
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 5847ec9053..82c8e239f6 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -373,7 +373,8 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) {
     ss = g_static_strtab[idx];
     if (ss == NULL) break;
     if (ss->hash == hash && GPR_SLICE_LENGTH(ss->slice) == length &&
-        0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length)) {
+        (length == 0 ||
+         0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length))) {
       GPR_TIMER_END("grpc_mdstr_from_buffer", 0);
       return ss;
     }
diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc
index f955a31494..f6c3e5747c 100644
--- a/src/cpp/server/server.cc
+++ b/src/cpp/server/server.cc
@@ -295,7 +295,12 @@ Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
   grpc_channel_args channel_args;
   args->SetChannelArgs(&channel_args);
   server_ = grpc_server_create(&channel_args, nullptr);
-  grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
+  if (thread_pool_ == nullptr) {
+    grpc_server_register_non_listening_completion_queue(server_, cq_.cq(),
+                                                        nullptr);
+  } else {
+    grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
+  }
 }
 
 Server::~Server() {
@@ -407,7 +412,9 @@ bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) {
       sync_methods_->push_back(SyncRequest(unknown_method_.get(), nullptr));
     }
     for (size_t i = 0; i < num_cqs; i++) {
-      new UnimplementedAsyncRequest(this, cqs[i]);
+      if (cqs[i]->IsFrequentlyPolled()) {
+        new UnimplementedAsyncRequest(this, cqs[i]);
+      }
     }
   }
   // Start processing rpcs.
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 61f0f6ae2a..54feac3982 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -60,8 +60,9 @@ ServerBuilder::ServerBuilder()
   }
 }
 
-std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue() {
-  ServerCompletionQueue* cq = new ServerCompletionQueue();
+std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue(
+    bool is_frequently_polled) {
+  ServerCompletionQueue* cq = new ServerCompletionQueue(is_frequently_polled);
   cqs_.push_back(cq);
   return std::unique_ptr<ServerCompletionQueue>(cq);
 }
@@ -99,10 +100,12 @@ void ServerBuilder::AddListeningPort(const grpc::string& addr,
 
 std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<ThreadPoolInterface> thread_pool;
+  bool has_sync_methods = false;
   for (auto it = services_.begin(); it != services_.end(); ++it) {
     if ((*it)->service->has_synchronous_methods()) {
       if (thread_pool == nullptr) {
         thread_pool.reset(CreateDefaultThreadPool());
+        has_sync_methods = true;
         break;
       }
     }
@@ -116,6 +119,7 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
     for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
       if ((*plugin).second->has_sync_methods()) {
         thread_pool.reset(CreateDefaultThreadPool());
+        has_sync_methods = true;
         break;
       }
     }
@@ -128,10 +132,33 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<Server> server(
       new Server(thread_pool.release(), true, max_message_size_, &args));
   ServerInitializer* initializer = server->initializer();
+
+  // If the server has atleast one sync methods, we know that this is a Sync
+  // server or a Hybrid server and the completion queue (server->cq_) would be
+  // frequently polled.
+  int num_frequently_polled_cqs = has_sync_methods ? 1 : 0;
+
   for (auto cq = cqs_.begin(); cq != cqs_.end(); ++cq) {
-    grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
-                                          nullptr);
+    // A completion queue that is not polled frequently (by calling Next() or
+    // AsyncNext()) is not safe to use for listening to incoming channels.
+    // Register all such completion queues as non-listening completion queues
+    // with the GRPC core library.
+    if ((*cq)->IsFrequentlyPolled()) {
+      grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
+                                            nullptr);
+      num_frequently_polled_cqs++;
+    } else {
+      grpc_server_register_non_listening_completion_queue(server->server_,
+                                                          (*cq)->cq(), nullptr);
+    }
+  }
+
+  if (num_frequently_polled_cqs == 0) {
+    gpr_log(GPR_ERROR,
+            "At least one of the completion queues must be frequently polled");
+    return nullptr;
   }
+
   for (auto service = services_.begin(); service != services_.end();
        service++) {
     if (!server->RegisterService((*service)->host.get(), (*service)->service)) {
diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
index d92addbf54..dcdddc769e 100644
--- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
@@ -235,8 +235,16 @@ namespace Grpc.Core.Tests
             await barrier.Task;  // make sure the handler has started.
             cts.Cancel();
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseAsync);
-            Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            try
+            {
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await call.ResponseAsync;
+                Assert.Fail();
+            }
+            catch (RpcException ex)
+            {
+                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            }
         }
 
         [Test]
@@ -265,9 +273,15 @@ namespace Grpc.Core.Tests
             await handlerStartedBarrier.Task;
             cts.Cancel();
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseAsync);
-            Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
-
+            try
+            {
+                await call.ResponseAsync;
+                Assert.Fail();
+            }
+            catch (RpcException ex)
+            {
+                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            }
             Assert.AreEqual("SUCCESS", await successTcs.Task);
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
index cec8c7ce6b..6a156293ad 100644
--- a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
@@ -105,7 +105,15 @@ namespace Grpc.Core.Tests
             var parentCall = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall(new CallOptions(cancellationToken: cts.Token)));
             await readyToCancelTcs.Task;
             cts.Cancel();
-            Assert.ThrowsAsync(typeof(RpcException), async () => await parentCall);
+            try
+            {
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await parentCall;
+                Assert.Fail();
+            }
+            catch (RpcException)
+            {
+            }
             Assert.AreEqual("CHILD_CALL_CANCELLED", await successTcs.Task);
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
index ab12c120cb..6fe382751a 100644
--- a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
+++ b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
@@ -32,7 +32,7 @@
 #endregion
 
 using System;
-using System.Threading;
+using System.Linq;
 using Grpc.Core;
 using NUnit.Framework;
 
@@ -44,7 +44,11 @@ namespace Grpc.Core.Tests
         public void InitializeAndShutdownGrpcEnvironment()
         {
             var env = GrpcEnvironment.AddRef();
-            Assert.IsNotNull(env.CompletionQueue);
+            Assert.IsTrue(env.CompletionQueues.Count > 0);
+            for (int i = 0; i < env.CompletionQueues.Count; i++)
+            {
+                Assert.IsNotNull(env.CompletionQueues.ElementAt(i));
+            }
             GrpcEnvironment.Release();
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
index 0e204761f6..c35aaf680f 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
@@ -53,8 +53,6 @@ namespace Grpc.Core.Internal.Tests
         [SetUp]
         public void Init()
         {
-            var environment = GrpcEnvironment.AddRef();
-
             // Create a fake server just so we have an instance to refer to.
             // The server won't actually be used at all.
             server = new Server()
@@ -66,7 +64,6 @@ namespace Grpc.Core.Internal.Tests
             fakeCall = new FakeNativeCall();
             asyncCallServer = new AsyncCallServer<string, string>(
                 Marshallers.StringMarshaller.Serializer, Marshallers.StringMarshaller.Deserializer,
-                environment,
                 server);
             asyncCallServer.InitializeForTesting(fakeCall);
         }
@@ -75,7 +72,6 @@ namespace Grpc.Core.Internal.Tests
         public void Cleanup()
         {
             server.ShutdownAsync().Wait();
-            GrpcEnvironment.Release();
         }
 
         [Test]
@@ -136,7 +132,6 @@ namespace Grpc.Core.Internal.Tests
         public void WriteAfterCancelNotificationFails()
         {
             var finishedTask = asyncCallServer.ServerSideCallAsync();
-            var requestStream = new ServerRequestStream<string, string>(asyncCallServer);
             var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
 
             fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
@@ -181,6 +176,21 @@ namespace Grpc.Core.Internal.Tests
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
 
+        [Test]
+        public void WriteAfterWriteStatusThrowsInvalidOperationException()
+        {
+            var finishedTask = asyncCallServer.ServerSideCallAsync();
+            var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
+
+            asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata(), null);
+            Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await responseStream.WriteAsync("request1"));
+
+            fakeCall.SendStatusFromServerHandler(true);
+            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+
+            AssertFinished(asyncCallServer, fakeCall, finishedTask);
+        }
+
         static void AssertFinished(AsyncCallServer<string, string> asyncCallServer, FakeNativeCall fakeCall, Task finishedTask)
         {
             Assert.IsTrue(fakeCall.IsDisposed);
diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
index 777a1c8c50..81897f8c77 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
@@ -33,7 +33,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 
 using Grpc.Core.Internal;
@@ -82,7 +81,7 @@ namespace Grpc.Core.Internal.Tests
             Assert.ThrowsAsync(typeof(InvalidOperationException),
                 async () => await asyncCall.ReadMessageAsync());
             Assert.Throws(typeof(InvalidOperationException),
-                () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {}));
+                () => asyncCall.SendMessageAsync("abc", new WriteFlags()));
         }
 
         [Test]
@@ -103,7 +102,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.UnaryCallAsync("request1");
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument);
@@ -148,7 +147,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.ClientStreamingCallAsync();
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument);
@@ -193,7 +192,7 @@ namespace Grpc.Core.Internal.Tests
 
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.Internal),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Internal);
@@ -211,7 +210,9 @@ namespace Grpc.Core.Internal.Tests
                 new Metadata());
 
             AssertUnaryResponseSuccess(asyncCall, fakeCall, resultTask);
-            var ex = Assert.Throws<RpcException>(() => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(Status.DefaultSuccess, ex.Status);
         }
 
@@ -223,11 +224,13 @@ namespace Grpc.Core.Internal.Tests
 
             fakeCall.UnaryResponseClientHandler(true,
                 new ClientSideStatus(new Status(StatusCode.OutOfRange, ""), new Metadata()),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.OutOfRange);
-            var ex = Assert.Throws<RpcException>(() => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(StatusCode.OutOfRange, ex.Status.StatusCode);
         }
 
@@ -267,7 +270,7 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void ClientStreaming_WriteAfterCancellationRequestThrowsOperationCancelledException()
+        public void ClientStreaming_WriteAfterCancellationRequestThrowsTaskCanceledException()
         {
             var resultTask = asyncCall.ClientStreamingCallAsync();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -275,11 +278,12 @@ namespace Grpc.Core.Internal.Tests
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
 
-            Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1"));
+            var writeTask = requestStream.WriteAsync("request1");
+            Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.Cancelled),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Cancelled);
@@ -290,7 +294,7 @@ namespace Grpc.Core.Internal.Tests
         {
             asyncCall.StartServerStreamingCall("request1");
             Assert.Throws(typeof(InvalidOperationException),
-                () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {}));
+                () => asyncCall.SendMessageAsync("abc", new WriteFlags()));
         }
 
         [Test]
@@ -390,12 +394,13 @@ namespace Grpc.Core.Internal.Tests
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await requestStream.WriteAsync("request1"));
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(Status.DefaultSuccess, ex.Status);
         }
 
         [Test]
-        public void DuplexStreaming_CompleteAfterReceivingStatusFails()
+        public void DuplexStreaming_CompleteAfterReceivingStatusSuceeds()
         {
             asyncCall.StartDuplexStreamingCall();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -411,7 +416,7 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void DuplexStreaming_WriteAfterCancellationRequestThrowsOperationCancelledException()
+        public void DuplexStreaming_WriteAfterCancellationRequestThrowsTaskCanceledException()
         {
             asyncCall.StartDuplexStreamingCall();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -419,7 +424,9 @@ namespace Grpc.Core.Internal.Tests
 
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
-            Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             var readTask = responseStream.MoveNext();
             fakeCall.ReceivedMessageHandler(true, null);
diff --git a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
index c6843f10af..195119f920 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
@@ -60,7 +60,7 @@ namespace Grpc.Core.Internal.Tests
             var ev = cq.Next();
             cq.Dispose();
             GrpcEnvironment.Release();
-            Assert.AreEqual(GRPCCompletionType.Shutdown, ev.type);
+            Assert.AreEqual(CompletionQueueEvent.CompletionType.Shutdown, ev.type);
             Assert.AreNotEqual(IntPtr.Zero, ev.success);
             Assert.AreEqual(IntPtr.Zero, ev.tag);
         }
diff --git a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
index 74f7f2497a..c124ea29af 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
@@ -61,15 +61,15 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void InfFuture()
+        public void InfFutureMatchesNativeValue()
         {
-            var timespec = Timespec.InfFuture;
+            Assert.AreEqual(Timespec.NativeInfFuture, Timespec.InfFuture);
         }
 
         [Test]
-        public void InfPast()
+        public void InfPastMatchesNativeValue()
         {
-            var timespec = Timespec.InfPast;
+            Assert.AreEqual(Timespec.NativeInfPast, Timespec.InfPast);
         }
 
         [Test]
@@ -108,7 +108,7 @@ namespace Grpc.Core.Internal.Tests
             Assert.Throws(typeof(InvalidOperationException),
                 () => new Timespec(0, 1000 * 1000 * 1000).ToDateTime());
             Assert.Throws(typeof(InvalidOperationException),
-                () => new Timespec(0, 0, GPRClockType.Monotonic).ToDateTime());
+                () => new Timespec(0, 0, ClockType.Monotonic).ToDateTime());
         }
 
         [Test]
diff --git a/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs b/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
index 0663e77d1e..d770f82390 100644
--- a/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
+++ b/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
@@ -134,7 +134,15 @@ namespace Grpc.Core.Tests
         {
             helper.ClientStreamingHandler = new ClientStreamingServerMethod<string, string>(async (requestStream, context) =>
             {
-                Assert.ThrowsAsync<IOException>(async () => await requestStream.MoveNext());
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await requestStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (IOException)
+                {
+                }
                 return "RESPONSE";
             });
 
diff --git a/src/csharp/Grpc.Core/Channel.cs b/src/csharp/Grpc.Core/Channel.cs
index 93a6e6a3d9..886adfec33 100644
--- a/src/csharp/Grpc.Core/Channel.cs
+++ b/src/csharp/Grpc.Core/Channel.cs
@@ -31,7 +31,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -56,6 +55,7 @@ namespace Grpc.Core
 
         readonly string target;
         readonly GrpcEnvironment environment;
+        readonly CompletionQueueSafeHandle completionQueue;
         readonly ChannelSafeHandle handle;
         readonly Dictionary<string, ChannelOption> options;
 
@@ -75,6 +75,7 @@ namespace Grpc.Core
             EnsureUserAgentChannelOption(this.options);
             this.environment = GrpcEnvironment.AddRef();
 
+            this.completionQueue = this.environment.PickCompletionQueue();
             using (var nativeCredentials = credentials.ToNativeCredentials())
             using (var nativeChannelArgs = ChannelOptions.CreateChannelArgs(this.options.Values))
             {
@@ -135,7 +136,7 @@ namespace Grpc.Core
                     tcs.SetCanceled();
                 }
             });
-            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, environment.CompletionQueue, environment.CompletionRegistry, handler);
+            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, completionQueue, handler);
             return tcs.Task;
         }
 
@@ -231,6 +232,14 @@ namespace Grpc.Core
             }
         }
 
+        internal CompletionQueueSafeHandle CompletionQueue
+        {
+            get
+            {
+                return this.completionQueue;
+            }
+        }
+
         internal void AddCallReference(object call)
         {
             activeCallCounter.Increment();
diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj
index 95077a6ca5..a8b7b5f00d 100644
--- a/src/csharp/Grpc.Core/Grpc.Core.csproj
+++ b/src/csharp/Grpc.Core/Grpc.Core.csproj
@@ -74,7 +74,6 @@
     <Compile Include="Internal\CallSafeHandle.cs" />
     <Compile Include="Internal\ChannelSafeHandle.cs" />
     <Compile Include="Internal\CompletionQueueSafeHandle.cs" />
-    <Compile Include="Internal\Enums.cs" />
     <Compile Include="Internal\SafeHandleZeroIsInvalid.cs" />
     <Compile Include="Internal\Timespec.cs" />
     <Compile Include="Internal\GrpcThreadPool.cs" />
@@ -87,7 +86,6 @@
     <Compile Include="Utils\BenchmarkUtil.cs" />
     <Compile Include="ChannelCredentials.cs" />
     <Compile Include="Internal\ChannelArgsSafeHandle.cs" />
-    <Compile Include="Internal\AsyncCompletion.cs" />
     <Compile Include="Internal\AsyncCallBase.cs" />
     <Compile Include="Internal\AsyncCallServer.cs" />
     <Compile Include="Internal\AsyncCall.cs" />
@@ -134,6 +132,10 @@
     <Compile Include="DefaultCallInvoker.cs" />
     <Compile Include="Internal\UnimplementedCallInvoker.cs" />
     <Compile Include="Internal\InterceptingCallInvoker.cs" />
+    <Compile Include="Internal\ServerRpcNew.cs" />
+    <Compile Include="Internal\ClientSideStatus.cs" />
+    <Compile Include="Internal\ClockType.cs" />
+    <Compile Include="Internal\CallError.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Grpc.Core.nuspec" />
diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs
index bee0ef1d62..18af1099f1 100644
--- a/src/csharp/Grpc.Core/GrpcEnvironment.cs
+++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs
@@ -32,8 +32,9 @@
 #endregion
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Runtime.InteropServices;
-using System.Threading.Tasks;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
 using Grpc.Core.Utils;
@@ -51,12 +52,13 @@ namespace Grpc.Core
         static GrpcEnvironment instance;
         static int refCount;
         static int? customThreadPoolSize;
+        static int? customCompletionQueueCount;
 
         static ILogger logger = new ConsoleLogger();
 
         readonly GrpcThreadPool threadPool;
-        readonly CompletionRegistry completionRegistry;
         readonly DebugStats debugStats = new DebugStats();
+        readonly AtomicCounter cqPickerCounter = new AtomicCounter();
         bool isClosed;
 
         /// <summary>
@@ -141,36 +143,51 @@ namespace Grpc.Core
         }
 
         /// <summary>
+        /// Sets the number of completion queues in the  gRPC thread pool that polls for internal RPC events.
+        /// Can be only invoke before the <c>GrpcEnviroment</c> is started and cannot be changed afterwards.
+        /// Setting the number of completions queues is an advanced setting and you should only use it if you know what you are doing.
+        /// Most users should rely on the default value provided by gRPC library.
+        /// Note: this method is part of an experimental API that can change or be removed without any prior notice.
+        /// </summary>
+        public static void SetCompletionQueueCount(int completionQueueCount)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckState(instance == null, "Can only be set before GrpcEnvironment is initialized");
+                GrpcPreconditions.CheckArgument(completionQueueCount > 0, "threadCount needs to be a positive number");
+                customCompletionQueueCount = completionQueueCount;
+            }
+        }
+
+        /// <summary>
         /// Creates gRPC environment.
         /// </summary>
         private GrpcEnvironment()
         {
             GrpcNativeInit();
-            completionRegistry = new CompletionRegistry(this);
-            threadPool = new GrpcThreadPool(this, GetThreadPoolSizeOrDefault());
+            threadPool = new GrpcThreadPool(this, GetThreadPoolSizeOrDefault(), GetCompletionQueueCountOrDefault());
             threadPool.Start();
         }
 
         /// <summary>
-        /// Gets the completion registry used by this gRPC environment.
+        /// Gets the completion queues used by this gRPC environment.
         /// </summary>
-        internal CompletionRegistry CompletionRegistry
+        internal IReadOnlyCollection<CompletionQueueSafeHandle> CompletionQueues
         {
             get
             {
-                return this.completionRegistry;
+                return this.threadPool.CompletionQueues;
             }
         }
 
         /// <summary>
-        /// Gets the completion queue used by this gRPC environment.
+        /// Picks a completion queue in a round-robin fashion.
+        /// Shouldn't be invoked on a per-call basis (used at per-channel basis).
         /// </summary>
-        internal CompletionQueueSafeHandle CompletionQueue
+        internal CompletionQueueSafeHandle PickCompletionQueue()
         {
-            get
-            {
-                return this.threadPool.CompletionQueue;
-            }
+            var cqIndex = (int) ((cqPickerCounter.Increment() - 1) % this.threadPool.CompletionQueues.Count);
+            return this.threadPool.CompletionQueues.ElementAt(cqIndex);
         }
 
         /// <summary>
@@ -230,5 +247,15 @@ namespace Grpc.Core
             // more work, but seems to work reasonably well for a start.
             return Math.Max(MinDefaultThreadPoolSize, Environment.ProcessorCount / 2);
         }
+
+        private int GetCompletionQueueCountOrDefault()
+        {
+            if (customCompletionQueueCount.HasValue)
+            {
+                return customCompletionQueueCount.Value;
+            }
+            // by default, create a completion queue for each thread
+            return GetThreadPoolSizeOrDefault();
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs
index 55351869b5..895be690a5 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs
@@ -32,12 +32,7 @@
 #endregion
 
 using System;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
 using System.Threading.Tasks;
-using Grpc.Core.Internal;
 using Grpc.Core.Logging;
 using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
@@ -57,9 +52,11 @@ namespace Grpc.Core.Internal
         // Completion of a pending unary response if not null.
         TaskCompletionSource<TResponse> unaryResponseTcs;
 
+        // TODO(jtattermusch): this field doesn't need to be initialized for unary response calls.
         // Indicates that response streaming call has finished.
         TaskCompletionSource<object> streamingCallFinishedTcs = new TaskCompletionSource<object>();
 
+        // TODO(jtattermusch): this field could be lazy-initialized (only if someone requests the response headers).
         // Response headers set here once received.
         TaskCompletionSource<Metadata> responseHeadersTcs = new TaskCompletionSource<Metadata>();
 
@@ -67,7 +64,7 @@ namespace Grpc.Core.Internal
         ClientSideStatus? finishedStatus;
 
         public AsyncCall(CallInvocationDetails<TRequest, TResponse> callDetails)
-            : base(callDetails.RequestMarshaller.Serializer, callDetails.ResponseMarshaller.Deserializer, callDetails.Channel.Environment)
+            : base(callDetails.RequestMarshaller.Serializer, callDetails.ResponseMarshaller.Deserializer)
         {
             this.details = callDetails.WithOptions(callDetails.Options.Normalize());
             this.initialMetadataSent = true;  // we always send metadata at the very beginning of the call.
@@ -144,7 +141,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 halfcloseRequested = true;
                 readingDone = true;
@@ -171,7 +168,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 readingDone = true;
 
@@ -195,7 +192,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 halfcloseRequested = true;
 
@@ -220,7 +217,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
@@ -232,11 +229,10 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Sends a streaming request. Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendMessage(TRequest msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendMessageAsync(TRequest msg, WriteFlags writeFlags)
         {
-            StartSendMessageInternal(msg, writeFlags, completionDelegate);
+            return SendMessageInternalAsync(msg, writeFlags);
         }
 
         /// <summary>
@@ -250,29 +246,32 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Sends halfclose, indicating client is done with streaming requests.
         /// Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendCloseFromClient(AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendCloseFromClientAsync()
         {
             lock (myLock)
             {
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
-                CheckSendingAllowed(allowFinished: true);
+                GrpcPreconditions.CheckState(started);
 
-                if (!disposed && !finished)
+                var earlyResult = CheckSendPreconditionsClientSide();
+                if (earlyResult != null)
                 {
-                    call.StartSendCloseFromClient(HandleSendCloseFromClientFinished);
+                    return earlyResult;
                 }
-                else
+
+                if (disposed || finished)
                 {
                     // In case the call has already been finished by the serverside,
-                    // the halfclose has already been done implicitly, so we only
-                    // emit the notification for the completion delegate.
-                    Task.Run(() => HandleSendCloseFromClientFinished(true));
+                    // the halfclose has already been done implicitly, so just return
+                    // completed task here.
+                    halfcloseRequested = true;
+                    return Task.FromResult<object>(null);
                 }
+                call.StartSendCloseFromClient(HandleSendCloseFromClientFinished);
 
                 halfcloseRequested = true;
-                sendCompletionDelegate = completionDelegate;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
@@ -342,6 +341,45 @@ namespace Grpc.Core.Internal
             get { return true; }
         }
 
+        protected override Task CheckSendAllowedOrEarlyResult()
+        {
+            var earlyResult = CheckSendPreconditionsClientSide();
+            if (earlyResult != null)
+            {
+                return earlyResult;
+            }
+
+            if (finishedStatus.HasValue)
+            {
+                // throwing RpcException if we already received status on client
+                // side makes the most sense.
+                // Note that this throws even for StatusCode.OK.
+                // Writing after the call has finished is not a programming error because server can close
+                // the call anytime, so don't throw directly, but let the write task finish with an error.
+                var tcs = new TaskCompletionSource<object>();
+                tcs.SetException(new RpcException(finishedStatus.Value.Status));
+                return tcs.Task;
+            }
+
+            return null;
+        }
+
+        private Task CheckSendPreconditionsClientSide()
+        {
+            GrpcPreconditions.CheckState(!halfcloseRequested, "Request stream has already been completed.");
+            GrpcPreconditions.CheckState(streamingWriteTcs == null, "Only one write can be pending at a time.");
+
+            if (cancelRequested)
+            {
+                // Return a cancelled task.
+                var tcs = new TaskCompletionSource<object>();
+                tcs.SetCanceled();
+                return tcs.Task;
+            }
+
+            return null;
+        }
+
         private void Initialize(CompletionQueueSafeHandle cq)
         {
             using (Profilers.ForCurrentThread().NewScope("AsyncCall.Initialize"))
@@ -368,7 +406,7 @@ namespace Grpc.Core.Internal
                 var credentials = details.Options.Credentials;
                 using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null)
                 {
-                    var result = details.Channel.Handle.CreateCall(environment.CompletionRegistry,
+                    var result = details.Channel.Handle.CreateCall(
                                  parentCall, ContextPropagationToken.DefaultMask, cq,
                                  details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials);
                     return result;
@@ -400,6 +438,7 @@ namespace Grpc.Core.Internal
         /// </summary>
         private void HandleReceivedResponseHeaders(bool success, Metadata responseHeaders)
         {
+            // TODO(jtattermusch): handle success==false
             responseHeadersTcs.SetResult(responseHeaders);
         }
 
@@ -443,19 +482,6 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected override void CheckSendingAllowed(bool allowFinished)
-        {
-            base.CheckSendingAllowed(true);
-
-            // throwing RpcException if we already received status on client
-            // side makes the most sense.
-            // Note that this throws even for StatusCode.OK.
-            if (!allowFinished && finishedStatus.HasValue)
-            {
-                throw new RpcException(finishedStatus.Value.Status);
-            }
-        }
-
         /// <summary>
         /// Handles receive status completion for calls with streaming response.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
index 4de23706b2..cb8366c216 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
@@ -58,7 +58,6 @@ namespace Grpc.Core.Internal
         readonly Func<TWrite, byte[]> serializer;
         readonly Func<byte[], TRead> deserializer;
 
-        protected readonly GrpcEnvironment environment;
         protected readonly object myLock = new object();
 
         protected INativeCall call;
@@ -67,8 +66,8 @@ namespace Grpc.Core.Internal
         protected bool started;
         protected bool cancelRequested;
 
-        protected AsyncCompletionDelegate<object> sendCompletionDelegate;  // Completion of a pending send or sendclose if not null.
         protected TaskCompletionSource<TRead> streamingReadTcs;  // Completion of a pending streaming read if not null.
+        protected TaskCompletionSource<object> streamingWriteTcs;  // Completion of a pending streaming write or send close from client if not null.
         protected TaskCompletionSource<object> sendStatusFromServerTcs;
 
         protected bool readingDone;  // True if last read (i.e. read with null payload) was already received.
@@ -78,11 +77,10 @@ namespace Grpc.Core.Internal
         protected bool initialMetadataSent;
         protected long streamingWritesCounter;  // Number of streaming send operations started so far.
 
-        public AsyncCallBase(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer, GrpcEnvironment environment)
+        public AsyncCallBase(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer)
         {
             this.serializer = GrpcPreconditions.CheckNotNull(serializer);
             this.deserializer = GrpcPreconditions.CheckNotNull(deserializer);
-            this.environment = GrpcPreconditions.CheckNotNull(environment);
         }
 
         /// <summary>
@@ -128,28 +126,31 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Initiates sending a message. Only one send operation can be active at a time.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
-        protected void StartSendMessageInternal(TWrite msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        protected Task SendMessageInternalAsync(TWrite msg, WriteFlags writeFlags)
         {
             byte[] payload = UnsafeSerialize(msg);
 
             lock (myLock)
             {
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
-                CheckSendingAllowed(allowFinished: false);
+                GrpcPreconditions.CheckState(started);
+                var earlyResult = CheckSendAllowedOrEarlyResult();
+                if (earlyResult != null)
+                {
+                    return earlyResult;
+                }
 
                 call.StartSendMessage(HandleSendFinished, payload, writeFlags, !initialMetadataSent);
 
-                sendCompletionDelegate = completionDelegate;
                 initialMetadataSent = true;
                 streamingWritesCounter++;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
         /// <summary>
         /// Initiates reading a message. Only one read operation can be active at a time.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
         protected Task<TRead> ReadMessageInternalAsync()
         {
@@ -159,7 +160,7 @@ namespace Grpc.Core.Internal
                 if (readingDone)
                 {
                     // the last read that returns null or throws an exception is idempotent
-                    // and maintain its state.
+                    // and maintains its state.
                     GrpcPreconditions.CheckState(streamingReadTcs != null, "Call does not support streaming reads.");
                     return streamingReadTcs.Task;
                 }
@@ -183,7 +184,7 @@ namespace Grpc.Core.Internal
             {
                 if (!disposed && call != null)
                 {
-                    bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished);
+                    bool noMoreSendCompletions = streamingWriteTcs == null && (halfcloseRequested || cancelRequested || finished);
                     if (noMoreSendCompletions && readingDone && finished)
                     {
                         ReleaseResources();
@@ -213,24 +214,11 @@ namespace Grpc.Core.Internal
         {
         }
 
-        protected virtual void CheckSendingAllowed(bool allowFinished)
-        {
-            GrpcPreconditions.CheckState(started);
-            CheckNotCancelled();
-            GrpcPreconditions.CheckState(!disposed || allowFinished);
-
-            GrpcPreconditions.CheckState(!halfcloseRequested, "Already halfclosed.");
-            GrpcPreconditions.CheckState(!finished || allowFinished, "Already finished.");
-            GrpcPreconditions.CheckState(sendCompletionDelegate == null, "Only one write can be pending at a time");
-        }
-
-        protected void CheckNotCancelled()
-        {
-            if (cancelRequested)
-            {
-                throw new OperationCanceledException("Remote call has been cancelled.");
-            }
-        }
+        /// <summary>
+        /// Checks if sending is allowed and possibly returns a Task that allows short-circuiting the send
+        /// logic by directly returning the write operation result task. Normally, null is returned.
+        /// </summary>
+        protected abstract Task CheckSendAllowedOrEarlyResult();
 
         protected byte[] UnsafeSerialize(TWrite msg)
         {
@@ -259,39 +247,27 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected void FireCompletion<T>(AsyncCompletionDelegate<T> completionDelegate, T value, Exception error)
-        {
-            try
-            {
-                completionDelegate(value, error);
-            }
-            catch (Exception e)
-            {
-                Logger.Error(e, "Exception occured while invoking completion delegate.");
-            }
-        }
-
         /// <summary>
         /// Handles send completion.
         /// </summary>
         protected void HandleSendFinished(bool success)
         {
-            AsyncCompletionDelegate<object> origCompletionDelegate = null;
+            TaskCompletionSource<object> origTcs = null;
             lock (myLock)
             {
-                origCompletionDelegate = sendCompletionDelegate;
-                sendCompletionDelegate = null;
+                origTcs = streamingWriteTcs;
+                streamingWriteTcs = null;
 
                 ReleaseResourcesIfPossible();
             }
 
             if (!success)
             {
-                FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Send failed"));
+                origTcs.SetException(new InvalidOperationException("Send failed"));
             }
             else
             {
-                FireCompletion(origCompletionDelegate, null, null);
+                origTcs.SetResult(null);
             }
         }
 
@@ -300,22 +276,23 @@ namespace Grpc.Core.Internal
         /// </summary>
         protected void HandleSendCloseFromClientFinished(bool success)
         {
-            AsyncCompletionDelegate<object> origCompletionDelegate = null;
+            TaskCompletionSource<object> origTcs = null;
             lock (myLock)
             {
-                origCompletionDelegate = sendCompletionDelegate;
-                sendCompletionDelegate = null;
+                origTcs = streamingWriteTcs;
+                streamingWriteTcs = null;
 
                 ReleaseResourcesIfPossible();
             }
 
             if (!success)
             {
-                FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Sending close from client has failed."));
+                // TODO(jtattermusch): this method is same as HandleSendFinished (only the error message differs).
+                origTcs.SetException(new InvalidOperationException("Sending close from client has failed."));
             }
             else
             {
-                FireCompletion(origCompletionDelegate, null, null);
+                origTcs.SetResult(null);
             }
         }
 
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
index b1566b44a7..56c23ba3ef 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
@@ -51,14 +51,14 @@ namespace Grpc.Core.Internal
         readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
         readonly Server server;
 
-        public AsyncCallServer(Func<TResponse, byte[]> serializer, Func<byte[], TRequest> deserializer, GrpcEnvironment environment, Server server) : base(serializer, deserializer, environment)
+        public AsyncCallServer(Func<TResponse, byte[]> serializer, Func<byte[], TRequest> deserializer, Server server) : base(serializer, deserializer)
         {
             this.server = GrpcPreconditions.CheckNotNull(server);
         }
 
-        public void Initialize(CallSafeHandle call)
+        public void Initialize(CallSafeHandle call, CompletionQueueSafeHandle completionQueue)
         {
-            call.Initialize(environment.CompletionRegistry, environment.CompletionQueue);
+            call.Initialize(completionQueue);
 
             server.AddCallReference(this);
             InitializeInternal(call);
@@ -91,11 +91,10 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Sends a streaming response. Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendMessage(TResponse msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendMessageAsync(TResponse msg, WriteFlags writeFlags)
         {
-            StartSendMessageInternal(msg, writeFlags, completionDelegate);
+            return SendMessageInternalAsync(msg, writeFlags);
         }
 
         /// <summary>
@@ -110,20 +109,22 @@ namespace Grpc.Core.Internal
         /// Initiates sending a initial metadata. 
         /// Even though C-core allows sending metadata in parallel to sending messages, we will treat sending metadata as a send message operation
         /// to make things simpler.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
-        public void StartSendInitialMetadata(Metadata headers, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendInitialMetadataAsync(Metadata headers)
         {
             lock (myLock)
             {
                 GrpcPreconditions.CheckNotNull(headers, "metadata");
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
 
+                GrpcPreconditions.CheckState(started);
                 GrpcPreconditions.CheckState(!initialMetadataSent, "Response headers can only be sent once per call.");
                 GrpcPreconditions.CheckState(streamingWritesCounter == 0, "Response headers can only be sent before the first write starts.");
-                CheckSendingAllowed(allowFinished: false);
 
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
+                var earlyResult = CheckSendAllowedOrEarlyResult();
+                if (earlyResult != null)
+                {
+                    return earlyResult;
+                }
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(headers))
                 {
@@ -131,7 +132,8 @@ namespace Grpc.Core.Internal
                 }
 
                 this.initialMetadataSent = true;
-                sendCompletionDelegate = completionDelegate;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
@@ -196,6 +198,16 @@ namespace Grpc.Core.Internal
             server.RemoveCallReference(this);
         }
 
+        protected override Task CheckSendAllowedOrEarlyResult()
+        {
+            GrpcPreconditions.CheckState(!halfcloseRequested, "Response stream has already been completed.");
+            GrpcPreconditions.CheckState(!finished, "Already finished.");
+            GrpcPreconditions.CheckState(streamingWriteTcs == null, "Only one write can be pending at a time");
+            GrpcPreconditions.CheckState(!disposed);
+
+            return null;
+        }
+
         /// <summary>
         /// Handles the server side close completion.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs b/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
index 66d2a66f99..c28a6f64d3 100644
--- a/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
@@ -120,107 +120,4 @@ namespace Grpc.Core.Internal
             return true;
         }
     }
-
-    /// <summary>
-    /// Status + metadata received on client side when call finishes.
-    /// (when receive_status_on_client operation finishes).
-    /// </summary>
-    internal struct ClientSideStatus
-    {
-        readonly Status status;
-        readonly Metadata trailers;
-
-        public ClientSideStatus(Status status, Metadata trailers)
-        {
-            this.status = status;
-            this.trailers = trailers;
-        }
-
-        public Status Status
-        {
-            get
-            {
-                return this.status;
-            }    
-        }
-
-        public Metadata Trailers
-        {
-            get
-            {
-                return this.trailers;
-            }
-        }
-    }
-
-    /// <summary>
-    /// Details of a newly received RPC.
-    /// </summary>
-    internal struct ServerRpcNew
-    {
-        readonly Server server;
-        readonly CallSafeHandle call;
-        readonly string method;
-        readonly string host;
-        readonly Timespec deadline;
-        readonly Metadata requestMetadata;
-
-        public ServerRpcNew(Server server, CallSafeHandle call, string method, string host, Timespec deadline, Metadata requestMetadata)
-        {
-            this.server = server;
-            this.call = call;
-            this.method = method;
-            this.host = host;
-            this.deadline = deadline;
-            this.requestMetadata = requestMetadata;
-        }
-
-        public Server Server
-        {
-            get
-            {
-                return this.server;
-            }
-        }
-
-        public CallSafeHandle Call
-        {
-            get
-            {
-                return this.call;
-            }
-        }
-
-        public string Method
-        {
-            get
-            {
-                return this.method;
-            }
-        }
-
-        public string Host
-        {
-            get
-            {
-                return this.host;
-            }
-        }
-
-        public Timespec Deadline
-        {
-            get
-            {
-                return this.deadline;
-            }
-        }
-
-        public Metadata RequestMetadata
-        {
-            get
-            {
-                return this.requestMetadata;
-            }
-        }
-    }
 }
diff --git a/src/csharp/Grpc.Core/Internal/Enums.cs b/src/csharp/Grpc.Core/Internal/CallError.cs
index 74f86d2a30..541575f5e6 100644
--- a/src/csharp/Grpc.Core/Internal/Enums.cs
+++ b/src/csharp/Grpc.Core/Internal/CallError.cs
@@ -40,7 +40,7 @@ namespace Grpc.Core.Internal
     /// <summary>
     /// grpc_call_error from grpc/grpc.h
     /// </summary>
-    internal enum GRPCCallError
+    internal enum CallError
     {
         /* everything went ok */
         OK = 0,
@@ -70,42 +70,9 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Checks the call API invocation's result is OK.
         /// </summary>
-        public static void CheckOk(this GRPCCallError callError)
+        public static void CheckOk(this CallError callError)
         {
-            GrpcPreconditions.CheckState(callError == GRPCCallError.OK, "Call error: " + callError);
+            GrpcPreconditions.CheckState(callError == CallError.OK, "Call error: " + callError);
         }
     }
-
-    /// <summary>
-    /// grpc_completion_type from grpc/grpc.h
-    /// </summary>
-    internal enum GRPCCompletionType
-    {
-        /* Shutting down */
-        Shutdown, 
-
-        /* No event before timeout */
-        Timeout,  
-
-        /* operation completion */
-        OpComplete
-    }
-
-    /// <summary>
-    /// gpr_clock_type from grpc/support/time.h
-    /// </summary>
-    internal enum GPRClockType
-    {
-        /* Monotonic clock */
-        Monotonic,
-
-        /* Realtime clock */
-        Realtime,
-
-        /* Precise clock good for performance profiling. */
-        Precise,
-
-        /* Timespan - the distance between two time points */
-        Timespan
-    }
 }
diff --git a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
index 244b97d4a4..82361f5797 100644
--- a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
@@ -47,16 +47,14 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
 
         const uint GRPC_WRITE_BUFFER_HINT = 1;
-        CompletionRegistry completionRegistry;
         CompletionQueueSafeHandle completionQueue;
 
         private CallSafeHandle()
         {
         }
 
-        public void Initialize(CompletionRegistry completionRegistry, CompletionQueueSafeHandle completionQueue)
+        public void Initialize(CompletionQueueSafeHandle completionQueue)
         {
-            this.completionRegistry = completionRegistry;
             this.completionQueue = completionQueue;
         }
 
@@ -70,7 +68,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
                     .CheckOk();
             }
@@ -90,7 +88,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_start_client_streaming(this, ctx, metadataArray).CheckOk();
             }
         }
@@ -100,7 +98,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
                 Native.grpcsharp_call_start_server_streaming(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags).CheckOk();
             }
         }
@@ -110,7 +108,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
                 Native.grpcsharp_call_start_duplex_streaming(this, ctx, metadataArray).CheckOk();
             }
         }
@@ -120,7 +118,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_message(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, sendEmptyInitialMetadata).CheckOk();
             }
         }
@@ -130,7 +128,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_close_from_client(this, ctx).CheckOk();
             }
         }
@@ -142,7 +140,7 @@ namespace Grpc.Core.Internal
             {
                 var ctx = BatchContextSafeHandle.Create();
                 var optionalPayloadLength = optionalPayload != null ? new UIntPtr((ulong)optionalPayload.Length) : UIntPtr.Zero;
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_status_from_server(this, ctx, status.StatusCode, status.Detail, metadataArray, sendEmptyInitialMetadata,
                     optionalPayload, optionalPayloadLength, writeFlags).CheckOk();
             }
@@ -153,7 +151,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
                 Native.grpcsharp_call_recv_message(this, ctx).CheckOk();
             }
         }
@@ -163,7 +161,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_recv_initial_metadata(this, ctx).CheckOk();
             }
         }
@@ -173,7 +171,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
                 Native.grpcsharp_call_start_serverside(this, ctx).CheckOk();
             }
         }
@@ -183,7 +181,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_initial_metadata(this, ctx, metadataArray).CheckOk();
             }
         }
diff --git a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
index 1dbd1f4e34..62864dff0c 100644
--- a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
@@ -63,7 +63,7 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_secure_channel_create(credentials, target, channelArgs);
         }
 
-        public CallSafeHandle CreateCall(CompletionRegistry registry, CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CallCredentialsSafeHandle credentials)
+        public CallSafeHandle CreateCall(CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CallCredentialsSafeHandle credentials)
         {
             using (Profilers.ForCurrentThread().NewScope("ChannelSafeHandle.CreateCall"))
             {
@@ -72,7 +72,7 @@ namespace Grpc.Core.Internal
                 {
                     result.SetCredentials(credentials);
                 }
-                result.Initialize(registry, cq);
+                result.Initialize(cq);
                 return result;
             }
         }
@@ -82,11 +82,10 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_channel_check_connectivity_state(this, tryToConnect ? 1 : 0);
         }
 
-        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq,
-            CompletionRegistry completionRegistry, BatchCompletionDelegate callback)
+        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq, BatchCompletionDelegate callback)
         {
             var ctx = BatchContextSafeHandle.Create();
-            completionRegistry.RegisterBatchCompletion(ctx, callback);
+            cq.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
             Native.grpcsharp_channel_watch_connectivity_state(this, lastObservedState, deadline, cq, ctx);
         }
 
diff --git a/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs b/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
index 013f00ff6f..924de028f5 100644
--- a/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
+++ b/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
@@ -50,16 +50,12 @@ namespace Grpc.Core.Internal
 
         public Task WriteAsync(TRequest message)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendMessage(message, GetWriteFlags(), taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendMessageAsync(message, GetWriteFlags());
         }
 
         public Task CompleteAsync()
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendCloseFromClient(taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendCloseFromClientAsync();
         }
 
         public WriteOptions WriteOptions
diff --git a/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs b/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs
new file mode 100644
index 0000000000..5727e3f11f
--- /dev/null
+++ b/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs
@@ -0,0 +1,70 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+// 
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using Grpc.Core;
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// Status + metadata received on client side when call finishes.
+    /// (when receive_status_on_client operation finishes).
+    /// </summary>
+    internal struct ClientSideStatus
+    {
+        readonly Status status;
+        readonly Metadata trailers;
+
+        public ClientSideStatus(Status status, Metadata trailers)
+        {
+            this.status = status;
+            this.trailers = trailers;
+        }
+
+        public Status Status
+        {
+            get
+            {
+                return this.status;
+            }
+        }
+
+        public Metadata Trailers
+        {
+            get
+            {
+                return this.trailers;
+            }
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core/Internal/ClockType.cs b/src/csharp/Grpc.Core/Internal/ClockType.cs
new file mode 100644
index 0000000000..57533c9d2f
--- /dev/null
+++ b/src/csharp/Grpc.Core/Internal/ClockType.cs
@@ -0,0 +1,53 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// gpr_clock_type from grpc/support/time.h
+    /// </summary>
+    internal enum ClockType
+    {
+        /* Monotonic clock */
+        Monotonic,
+
+        /* Realtime clock */
+        Realtime,
+
+        /* Precise clock good for performance profiling. */
+        Precise,
+
+        /* Timespan - the distance between two time points */
+        Timespan
+    }
+}
diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
index 288680792a..a78e9b70f3 100644
--- a/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
+++ b/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
@@ -44,7 +44,7 @@ namespace Grpc.Core.Internal
     {
         static readonly NativeMethods Native = NativeMethods.Get();
 
-        public GRPCCompletionType type;
+        public CompletionType type;
         public int success;
         public IntPtr tag;
 
@@ -55,5 +55,20 @@ namespace Grpc.Core.Internal
                 return Native.grpcsharp_sizeof_grpc_event();
             }
         }
+
+        /// <summary>
+        /// grpc_completion_type from grpc/grpc.h
+        /// </summary>
+        internal enum CompletionType
+        {
+            /* Shutting down */
+            Shutdown, 
+
+            /* No event before timeout */
+            Timeout,  
+
+            /* operation completion */
+            OpComplete
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
index 91364cdc70..46f5624223 100644
--- a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
@@ -45,6 +45,7 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
 
         AtomicCounter shutdownRefcount = new AtomicCounter(1);
+        CompletionRegistry completionRegistry;
 
         private CompletionQueueSafeHandle()
         {
@@ -53,7 +54,13 @@ namespace Grpc.Core.Internal
         public static CompletionQueueSafeHandle Create()
         {
             return Native.grpcsharp_completion_queue_create();
+        }
 
+        public static CompletionQueueSafeHandle Create(CompletionRegistry completionRegistry)
+        {
+            var cq = Native.grpcsharp_completion_queue_create();
+            cq.completionRegistry = completionRegistry;
+            return cq;
         }
 
         public CompletionQueueEvent Next()
@@ -83,6 +90,15 @@ namespace Grpc.Core.Internal
             DecrementShutdownRefcount();
         }
 
+        /// <summary>
+        /// Completion registry associated with this completion queue.
+        /// Doesn't need to be set if only using Pluck() operations.
+        /// </summary>
+        public CompletionRegistry CompletionRegistry
+        {
+            get { return completionRegistry; }
+        }
+
         protected override bool ReleaseHandle()
         {
             Native.grpcsharp_completion_queue_destroy(handle);
diff --git a/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs b/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
index 4b7124ee74..4de543bef7 100644
--- a/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
+++ b/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
@@ -33,15 +33,15 @@
 
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
+using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
 using Grpc.Core.Logging;
+using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
-    /// Pool of threads polling on the same completion queue.
+    /// Pool of threads polling on a set of completions queues.
     /// </summary>
     internal class GrpcThreadPool
     {
@@ -51,25 +51,31 @@ namespace Grpc.Core.Internal
         readonly object myLock = new object();
         readonly List<Thread> threads = new List<Thread>();
         readonly int poolSize;
+        readonly int completionQueueCount;
 
-        CompletionQueueSafeHandle cq;
+        IReadOnlyCollection<CompletionQueueSafeHandle> completionQueues;
 
-        public GrpcThreadPool(GrpcEnvironment environment, int poolSize)
+        /// <summary>
+        /// Creates a thread pool threads polling on a set of completions queues.
+        /// </summary>
+        /// <param name="environment">Environment.</param>
+        /// <param name="poolSize">Pool size.</param>
+        /// <param name="completionQueueCount">Completion queue count.</param>
+        public GrpcThreadPool(GrpcEnvironment environment, int poolSize, int completionQueueCount)
         {
             this.environment = environment;
             this.poolSize = poolSize;
+            this.completionQueueCount = completionQueueCount;
+            GrpcPreconditions.CheckArgument(poolSize >= completionQueueCount,
+                "Thread pool size cannot be smaller than the number of completion queues used.");
         }
 
         public void Start()
         {
             lock (myLock)
             {
-                if (cq != null)
-                {
-                    throw new InvalidOperationException("Already started.");
-                }
-
-                cq = CompletionQueueSafeHandle.Create();
+                GrpcPreconditions.CheckState(completionQueues == null, "Already started.");
+                completionQueues = CreateCompletionQueueList(environment, completionQueueCount);
 
                 for (int i = 0; i < poolSize; i++)
                 {
@@ -82,49 +88,60 @@ namespace Grpc.Core.Internal
         {
             lock (myLock)
             {
-                cq.Shutdown();
+                foreach (var cq in completionQueues)
+                {
+                    cq.Shutdown();
+                }
+
                 foreach (var thread in threads)
                 {
                     thread.Join();
                 }
 
-                cq.Dispose();
+                foreach (var cq in completionQueues)
+                {
+                    cq.Dispose();
+                }
             }
         }
 
-        internal CompletionQueueSafeHandle CompletionQueue
+        internal IReadOnlyCollection<CompletionQueueSafeHandle> CompletionQueues
         {
             get
             {
-                return cq;
+                return completionQueues;
             }
         }
 
-        private Thread CreateAndStartThread(int i)
+        private Thread CreateAndStartThread(int threadIndex)
         {
-            var thread = new Thread(new ThreadStart(RunHandlerLoop));
+            var cqIndex = threadIndex % completionQueues.Count;
+            var cq = completionQueues.ElementAt(cqIndex);
+
+            var thread = new Thread(new ThreadStart(() => RunHandlerLoop(cq)));
             thread.IsBackground = false;
+            thread.Name = string.Format("grpc {0} (cq {1})", threadIndex, cqIndex);
             thread.Start();
-            thread.Name = "grpc " + i;
+
             return thread;
         }
 
         /// <summary>
         /// Body of the polling thread.
         /// </summary>
-        private void RunHandlerLoop()
+        private void RunHandlerLoop(CompletionQueueSafeHandle cq)
         {
             CompletionQueueEvent ev;
             do
             {
                 ev = cq.Next();
-                if (ev.type == GRPCCompletionType.OpComplete)
+                if (ev.type == CompletionQueueEvent.CompletionType.OpComplete)
                 {
                     bool success = (ev.success != 0);
                     IntPtr tag = ev.tag;
                     try
                     {
-                        var callback = environment.CompletionRegistry.Extract(tag);
+                        var callback = cq.CompletionRegistry.Extract(tag);
                         callback(success);
                     }
                     catch (Exception e)
@@ -133,7 +150,18 @@ namespace Grpc.Core.Internal
                     }
                 }
             }
-            while (ev.type != GRPCCompletionType.Shutdown);
+            while (ev.type != CompletionQueueEvent.CompletionType.Shutdown);
+        }
+
+        private static IReadOnlyCollection<CompletionQueueSafeHandle> CreateCompletionQueueList(GrpcEnvironment environment, int completionQueueCount)
+        {
+            var list = new List<CompletionQueueSafeHandle>();
+            for (int i = 0; i < completionQueueCount; i++)
+            {
+                var completionRegistry = new CompletionRegistry(environment);
+                list.Add(CompletionQueueSafeHandle.Create(completionRegistry));
+            }
+            return list.AsReadOnly();
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
index 25735d5262..dc9f62fdab 100644
--- a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
@@ -50,6 +50,11 @@ namespace Grpc.Core.Internal
         {
             using (Profilers.ForCurrentThread().NewScope("MetadataArraySafeHandle.Create"))
             {
+                if (metadata.Count == 0)
+                {
+                    return new MetadataArraySafeHandle();
+                }
+
                 // TODO(jtattermusch): we might wanna check that the metadata is readonly 
                 var metadataArray = Native.grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count));
                 for (int i = 0; i < metadata.Count; i++)
diff --git a/src/csharp/Grpc.Core/Internal/NativeMethods.cs b/src/csharp/Grpc.Core/Internal/NativeMethods.cs
index c277c73ef0..65607ed120 100644
--- a/src/csharp/Grpc.Core/Internal/NativeMethods.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeMethods.cs
@@ -137,6 +137,7 @@ namespace Grpc.Core.Internal
         public readonly Delegates.grpcsharp_server_credentials_release_delegate grpcsharp_server_credentials_release;
 
         public readonly Delegates.grpcsharp_server_create_delegate grpcsharp_server_create;
+        public readonly Delegates.grpcsharp_server_register_completion_queue_delegate grpcsharp_server_register_completion_queue;
         public readonly Delegates.grpcsharp_server_add_insecure_http2_port_delegate grpcsharp_server_add_insecure_http2_port;
         public readonly Delegates.grpcsharp_server_add_secure_http2_port_delegate grpcsharp_server_add_secure_http2_port;
         public readonly Delegates.grpcsharp_server_start_delegate grpcsharp_server_start;
@@ -244,6 +245,7 @@ namespace Grpc.Core.Internal
                 this.grpcsharp_server_credentials_release = GetMethodDelegate<Delegates.grpcsharp_server_credentials_release_delegate>(library);
 
                 this.grpcsharp_server_create = GetMethodDelegate<Delegates.grpcsharp_server_create_delegate>(library);
+                this.grpcsharp_server_register_completion_queue = GetMethodDelegate<Delegates.grpcsharp_server_register_completion_queue_delegate>(library);
                 this.grpcsharp_server_add_insecure_http2_port = GetMethodDelegate<Delegates.grpcsharp_server_add_insecure_http2_port_delegate>(library);
                 this.grpcsharp_server_add_secure_http2_port = GetMethodDelegate<Delegates.grpcsharp_server_add_secure_http2_port_delegate>(library);
                 this.grpcsharp_server_start = GetMethodDelegate<Delegates.grpcsharp_server_start_delegate>(library);
@@ -348,6 +350,7 @@ namespace Grpc.Core.Internal
                 this.grpcsharp_server_credentials_release = PInvokeMethods.grpcsharp_server_credentials_release;
 
                 this.grpcsharp_server_create = PInvokeMethods.grpcsharp_server_create;
+                this.grpcsharp_server_register_completion_queue = PInvokeMethods.grpcsharp_server_register_completion_queue;
                 this.grpcsharp_server_add_insecure_http2_port = PInvokeMethods.grpcsharp_server_add_insecure_http2_port;
                 this.grpcsharp_server_add_secure_http2_port = PInvokeMethods.grpcsharp_server_add_secure_http2_port;
                 this.grpcsharp_server_start = PInvokeMethods.grpcsharp_server_start;
@@ -418,33 +421,33 @@ namespace Grpc.Core.Internal
             public delegate CallCredentialsSafeHandle grpcsharp_composite_call_credentials_create_delegate(CallCredentialsSafeHandle creds1, CallCredentialsSafeHandle creds2);
             public delegate void grpcsharp_call_credentials_release_delegate(IntPtr credentials);
 
-            public delegate GRPCCallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
-            public delegate GRPCCallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
-            public delegate GRPCCallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
+            public delegate CallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
+            public delegate CallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
                 MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
-            public delegate GRPCCallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
                 byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_recv_message_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_recv_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_start_serverside_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_serverside_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_send_initial_metadata_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_initial_metadata_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_set_credentials_delegate(CallSafeHandle call, CallCredentialsSafeHandle credentials);
+            public delegate CallError grpcsharp_call_set_credentials_delegate(CallSafeHandle call, CallCredentialsSafeHandle credentials);
             public delegate CStringSafeHandle grpcsharp_call_get_peer_delegate(CallSafeHandle call);
             public delegate void grpcsharp_call_destroy_delegate(IntPtr call);
 
@@ -493,23 +496,24 @@ namespace Grpc.Core.Internal
             public delegate ServerCredentialsSafeHandle grpcsharp_ssl_server_credentials_create_delegate(string pemRootCerts, string[] keyCertPairCertChainArray, string[] keyCertPairPrivateKeyArray, UIntPtr numKeyCertPairs, bool forceClientAuth);
             public delegate void grpcsharp_server_credentials_release_delegate(IntPtr credentials);
 
-            public delegate ServerSafeHandle grpcsharp_server_create_delegate(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args);
+            public delegate ServerSafeHandle grpcsharp_server_create_delegate(ChannelArgsSafeHandle args);
+            public delegate void grpcsharp_server_register_completion_queue_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq);
             public delegate int grpcsharp_server_add_insecure_http2_port_delegate(ServerSafeHandle server, string addr);
             public delegate int grpcsharp_server_add_secure_http2_port_delegate(ServerSafeHandle server, string addr, ServerCredentialsSafeHandle creds);
             public delegate void grpcsharp_server_start_delegate(ServerSafeHandle server);
-            public delegate GRPCCallError grpcsharp_server_request_call_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
+            public delegate CallError grpcsharp_server_request_call_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
             public delegate void grpcsharp_server_cancel_all_calls_delegate(ServerSafeHandle server);
             public delegate void grpcsharp_server_shutdown_and_notify_callback_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
             public delegate void grpcsharp_server_destroy_delegate(IntPtr server);
 
-            public delegate Timespec gprsharp_now_delegate(GPRClockType clockType);
-            public delegate Timespec gprsharp_inf_future_delegate(GPRClockType clockType);
-            public delegate Timespec gprsharp_inf_past_delegate(GPRClockType clockType);
+            public delegate Timespec gprsharp_now_delegate(ClockType clockType);
+            public delegate Timespec gprsharp_inf_future_delegate(ClockType clockType);
+            public delegate Timespec gprsharp_inf_past_delegate(ClockType clockType);
 
-            public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, GPRClockType targetClock);
+            public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, ClockType targetClock);
             public delegate int gprsharp_sizeof_timespec_delegate();
 
-            public delegate GRPCCallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public delegate CallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
             public delegate IntPtr grpcsharp_test_nop_delegate(IntPtr ptr);
         }
 
@@ -587,59 +591,59 @@ namespace Grpc.Core.Internal
             // CallSafeHandle
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_cancel(CallSafeHandle call);
+            public static extern CallError grpcsharp_call_cancel(CallSafeHandle call);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
+            public static extern CallError grpcsharp_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_unary(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_unary(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_client_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_client_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_server_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_server_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
                 MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_duplex_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_duplex_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_message(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_message(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_close_from_client(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_close_from_client(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_status_from_server(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_status_from_server(CallSafeHandle call,
                 BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
                 byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_recv_message(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_recv_message(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_recv_initial_metadata(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_recv_initial_metadata(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_serverside(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_serverside(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_initial_metadata(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_initial_metadata(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_set_credentials(CallSafeHandle call, CallCredentialsSafeHandle credentials);
+            public static extern CallError grpcsharp_call_set_credentials(CallSafeHandle call, CallCredentialsSafeHandle credentials);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern CStringSafeHandle grpcsharp_call_get_peer(CallSafeHandle call);
@@ -773,7 +777,10 @@ namespace Grpc.Core.Internal
             // ServerSafeHandle
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern ServerSafeHandle grpcsharp_server_create(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args);
+            public static extern ServerSafeHandle grpcsharp_server_create(ChannelArgsSafeHandle args);
+
+            [DllImport("grpc_csharp_ext.dll")]
+            public static extern void grpcsharp_server_register_completion_queue(ServerSafeHandle server, CompletionQueueSafeHandle cq);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern int grpcsharp_server_add_insecure_http2_port(ServerSafeHandle server, string addr);
@@ -785,7 +792,7 @@ namespace Grpc.Core.Internal
             public static extern void grpcsharp_server_start(ServerSafeHandle server);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_server_request_call(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
+            public static extern CallError grpcsharp_server_request_call(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern void grpcsharp_server_cancel_all_calls(ServerSafeHandle server);
@@ -799,16 +806,16 @@ namespace Grpc.Core.Internal
             // Timespec
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_now(GPRClockType clockType);
+            public static extern Timespec gprsharp_now(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_inf_future(GPRClockType clockType);
+            public static extern Timespec gprsharp_inf_future(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_inf_past(GPRClockType clockType);
+            public static extern Timespec gprsharp_inf_past(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_convert_clock_type(Timespec t, GPRClockType targetClock);
+            public static extern Timespec gprsharp_convert_clock_type(Timespec t, ClockType targetClock);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern int gprsharp_sizeof_timespec();
@@ -816,7 +823,7 @@ namespace Grpc.Core.Internal
             // Testing
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_test_callback([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public static extern CallError grpcsharp_test_callback([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern IntPtr grpcsharp_test_nop(IntPtr ptr);
diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
index 85b7a4b01e..6a2f520163 100644
--- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
@@ -44,7 +44,7 @@ namespace Grpc.Core.Internal
 {
     internal interface IServerCallHandler
     {
-        Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment);
+        Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq);
     }
 
     internal class UnaryServerCallHandler<TRequest, TResponse> : IServerCallHandler
@@ -62,14 +62,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -121,14 +121,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -179,14 +179,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -237,14 +237,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -281,13 +281,13 @@ namespace Grpc.Core.Internal
     {
         public static readonly NoSuchMethodCallHandler Instance = new NoSuchMethodCallHandler();
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             // We don't care about the payload type here.
             var asyncCall = new AsyncCallServer<byte[], byte[]>(
-                (payload) => payload, (payload) => payload, environment, newRpc.Server);
+                (payload) => payload, (payload) => payload, newRpc.Server);
             
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             await asyncCall.SendStatusFromServerAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty, null).ConfigureAwait(false);
             await finishedTask.ConfigureAwait(false);
@@ -317,7 +317,7 @@ namespace Grpc.Core.Internal
             where TRequest : class
             where TResponse : class
         {
-            DateTime realtimeDeadline = newRpc.Deadline.ToClockType(GPRClockType.Realtime).ToDateTime();
+            DateTime realtimeDeadline = newRpc.Deadline.ToClockType(ClockType.Realtime).ToDateTime();
 
             return new ServerCallContext(newRpc.Call, newRpc.Method, newRpc.Host, peer, realtimeDeadline,
                 newRpc.RequestMetadata, cancellationToken, serverResponseStream.WriteResponseHeadersAsync, serverResponseStream);
diff --git a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
index ecfee0bfdd..25b79b4398 100644
--- a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
@@ -52,16 +52,12 @@ namespace Grpc.Core.Internal
 
         public Task WriteAsync(TResponse message)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendMessage(message, GetWriteFlags(), taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendMessageAsync(message, GetWriteFlags());
         }
 
         public Task WriteResponseHeadersAsync(Metadata responseHeaders)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendInitialMetadata(responseHeaders, taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendInitialMetadataAsync(responseHeaders);
         }
 
         public WriteOptions WriteOptions
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCompletion.cs b/src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
index 7e86fddb4d..e4f1880bdb 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCompletion.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
@@ -32,63 +32,78 @@
 #endregion
 
 using System;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
-using System.Threading.Tasks;
-using Grpc.Core.Internal;
-using Grpc.Core.Utils;
+using Grpc.Core;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
-    /// If error != null, there's been an error or operation has been cancelled.
+    /// Details of a newly received RPC.
     /// </summary>
-    internal delegate void AsyncCompletionDelegate<T>(T result, Exception error);
-
-    /// <summary>
-    /// Helper for transforming AsyncCompletionDelegate into full-fledged Task.
-    /// </summary>
-    internal class AsyncCompletionTaskSource<T>
+    internal struct ServerRpcNew
     {
-        readonly TaskCompletionSource<T> tcs = new TaskCompletionSource<T>();
-        readonly AsyncCompletionDelegate<T> completionDelegate;
+        readonly Server server;
+        readonly CallSafeHandle call;
+        readonly string method;
+        readonly string host;
+        readonly Timespec deadline;
+        readonly Metadata requestMetadata;
+
+        public ServerRpcNew(Server server, CallSafeHandle call, string method, string host, Timespec deadline, Metadata requestMetadata)
+        {
+            this.server = server;
+            this.call = call;
+            this.method = method;
+            this.host = host;
+            this.deadline = deadline;
+            this.requestMetadata = requestMetadata;
+        }
+
+        public Server Server
+        {
+            get
+            {
+                return this.server;
+            }
+        }
 
-        public AsyncCompletionTaskSource()
+        public CallSafeHandle Call
         {
-            completionDelegate = new AsyncCompletionDelegate<T>(HandleCompletion);
+            get
+            {
+                return this.call;
+            }
         }
 
-        public Task<T> Task
+        public string Method
         {
             get
             {
-                return tcs.Task;
+                return this.method;
             }
         }
 
-        public AsyncCompletionDelegate<T> CompletionDelegate
+        public string Host
         {
             get
             {
-                return completionDelegate;
+                return this.host;
             }
         }
 
-        private void HandleCompletion(T value, Exception error)
+        public Timespec Deadline
         {
-            if (error == null)
+            get
             {
-                tcs.SetResult(value);
-                return;
+                return this.deadline;
             }
-            if (error is OperationCanceledException)
+        }
+
+        public Metadata RequestMetadata
+        {
+            get
             {
-                tcs.SetCanceled();
-                return;
+                return this.requestMetadata;
             }
-            tcs.SetException(error);
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
index 6b5f70e220..8581302706 100644
--- a/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
@@ -31,12 +31,6 @@
 
 #endregion
 
-using System;
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-using Grpc.Core.Utils;
-
 namespace Grpc.Core.Internal
 {
     /// <summary>
@@ -50,12 +44,17 @@ namespace Grpc.Core.Internal
         {
         }
 
-        public static ServerSafeHandle NewServer(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args)
+        public static ServerSafeHandle NewServer(ChannelArgsSafeHandle args)
         {
             // Increment reference count for the native gRPC environment to make sure we don't do grpc_shutdown() before destroying the server handle.
             // Doing so would make object finalizer crash if we end up abandoning the handle.
             GrpcEnvironment.GrpcNativeInit();
-            return Native.grpcsharp_server_create(cq, args);
+            return Native.grpcsharp_server_create(args);
+        }
+
+        public void RegisterCompletionQueue(CompletionQueueSafeHandle cq)
+        {
+            Native.grpcsharp_server_register_completion_queue(this, cq);
         }
 
         public int AddInsecurePort(string addr)
@@ -73,18 +72,18 @@ namespace Grpc.Core.Internal
             Native.grpcsharp_server_start(this);
         }
     
-        public void ShutdownAndNotify(BatchCompletionDelegate callback, GrpcEnvironment environment)
+        public void ShutdownAndNotify(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
             var ctx = BatchContextSafeHandle.Create();
-            environment.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
-            Native.grpcsharp_server_shutdown_and_notify_callback(this, environment.CompletionQueue, ctx);
+            completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+            Native.grpcsharp_server_shutdown_and_notify_callback(this, completionQueue, ctx);
         }
 
-        public void RequestCall(BatchCompletionDelegate callback, GrpcEnvironment environment)
+        public void RequestCall(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
             var ctx = BatchContextSafeHandle.Create();
-            environment.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
-            Native.grpcsharp_server_request_call(this, environment.CompletionQueue, ctx).CheckOk();
+            completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+            Native.grpcsharp_server_request_call(this, completionQueue, ctx).CheckOk();
         }
 
         protected override bool ReleaseHandle()
diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs
index 56172a5dda..c9fd710e1e 100644
--- a/src/csharp/Grpc.Core/Internal/Timespec.cs
+++ b/src/csharp/Grpc.Core/Internal/Timespec.cs
@@ -49,11 +49,11 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
         static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
 
-        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, GPRClockType.Realtime)
+        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, ClockType.Realtime)
         {
         }
 
-        public Timespec(long tv_sec, int tv_nsec, GPRClockType clock_type)
+        public Timespec(long tv_sec, int tv_nsec, ClockType clock_type)
         {
             this.tv_sec = tv_sec;
             this.tv_nsec = tv_nsec;
@@ -62,7 +62,7 @@ namespace Grpc.Core.Internal
 
         private long tv_sec;
         private int tv_nsec;
-        private GPRClockType clock_type;
+        private ClockType clock_type;
 
         /// <summary>
         /// Timespec a long time in the future.
@@ -71,7 +71,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_inf_future(GPRClockType.Realtime);
+                return new Timespec(long.MaxValue, 0, ClockType.Realtime);
             }
         }
 
@@ -82,7 +82,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_inf_past(GPRClockType.Realtime);
+                return new Timespec(long.MinValue, 0, ClockType.Realtime);
             }
         }
 
@@ -93,7 +93,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_now(GPRClockType.Realtime);
+                return Native.gprsharp_now(ClockType.Realtime);
             }
         }
 
@@ -122,7 +122,7 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Converts the timespec to desired clock type.
         /// </summary>
-        public Timespec ToClockType(GPRClockType targetClock)
+        public Timespec ToClockType(ClockType targetClock)
         {
             return Native.gprsharp_convert_clock_type(this, targetClock);
         }
@@ -142,7 +142,7 @@ namespace Grpc.Core.Internal
         public DateTime ToDateTime()
         {
             GrpcPreconditions.CheckState(tv_nsec >= 0 && tv_nsec < NanosPerSecond);
-            GrpcPreconditions.CheckState(clock_type == GPRClockType.Realtime);
+            GrpcPreconditions.CheckState(clock_type == ClockType.Realtime);
 
             // fast path for InfFuture
             if (this.Equals(InfFuture))
@@ -227,10 +227,11 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_now(GPRClockType.Precise);
+                return Native.gprsharp_now(ClockType.Precise);
             }
         }
 
+        // for tests only
         internal static int NativeSize
         {
             get
@@ -238,5 +239,23 @@ namespace Grpc.Core.Internal
                 return Native.gprsharp_sizeof_timespec();
             }
         }
+
+        // for tests only
+        internal static Timespec NativeInfFuture
+        {
+            get
+            {
+                return Native.gprsharp_inf_future(ClockType.Realtime);
+            }
+        }
+
+        // for tests only
+        public static Timespec NativeInfPast
+        {
+            get
+            {
+                return Native.gprsharp_inf_past(ClockType.Realtime);
+            }
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
index bde74945fb..370fa98687 100644
--- a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
+++ b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
@@ -16,6 +16,12 @@ using System.Runtime.CompilerServices;
     "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" +
     "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" +
     "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")]
+[assembly: InternalsVisibleTo("Grpc.IntegrationTesting,PublicKey=" +
+    "00240000048000009400000006020000002400005253413100040000010001002f5797a92c6fcde81bd4098f43" +
+    "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" +
+    "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" +
+    "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")]
 #else
 [assembly: InternalsVisibleTo("Grpc.Core.Tests")]
+[assembly: InternalsVisibleTo("Grpc.IntegrationTesting")]
 #endif
diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs
index d538a4671f..069185e13a 100644
--- a/src/csharp/Grpc.Core/Server.cs
+++ b/src/csharp/Grpc.Core/Server.cs
@@ -34,8 +34,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
+using System.Linq;
 using System.Threading.Tasks;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
@@ -48,7 +47,7 @@ namespace Grpc.Core
     /// </summary>
     public class Server
     {
-        const int InitialAllowRpcTokenCount = 10;
+        const int InitialAllowRpcTokenCountPerCq = 10;
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<Server>();
 
         readonly AtomicCounter activeCallCounter = new AtomicCounter();
@@ -80,7 +79,12 @@ namespace Grpc.Core
             this.options = options != null ? new List<ChannelOption>(options) : new List<ChannelOption>();
             using (var channelArgs = ChannelOptions.CreateChannelArgs(this.options))
             {
-                this.handle = ServerSafeHandle.NewServer(environment.CompletionQueue, channelArgs);
+                this.handle = ServerSafeHandle.NewServer(channelArgs);
+            }
+
+            foreach (var cq in environment.CompletionQueues)
+            {
+                this.handle.RegisterCompletionQueue(cq);
             }
         }
 
@@ -133,9 +137,12 @@ namespace Grpc.Core
 
                 // Starting with more than one AllowOneRpc tokens can significantly increase
                 // unary RPC throughput.
-                for (int i = 0; i < InitialAllowRpcTokenCount; i++)
+                for (int i = 0; i < InitialAllowRpcTokenCountPerCq; i++)
                 {
-                    AllowOneRpc();
+                    foreach (var cq in environment.CompletionQueues)
+                    {
+                        AllowOneRpc(cq);
+                    }
                 }
             }
         }
@@ -154,7 +161,8 @@ namespace Grpc.Core
                 shutdownRequested = true;
             }
 
-            handle.ShutdownAndNotify(HandleServerShutdown, environment);
+            var cq = environment.CompletionQueues.First();  // any cq will do
+            handle.ShutdownAndNotify(HandleServerShutdown, cq);
             await shutdownTcs.Task.ConfigureAwait(false);
             DisposeHandle();
 
@@ -174,7 +182,8 @@ namespace Grpc.Core
                 shutdownRequested = true;
             }
 
-            handle.ShutdownAndNotify(HandleServerShutdown, environment);
+            var cq = environment.CompletionQueues.First();  // any cq will do
+            handle.ShutdownAndNotify(HandleServerShutdown, cq);
             handle.CancelAllCalls();
             await shutdownTcs.Task.ConfigureAwait(false);
             DisposeHandle();
@@ -244,11 +253,11 @@ namespace Grpc.Core
         /// <summary>
         /// Allows one new RPC call to be received by server.
         /// </summary>
-        private void AllowOneRpc()
+        private void AllowOneRpc(CompletionQueueSafeHandle cq)
         {
             if (!shutdownRequested)
             {
-                handle.RequestCall(HandleNewServerRpc, environment);
+                handle.RequestCall((success, ctx) => HandleNewServerRpc(success, ctx, cq), cq);
             }
         }
 
@@ -265,7 +274,7 @@ namespace Grpc.Core
         /// <summary>
         /// Selects corresponding handler for given call and handles the call.
         /// </summary>
-        private async Task HandleCallAsync(ServerRpcNew newRpc)
+        private async Task HandleCallAsync(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             try
             {
@@ -274,7 +283,7 @@ namespace Grpc.Core
                 {
                     callHandler = NoSuchMethodCallHandler.Instance;
                 }
-                await callHandler.HandleCall(newRpc, environment).ConfigureAwait(false);
+                await callHandler.HandleCall(newRpc, cq).ConfigureAwait(false);
             }
             catch (Exception e)
             {
@@ -285,9 +294,9 @@ namespace Grpc.Core
         /// <summary>
         /// Handles the native callback.
         /// </summary>
-        private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx)
+        private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx, CompletionQueueSafeHandle cq)
         {
-			Task.Run(() => AllowOneRpc());
+			Task.Run(() => AllowOneRpc(cq));
 
             if (success)
             {
@@ -296,7 +305,7 @@ namespace Grpc.Core
                 // after server shutdown, the callback returns with null call
                 if (!newRpc.Call.IsInvalid)
                 {
-                    HandleCallAsync(newRpc);  // we don't need to await.
+                    HandleCallAsync(newRpc, cq);  // we don't need to await.
                 }
             }
         }
diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
index cfe668b6be..3fd28c6528 100644
--- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
+++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
@@ -37,7 +37,7 @@
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="Google.Protobuf">
-      <HintPath>..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
+      <HintPath>..\packages\Google.Protobuf.3.0.0-beta3\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
     </Reference>
     <Reference Include="nunit.framework">
       <HintPath>..\packages\NUnit.3.2.0\lib\net45\nunit.framework.dll</HintPath>
diff --git a/src/csharp/Grpc.Examples.Tests/packages.config b/src/csharp/Grpc.Examples.Tests/packages.config
index ce030f9d77..668601af8e 100644
--- a/src/csharp/Grpc.Examples.Tests/packages.config
+++ b/src/csharp/Grpc.Examples.Tests/packages.config
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Google.Protobuf" version="3.0.0-beta2" targetFramework="net45" />
+  <package id="Google.Protobuf" version="3.0.0-beta3" targetFramework="net45" />
   <package id="Ix-Async" version="1.2.5" targetFramework="net45" />
   <package id="NUnit" version="3.2.0" targetFramework="net45" />
   <package id="NUnitLite" version="3.2.0" targetFramework="net45" />
diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj
index f0a0aa3a26..30170ab03c 100644
--- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj
+++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj
@@ -39,7 +39,7 @@
   <ItemGroup>
     <Reference Include="Google.Protobuf, Version=3.0.0.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
+      <HintPath>..\packages\Google.Protobuf.3.0.0-beta3\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
     </Reference>
     <Reference Include="nunit.framework">
       <HintPath>..\packages\NUnit.3.2.0\lib\net45\nunit.framework.dll</HintPath>
diff --git a/src/csharp/Grpc.Examples/Math.cs b/src/csharp/Grpc.Examples/Math.cs
index 33c4f8d9c0..a17228c8c5 100644
--- a/src/csharp/Grpc.Examples/Math.cs
+++ b/src/csharp/Grpc.Examples/Math.cs
@@ -34,12 +34,12 @@ namespace Math {
             "Mw=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Math.DivArgs), global::Math.DivArgs.Parser, new[]{ "Dividend", "Divisor" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Math.DivReply), global::Math.DivReply.Parser, new[]{ "Quotient", "Remainder" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Math.FibArgs), global::Math.FibArgs.Parser, new[]{ "Limit" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Math.Num), global::Math.Num.Parser, new[]{ "Num_" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Math.FibReply), global::Math.FibReply.Parser, new[]{ "Count" }, null, null, null)
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Math.DivArgs), global::Math.DivArgs.Parser, new[]{ "Dividend", "Divisor" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Math.DivReply), global::Math.DivReply.Parser, new[]{ "Quotient", "Remainder" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Math.FibArgs), global::Math.FibArgs.Parser, new[]{ "Limit" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Math.Num), global::Math.Num.Parser, new[]{ "Num_" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Math.FibReply), global::Math.FibReply.Parser, new[]{ "Count" }, null, null, null)
           }));
     }
     #endregion
diff --git a/src/csharp/Grpc.Examples/packages.config b/src/csharp/Grpc.Examples/packages.config
index a424cd2ea0..a70dcbd4c6 100644
--- a/src/csharp/Grpc.Examples/packages.config
+++ b/src/csharp/Grpc.Examples/packages.config
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Google.Protobuf" version="3.0.0-beta2" targetFramework="net45" />
+  <package id="Google.Protobuf" version="3.0.0-beta3" targetFramework="net45" />
   <package id="Ix-Async" version="1.2.5" targetFramework="net45" />
   <package id="NUnit" version="3.2.0" targetFramework="net45" />
 </packages>
\ No newline at end of file
diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
index 0bea9c03e7..a5ee4fdb46 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
+++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
@@ -45,7 +45,7 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
     <Reference Include="Google.Protobuf">
-      <HintPath>..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
+      <HintPath>..\packages\Google.Protobuf.3.0.0-beta3\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
     </Reference>
     <Reference Include="nunit.framework">
       <HintPath>..\packages\NUnit.3.2.0\lib\net45\nunit.framework.dll</HintPath>
diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs
index fb292945a6..070674bae9 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs
+++ b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs
@@ -79,16 +79,17 @@ namespace Grpc.HealthCheck.Tests
         [Test]
         public void ServiceIsRunning()
         {
-            serviceImpl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING);
+            serviceImpl.SetStatus("", HealthCheckResponse.Types.ServingStatus.Serving);
 
             var response = client.Check(new HealthCheckRequest { Service = "" });
-            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, response.Status);
+            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.Serving, response.Status);
         }
 
         [Test]
         public void ServiceDoesntExist()
         {
-            Assert.Throws(Is.TypeOf(typeof(RpcException)).And.Property("Status").Property("StatusCode").EqualTo(StatusCode.NotFound), () => client.Check(new HealthCheckRequest { Service = "nonexistent.service" }));
+            var ex = Assert.Throws<RpcException>(() => client.Check(new HealthCheckRequest { Service = "nonexistent.service" }));
+            Assert.AreEqual(StatusCode.NotFound, ex.Status.StatusCode);
         }
 
         // TODO(jtattermusch): add test with timeout once timeouts are supported
diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs
index a4b79e3a7d..15703604ba 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs
+++ b/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs
@@ -50,38 +50,39 @@ namespace Grpc.HealthCheck.Tests
         public void SetStatus()
         {
             var impl = new HealthServiceImpl();
-            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING);
-            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, GetStatusHelper(impl, ""));
+            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.Serving);
+            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.Serving, GetStatusHelper(impl, ""));
 
-            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.NOT_SERVING);
-            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.NOT_SERVING, GetStatusHelper(impl, ""));
+            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.NotServing);
+            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.NotServing, GetStatusHelper(impl, ""));
 
-            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.UNKNOWN);
-            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.UNKNOWN, GetStatusHelper(impl, ""));
+            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.Unknown);
+            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.Unknown, GetStatusHelper(impl, ""));
 
-            impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.SERVING);
-            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, GetStatusHelper(impl, "grpc.test.TestService"));
+            impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.Serving);
+            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.Serving, GetStatusHelper(impl, "grpc.test.TestService"));
         }
 
         [Test]
         public void ClearStatus()
         {
             var impl = new HealthServiceImpl();
-            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING);
-            impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.UNKNOWN);
+            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.Serving);
+            impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.Unknown);
 
             impl.ClearStatus("");
 
-            Assert.Throws(Is.TypeOf(typeof(RpcException)).And.Property("Status").Property("StatusCode").EqualTo(StatusCode.NotFound), () => GetStatusHelper(impl, ""));
-            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.UNKNOWN, GetStatusHelper(impl, "grpc.test.TestService"));
+            var ex = Assert.Throws<RpcException>(() => GetStatusHelper(impl, ""));
+            Assert.AreEqual(StatusCode.NotFound, ex.Status.StatusCode);
+            Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.Unknown, GetStatusHelper(impl, "grpc.test.TestService"));
         }
 
         [Test]
         public void ClearAll()
         {
             var impl = new HealthServiceImpl();
-            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING);
-            impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.UNKNOWN);
+            impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.Serving);
+            impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.Unknown);
 
             impl.ClearAll();
             Assert.Throws(typeof(RpcException), () => GetStatusHelper(impl, ""));
@@ -92,7 +93,7 @@ namespace Grpc.HealthCheck.Tests
         public void NullsRejected()
         {
             var impl = new HealthServiceImpl();
-            Assert.Throws(typeof(ArgumentNullException), () => impl.SetStatus(null, HealthCheckResponse.Types.ServingStatus.SERVING));
+            Assert.Throws(typeof(ArgumentNullException), () => impl.SetStatus(null, HealthCheckResponse.Types.ServingStatus.Serving));
 
             Assert.Throws(typeof(ArgumentNullException), () => impl.ClearStatus(null));
         }
diff --git a/src/csharp/Grpc.HealthCheck.Tests/packages.config b/src/csharp/Grpc.HealthCheck.Tests/packages.config
index 8066d8fceb..2bcfec8829 100644
--- a/src/csharp/Grpc.HealthCheck.Tests/packages.config
+++ b/src/csharp/Grpc.HealthCheck.Tests/packages.config
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Google.Protobuf" version="3.0.0-beta2" targetFramework="net45" />
+  <package id="Google.Protobuf" version="3.0.0-beta3" targetFramework="net45" />
   <package id="NUnit" version="3.2.0" targetFramework="net45" />
   <package id="NUnitLite" version="3.2.0" targetFramework="net45" />
 </packages>
\ No newline at end of file
diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
index 498528aa18..2697b74f59 100644
--- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
+++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
@@ -40,7 +40,7 @@
   <ItemGroup>
     <Reference Include="Google.Protobuf, Version=3.0.0.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
+      <HintPath>..\packages\Google.Protobuf.3.0.0-beta3\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
diff --git a/src/csharp/Grpc.HealthCheck/Health.cs b/src/csharp/Grpc.HealthCheck/Health.cs
index d0d0c0b519..100ad187d7 100644
--- a/src/csharp/Grpc.HealthCheck/Health.cs
+++ b/src/csharp/Grpc.HealthCheck/Health.cs
@@ -33,9 +33,9 @@ namespace Grpc.Health.V1 {
             "Ag5HcnBjLkhlYWx0aC5WMWIGcHJvdG8z"));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Health.V1.HealthCheckRequest), global::Grpc.Health.V1.HealthCheckRequest.Parser, new[]{ "Service" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Health.V1.HealthCheckResponse), global::Grpc.Health.V1.HealthCheckResponse.Parser, new[]{ "Status" }, null, new[]{ typeof(global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus) }, null)
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Health.V1.HealthCheckRequest), global::Grpc.Health.V1.HealthCheckRequest.Parser, new[]{ "Service" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Health.V1.HealthCheckResponse), global::Grpc.Health.V1.HealthCheckResponse.Parser, new[]{ "Status" }, null, new[]{ typeof(global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus) }, null)
           }));
     }
     #endregion
@@ -75,7 +75,7 @@ namespace Grpc.Health.V1 {
     public string Service {
       get { return service_; }
       set {
-        service_ = pb::Preconditions.CheckNotNull(value, "value");
+        service_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -174,7 +174,7 @@ namespace Grpc.Health.V1 {
 
     /// <summary>Field number for the "status" field.</summary>
     public const int StatusFieldNumber = 1;
-    private global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus status_ = global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN;
+    private global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus status_ = 0;
     public global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus Status {
       get { return status_; }
       set {
@@ -199,7 +199,7 @@ namespace Grpc.Health.V1 {
 
     public override int GetHashCode() {
       int hash = 1;
-      if (Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) hash ^= Status.GetHashCode();
+      if (Status != 0) hash ^= Status.GetHashCode();
       return hash;
     }
 
@@ -208,7 +208,7 @@ namespace Grpc.Health.V1 {
     }
 
     public void WriteTo(pb::CodedOutputStream output) {
-      if (Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) {
+      if (Status != 0) {
         output.WriteRawTag(8);
         output.WriteEnum((int) Status);
       }
@@ -216,7 +216,7 @@ namespace Grpc.Health.V1 {
 
     public int CalculateSize() {
       int size = 0;
-      if (Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) {
+      if (Status != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) Status);
       }
       return size;
@@ -226,7 +226,7 @@ namespace Grpc.Health.V1 {
       if (other == null) {
         return;
       }
-      if (other.Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) {
+      if (other.Status != 0) {
         Status = other.Status;
       }
     }
@@ -251,9 +251,9 @@ namespace Grpc.Health.V1 {
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
     public static partial class Types {
       public enum ServingStatus {
-        UNKNOWN = 0,
-        SERVING = 1,
-        NOT_SERVING = 2,
+        [pbr::OriginalName("UNKNOWN")] Unknown = 0,
+        [pbr::OriginalName("SERVING")] Serving = 1,
+        [pbr::OriginalName("NOT_SERVING")] NotServing = 2,
       }
 
     }
diff --git a/src/csharp/Grpc.HealthCheck/packages.config b/src/csharp/Grpc.HealthCheck/packages.config
index 358a978ba9..a52d9e508f 100644
--- a/src/csharp/Grpc.HealthCheck/packages.config
+++ b/src/csharp/Grpc.HealthCheck/packages.config
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Google.Protobuf" version="3.0.0-beta2" targetFramework="net45" />
+  <package id="Google.Protobuf" version="3.0.0-beta3" targetFramework="net45" />
   <package id="Ix-Async" version="1.2.5" targetFramework="net45" />
 </packages>
\ No newline at end of file
diff --git a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
index 9eaf6bf7ce..39b9ae08e6 100644
--- a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
@@ -32,6 +32,7 @@
 #endregion
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -41,7 +42,9 @@ using System.Threading;
 using System.Threading.Tasks;
 using Google.Protobuf;
 using Grpc.Core;
+using Grpc.Core.Internal;
 using Grpc.Core.Logging;
+using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 using Grpc.Testing;
@@ -55,6 +58,15 @@ namespace Grpc.IntegrationTesting
     {
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<ClientRunners>();
 
+        // Profilers to use for clients.
+        static readonly BlockingCollection<BasicProfiler> profilers = new BlockingCollection<BasicProfiler>();
+
+        internal static void AddProfiler(BasicProfiler profiler)
+        {
+            GrpcPreconditions.CheckNotNull(profiler);
+            profilers.Add(profiler);
+        }
+
         /// <summary>
         /// Creates a started client runner.
         /// </summary>
@@ -83,7 +95,8 @@ namespace Grpc.IntegrationTesting
                 config.OutstandingRpcsPerChannel,
                 config.LoadParams,
                 config.PayloadConfig,
-                config.HistogramParams);
+                config.HistogramParams,
+                () => GetNextProfiler());
         }
 
         private static List<Channel> CreateChannels(int clientChannels, IEnumerable<string> serverTargets, SecurityParams securityParams)
@@ -110,9 +123,16 @@ namespace Grpc.IntegrationTesting
             }
             return result;
         }
+
+        private static BasicProfiler GetNextProfiler()
+        {
+            BasicProfiler result = null;
+            profilers.TryTake(out result);
+            return result;
+        }
     }
 
-    public class ClientRunnerImpl : IClientRunner
+    internal class ClientRunnerImpl : IClientRunner
     {
         const double SecondsToNanos = 1e9;
 
@@ -125,8 +145,9 @@ namespace Grpc.IntegrationTesting
         readonly List<Task> runnerTasks;
         readonly CancellationTokenSource stoppedCts = new CancellationTokenSource();
         readonly WallClockStopwatch wallClockStopwatch = new WallClockStopwatch();
+        readonly AtomicCounter statsResetCount = new AtomicCounter();
         
-        public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams)
+        public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams, Func<BasicProfiler> profilerFactory)
         {
             GrpcPreconditions.CheckArgument(outstandingRpcsPerChannel > 0, "outstandingRpcsPerChannel");
             GrpcPreconditions.CheckNotNull(histogramParams, "histogramParams");
@@ -142,7 +163,8 @@ namespace Grpc.IntegrationTesting
                 for (int i = 0; i < outstandingRpcsPerChannel; i++)
                 {
                     var timer = CreateTimer(loadParams, 1.0 / this.channels.Count / outstandingRpcsPerChannel);
-                    this.runnerTasks.Add(RunClientAsync(channel, timer));
+                    var optionalProfiler = profilerFactory();
+                    this.runnerTasks.Add(RunClientAsync(channel, timer, optionalProfiler));
                 }
             }
         }
@@ -152,6 +174,11 @@ namespace Grpc.IntegrationTesting
             var histogramData = histogram.GetSnapshot(reset);
             var secondsElapsed = wallClockStopwatch.GetElapsedSnapshot(reset).TotalSeconds;
 
+            if (reset)
+            {
+                statsResetCount.Increment();
+            }
+
             // TODO: populate user time and system time
             return new ClientStats
             {
@@ -175,14 +202,28 @@ namespace Grpc.IntegrationTesting
             }
         }
 
-        private void RunUnary(Channel channel, IInterarrivalTimer timer)
+        private void RunUnary(Channel channel, IInterarrivalTimer timer, BasicProfiler optionalProfiler)
         {
+            if (optionalProfiler != null)
+            {
+                Profilers.SetForCurrentThread(optionalProfiler);
+            }
+
+            bool profilerReset = false;
+
             var client = BenchmarkService.NewClient(channel);
             var request = CreateSimpleRequest();
             var stopwatch = new Stopwatch();
 
             while (!stoppedCts.Token.IsCancellationRequested)
             {
+                // after the first stats reset, also reset the profiler.
+                if (optionalProfiler != null && !profilerReset && statsResetCount.Count > 0)
+                {
+                    optionalProfiler.Reset();
+                    profilerReset = true;
+                }
+
                 stopwatch.Restart();
                 client.UnaryCall(request);
                 stopwatch.Stop();
@@ -268,29 +309,29 @@ namespace Grpc.IntegrationTesting
             }
         }
 
-        private Task RunClientAsync(Channel channel, IInterarrivalTimer timer)
+        private Task RunClientAsync(Channel channel, IInterarrivalTimer timer, BasicProfiler optionalProfiler)
         {
             if (payloadConfig.PayloadCase == PayloadConfig.PayloadOneofCase.BytebufParams)
             {
-                GrpcPreconditions.CheckArgument(clientType == ClientType.ASYNC_CLIENT, "Generic client only supports async API");
-                GrpcPreconditions.CheckArgument(rpcType == RpcType.STREAMING, "Generic client only supports streaming calls");
+                GrpcPreconditions.CheckArgument(clientType == ClientType.AsyncClient, "Generic client only supports async API");
+                GrpcPreconditions.CheckArgument(rpcType == RpcType.Streaming, "Generic client only supports streaming calls");
                 return RunGenericStreamingAsync(channel, timer);
             }
 
             GrpcPreconditions.CheckNotNull(payloadConfig.SimpleParams);
-            if (clientType == ClientType.SYNC_CLIENT)
+            if (clientType == ClientType.SyncClient)
             {
-                GrpcPreconditions.CheckArgument(rpcType == RpcType.UNARY, "Sync client can only be used for Unary calls in C#");
+                GrpcPreconditions.CheckArgument(rpcType == RpcType.Unary, "Sync client can only be used for Unary calls in C#");
                 // create a dedicated thread for the synchronous client
-                return Task.Factory.StartNew(() => RunUnary(channel, timer), TaskCreationOptions.LongRunning);
+                return Task.Factory.StartNew(() => RunUnary(channel, timer, optionalProfiler), TaskCreationOptions.LongRunning);
             }
-            else if (clientType == ClientType.ASYNC_CLIENT)
+            else if (clientType == ClientType.AsyncClient)
             {
                 switch (rpcType)
                 {
-                    case RpcType.UNARY:
+                    case RpcType.Unary:
                         return RunUnaryAsync(channel, timer);
-                    case RpcType.STREAMING:
+                    case RpcType.Streaming:
                         return RunStreamingPingPongAsync(channel, timer);
                 }
             }
diff --git a/src/csharp/Grpc.IntegrationTesting/Control.cs b/src/csharp/Grpc.IntegrationTesting/Control.cs
index 3fa8d43f38..412f800ff9 100644
--- a/src/csharp/Grpc.IntegrationTesting/Control.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Control.cs
@@ -85,25 +85,25 @@ namespace Grpc.Testing {
             "RUFNSU5HEAFiBnByb3RvMw=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { global::Grpc.Testing.PayloadsReflection.Descriptor, global::Grpc.Testing.StatsReflection.Descriptor, },
-          new pbr::GeneratedCodeInfo(new[] {typeof(global::Grpc.Testing.ClientType), typeof(global::Grpc.Testing.ServerType), typeof(global::Grpc.Testing.RpcType), }, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.PoissonParams), global::Grpc.Testing.PoissonParams.Parser, new[]{ "OfferedLoad" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClosedLoopParams), global::Grpc.Testing.ClosedLoopParams.Parser, null, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.LoadParams), global::Grpc.Testing.LoadParams.Parser, new[]{ "ClosedLoop", "Poisson" }, new[]{ "Load" }, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.SecurityParams), global::Grpc.Testing.SecurityParams.Parser, new[]{ "UseTestCa", "ServerHostOverride" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientConfig), global::Grpc.Testing.ClientConfig.Parser, new[]{ "ServerTargets", "ClientType", "SecurityParams", "OutstandingRpcsPerChannel", "ClientChannels", "AsyncClientThreads", "RpcType", "LoadParams", "PayloadConfig", "HistogramParams", "CoreList", "CoreLimit", "OtherClientApi" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientStatus), global::Grpc.Testing.ClientStatus.Parser, new[]{ "Stats" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Mark), global::Grpc.Testing.Mark.Parser, new[]{ "Reset" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientArgs), global::Grpc.Testing.ClientArgs.Parser, new[]{ "Setup", "Mark" }, new[]{ "Argtype" }, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerConfig), global::Grpc.Testing.ServerConfig.Parser, new[]{ "ServerType", "SecurityParams", "Port", "AsyncServerThreads", "CoreLimit", "PayloadConfig", "CoreList", "OtherServerApi" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerArgs), global::Grpc.Testing.ServerArgs.Parser, new[]{ "Setup", "Mark" }, new[]{ "Argtype" }, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerStatus), global::Grpc.Testing.ServerStatus.Parser, new[]{ "Stats", "Port", "Cores" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.CoreRequest), global::Grpc.Testing.CoreRequest.Parser, null, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.CoreResponse), global::Grpc.Testing.CoreResponse.Parser, new[]{ "Cores" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Void), global::Grpc.Testing.Void.Parser, null, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Scenario), global::Grpc.Testing.Scenario.Parser, new[]{ "Name", "ClientConfig", "NumClients", "ServerConfig", "NumServers", "WarmupSeconds", "BenchmarkSeconds", "SpawnLocalWorkerCount" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Scenarios), global::Grpc.Testing.Scenarios.Parser, new[]{ "Scenarios_" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ScenarioResultSummary), global::Grpc.Testing.ScenarioResultSummary.Parser, new[]{ "Qps", "QpsPerServerCore", "ServerSystemTime", "ServerUserTime", "ClientSystemTime", "ClientUserTime", "Latency50", "Latency90", "Latency95", "Latency99", "Latency999" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ScenarioResult), global::Grpc.Testing.ScenarioResult.Parser, new[]{ "Scenario", "Latencies", "ClientStats", "ServerStats", "ServerCores", "Summary" }, null, null, null)
+          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Grpc.Testing.ClientType), typeof(global::Grpc.Testing.ServerType), typeof(global::Grpc.Testing.RpcType), }, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.PoissonParams), global::Grpc.Testing.PoissonParams.Parser, new[]{ "OfferedLoad" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ClosedLoopParams), global::Grpc.Testing.ClosedLoopParams.Parser, null, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.LoadParams), global::Grpc.Testing.LoadParams.Parser, new[]{ "ClosedLoop", "Poisson" }, new[]{ "Load" }, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SecurityParams), global::Grpc.Testing.SecurityParams.Parser, new[]{ "UseTestCa", "ServerHostOverride" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ClientConfig), global::Grpc.Testing.ClientConfig.Parser, new[]{ "ServerTargets", "ClientType", "SecurityParams", "OutstandingRpcsPerChannel", "ClientChannels", "AsyncClientThreads", "RpcType", "LoadParams", "PayloadConfig", "HistogramParams", "CoreList", "CoreLimit", "OtherClientApi" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ClientStatus), global::Grpc.Testing.ClientStatus.Parser, new[]{ "Stats" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Mark), global::Grpc.Testing.Mark.Parser, new[]{ "Reset" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ClientArgs), global::Grpc.Testing.ClientArgs.Parser, new[]{ "Setup", "Mark" }, new[]{ "Argtype" }, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ServerConfig), global::Grpc.Testing.ServerConfig.Parser, new[]{ "ServerType", "SecurityParams", "Port", "AsyncServerThreads", "CoreLimit", "PayloadConfig", "CoreList", "OtherServerApi" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ServerArgs), global::Grpc.Testing.ServerArgs.Parser, new[]{ "Setup", "Mark" }, new[]{ "Argtype" }, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ServerStatus), global::Grpc.Testing.ServerStatus.Parser, new[]{ "Stats", "Port", "Cores" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.CoreRequest), global::Grpc.Testing.CoreRequest.Parser, null, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.CoreResponse), global::Grpc.Testing.CoreResponse.Parser, new[]{ "Cores" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Void), global::Grpc.Testing.Void.Parser, null, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Scenario), global::Grpc.Testing.Scenario.Parser, new[]{ "Name", "ClientConfig", "NumClients", "ServerConfig", "NumServers", "WarmupSeconds", "BenchmarkSeconds", "SpawnLocalWorkerCount" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Scenarios), global::Grpc.Testing.Scenarios.Parser, new[]{ "Scenarios_" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ScenarioResultSummary), global::Grpc.Testing.ScenarioResultSummary.Parser, new[]{ "Qps", "QpsPerServerCore", "ServerSystemTime", "ServerUserTime", "ClientSystemTime", "ClientUserTime", "Latency50", "Latency90", "Latency95", "Latency99", "Latency999" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ScenarioResult), global::Grpc.Testing.ScenarioResult.Parser, new[]{ "Scenario", "Latencies", "ClientStats", "ServerStats", "ServerCores", "Summary" }, null, null, null)
           }));
     }
     #endregion
@@ -115,27 +115,27 @@ namespace Grpc.Testing {
     ///  Many languages support a basic distinction between using
     ///  sync or async client, and this allows the specification
     /// </summary>
-    SYNC_CLIENT = 0,
-    ASYNC_CLIENT = 1,
+    [pbr::OriginalName("SYNC_CLIENT")] SyncClient = 0,
+    [pbr::OriginalName("ASYNC_CLIENT")] AsyncClient = 1,
     /// <summary>
     ///  used for some language-specific variants
     /// </summary>
-    OTHER_CLIENT = 2,
+    [pbr::OriginalName("OTHER_CLIENT")] OtherClient = 2,
   }
 
   public enum ServerType {
-    SYNC_SERVER = 0,
-    ASYNC_SERVER = 1,
-    ASYNC_GENERIC_SERVER = 2,
+    [pbr::OriginalName("SYNC_SERVER")] SyncServer = 0,
+    [pbr::OriginalName("ASYNC_SERVER")] AsyncServer = 1,
+    [pbr::OriginalName("ASYNC_GENERIC_SERVER")] AsyncGenericServer = 2,
     /// <summary>
     ///  used for some language-specific variants
     /// </summary>
-    OTHER_SERVER = 3,
+    [pbr::OriginalName("OTHER_SERVER")] OtherServer = 3,
   }
 
   public enum RpcType {
-    UNARY = 0,
-    STREAMING = 1,
+    [pbr::OriginalName("UNARY")] Unary = 0,
+    [pbr::OriginalName("STREAMING")] Streaming = 1,
   }
 
   #endregion
@@ -547,7 +547,7 @@ namespace Grpc.Testing {
     public string ServerHostOverride {
       get { return serverHostOverride_; }
       set {
-        serverHostOverride_ = pb::Preconditions.CheckNotNull(value, "value");
+        serverHostOverride_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -686,7 +686,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "client_type" field.</summary>
     public const int ClientTypeFieldNumber = 2;
-    private global::Grpc.Testing.ClientType clientType_ = global::Grpc.Testing.ClientType.SYNC_CLIENT;
+    private global::Grpc.Testing.ClientType clientType_ = 0;
     public global::Grpc.Testing.ClientType ClientType {
       get { return clientType_; }
       set {
@@ -747,7 +747,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "rpc_type" field.</summary>
     public const int RpcTypeFieldNumber = 8;
-    private global::Grpc.Testing.RpcType rpcType_ = global::Grpc.Testing.RpcType.UNARY;
+    private global::Grpc.Testing.RpcType rpcType_ = 0;
     public global::Grpc.Testing.RpcType RpcType {
       get { return rpcType_; }
       set {
@@ -819,7 +819,7 @@ namespace Grpc.Testing {
     public string OtherClientApi {
       get { return otherClientApi_; }
       set {
-        otherClientApi_ = pb::Preconditions.CheckNotNull(value, "value");
+        otherClientApi_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -853,12 +853,12 @@ namespace Grpc.Testing {
     public override int GetHashCode() {
       int hash = 1;
       hash ^= serverTargets_.GetHashCode();
-      if (ClientType != global::Grpc.Testing.ClientType.SYNC_CLIENT) hash ^= ClientType.GetHashCode();
+      if (ClientType != 0) hash ^= ClientType.GetHashCode();
       if (securityParams_ != null) hash ^= SecurityParams.GetHashCode();
       if (OutstandingRpcsPerChannel != 0) hash ^= OutstandingRpcsPerChannel.GetHashCode();
       if (ClientChannels != 0) hash ^= ClientChannels.GetHashCode();
       if (AsyncClientThreads != 0) hash ^= AsyncClientThreads.GetHashCode();
-      if (RpcType != global::Grpc.Testing.RpcType.UNARY) hash ^= RpcType.GetHashCode();
+      if (RpcType != 0) hash ^= RpcType.GetHashCode();
       if (loadParams_ != null) hash ^= LoadParams.GetHashCode();
       if (payloadConfig_ != null) hash ^= PayloadConfig.GetHashCode();
       if (histogramParams_ != null) hash ^= HistogramParams.GetHashCode();
@@ -874,7 +874,7 @@ namespace Grpc.Testing {
 
     public void WriteTo(pb::CodedOutputStream output) {
       serverTargets_.WriteTo(output, _repeated_serverTargets_codec);
-      if (ClientType != global::Grpc.Testing.ClientType.SYNC_CLIENT) {
+      if (ClientType != 0) {
         output.WriteRawTag(16);
         output.WriteEnum((int) ClientType);
       }
@@ -894,7 +894,7 @@ namespace Grpc.Testing {
         output.WriteRawTag(56);
         output.WriteInt32(AsyncClientThreads);
       }
-      if (RpcType != global::Grpc.Testing.RpcType.UNARY) {
+      if (RpcType != 0) {
         output.WriteRawTag(64);
         output.WriteEnum((int) RpcType);
       }
@@ -924,7 +924,7 @@ namespace Grpc.Testing {
     public int CalculateSize() {
       int size = 0;
       size += serverTargets_.CalculateSize(_repeated_serverTargets_codec);
-      if (ClientType != global::Grpc.Testing.ClientType.SYNC_CLIENT) {
+      if (ClientType != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ClientType);
       }
       if (securityParams_ != null) {
@@ -939,7 +939,7 @@ namespace Grpc.Testing {
       if (AsyncClientThreads != 0) {
         size += 1 + pb::CodedOutputStream.ComputeInt32Size(AsyncClientThreads);
       }
-      if (RpcType != global::Grpc.Testing.RpcType.UNARY) {
+      if (RpcType != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) RpcType);
       }
       if (loadParams_ != null) {
@@ -966,7 +966,7 @@ namespace Grpc.Testing {
         return;
       }
       serverTargets_.Add(other.serverTargets_);
-      if (other.ClientType != global::Grpc.Testing.ClientType.SYNC_CLIENT) {
+      if (other.ClientType != 0) {
         ClientType = other.ClientType;
       }
       if (other.securityParams_ != null) {
@@ -984,7 +984,7 @@ namespace Grpc.Testing {
       if (other.AsyncClientThreads != 0) {
         AsyncClientThreads = other.AsyncClientThreads;
       }
-      if (other.RpcType != global::Grpc.Testing.RpcType.UNARY) {
+      if (other.RpcType != 0) {
         RpcType = other.RpcType;
       }
       if (other.loadParams_ != null) {
@@ -1515,7 +1515,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "server_type" field.</summary>
     public const int ServerTypeFieldNumber = 1;
-    private global::Grpc.Testing.ServerType serverType_ = global::Grpc.Testing.ServerType.SYNC_SERVER;
+    private global::Grpc.Testing.ServerType serverType_ = 0;
     public global::Grpc.Testing.ServerType ServerType {
       get { return serverType_; }
       set {
@@ -1606,7 +1606,7 @@ namespace Grpc.Testing {
     public string OtherServerApi {
       get { return otherServerApi_; }
       set {
-        otherServerApi_ = pb::Preconditions.CheckNotNull(value, "value");
+        otherServerApi_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -1634,7 +1634,7 @@ namespace Grpc.Testing {
 
     public override int GetHashCode() {
       int hash = 1;
-      if (ServerType != global::Grpc.Testing.ServerType.SYNC_SERVER) hash ^= ServerType.GetHashCode();
+      if (ServerType != 0) hash ^= ServerType.GetHashCode();
       if (securityParams_ != null) hash ^= SecurityParams.GetHashCode();
       if (Port != 0) hash ^= Port.GetHashCode();
       if (AsyncServerThreads != 0) hash ^= AsyncServerThreads.GetHashCode();
@@ -1650,7 +1650,7 @@ namespace Grpc.Testing {
     }
 
     public void WriteTo(pb::CodedOutputStream output) {
-      if (ServerType != global::Grpc.Testing.ServerType.SYNC_SERVER) {
+      if (ServerType != 0) {
         output.WriteRawTag(8);
         output.WriteEnum((int) ServerType);
       }
@@ -1683,7 +1683,7 @@ namespace Grpc.Testing {
 
     public int CalculateSize() {
       int size = 0;
-      if (ServerType != global::Grpc.Testing.ServerType.SYNC_SERVER) {
+      if (ServerType != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ServerType);
       }
       if (securityParams_ != null) {
@@ -1712,7 +1712,7 @@ namespace Grpc.Testing {
       if (other == null) {
         return;
       }
-      if (other.ServerType != global::Grpc.Testing.ServerType.SYNC_SERVER) {
+      if (other.ServerType != 0) {
         ServerType = other.ServerType;
       }
       if (other.securityParams_ != null) {
@@ -2436,7 +2436,7 @@ namespace Grpc.Testing {
     public string Name {
       get { return name_; }
       set {
-        name_ = pb::Preconditions.CheckNotNull(value, "value");
+        name_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
diff --git a/src/csharp/Grpc.IntegrationTesting/Empty.cs b/src/csharp/Grpc.IntegrationTesting/Empty.cs
index 4323c5a09f..cf1c23fb0f 100644
--- a/src/csharp/Grpc.IntegrationTesting/Empty.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Empty.cs
@@ -27,8 +27,8 @@ namespace Grpc.Testing {
             "c3RpbmciBwoFRW1wdHliBnByb3RvMw=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Empty), global::Grpc.Testing.Empty.Parser, null, null, null, null)
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Empty), global::Grpc.Testing.Empty.Parser, null, null, null, null)
           }));
     }
     #endregion
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
index 9685cf1837..0089049408 100644
--- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
@@ -61,7 +61,7 @@
       <HintPath>..\packages\Google.Apis.Core.1.11.1\lib\net45\Google.Apis.Core.dll</HintPath>
     </Reference>
     <Reference Include="Google.Protobuf">
-      <HintPath>..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
+      <HintPath>..\packages\Google.Protobuf.3.0.0-beta3\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll</HintPath>
     </Reference>
     <Reference Include="Newtonsoft.Json">
       <HintPath>..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
index cff8508631..aea40afee2 100644
--- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
+++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
@@ -230,13 +230,13 @@ namespace Grpc.IntegrationTesting
             Console.WriteLine("running large_unary");
             var request = new SimpleRequest
             {
-                ResponseType = PayloadType.COMPRESSABLE,
+                ResponseType = PayloadType.Compressable,
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828)
             };
             var response = client.UnaryCall(request);
 
-            Assert.AreEqual(PayloadType.COMPRESSABLE, response.Payload.Type);
+            Assert.AreEqual(PayloadType.Compressable, response.Payload.Type);
             Assert.AreEqual(314159, response.Payload.Body.Length);
             Console.WriteLine("Passed!");
         }
@@ -265,7 +265,7 @@ namespace Grpc.IntegrationTesting
 
             var request = new StreamingOutputCallRequest
             {
-                ResponseType = PayloadType.COMPRESSABLE,
+                ResponseType = PayloadType.Compressable,
                 ResponseParameters = { bodySizes.ConvertAll((size) => new ResponseParameters { Size = size }) }
             };
 
@@ -274,7 +274,7 @@ namespace Grpc.IntegrationTesting
                 var responseList = await call.ResponseStream.ToListAsync();
                 foreach (var res in responseList)
                 {
-                    Assert.AreEqual(PayloadType.COMPRESSABLE, res.Payload.Type);
+                    Assert.AreEqual(PayloadType.Compressable, res.Payload.Type);
                 }
                 CollectionAssert.AreEqual(bodySizes, responseList.ConvertAll((item) => item.Payload.Body.Length));
             }
@@ -289,46 +289,46 @@ namespace Grpc.IntegrationTesting
             {
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 31415 } },
                     Payload = CreateZerosPayload(27182)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.COMPRESSABLE, call.ResponseStream.Current.Payload.Type);
+                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(31415, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 9 } },
                     Payload = CreateZerosPayload(8)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.COMPRESSABLE, call.ResponseStream.Current.Payload.Type);
+                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(9, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 2653 } },
                     Payload = CreateZerosPayload(1828)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.COMPRESSABLE, call.ResponseStream.Current.Payload.Type);
+                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(2653, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 58979 } },
                     Payload = CreateZerosPayload(45904)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.COMPRESSABLE, call.ResponseStream.Current.Payload.Type);
+                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(58979, call.ResponseStream.Current.Payload.Body.Length);
 
                 await call.RequestStream.CompleteAsync();
@@ -357,7 +357,7 @@ namespace Grpc.IntegrationTesting
 
             var request = new SimpleRequest
             {
-                ResponseType = PayloadType.COMPRESSABLE,
+                ResponseType = PayloadType.Compressable,
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828),
                 FillUsername = true,
@@ -367,7 +367,7 @@ namespace Grpc.IntegrationTesting
             // not setting credentials here because they were set on channel already
             var response = client.UnaryCall(request);
 
-            Assert.AreEqual(PayloadType.COMPRESSABLE, response.Payload.Type);
+            Assert.AreEqual(PayloadType.Compressable, response.Payload.Type);
             Assert.AreEqual(314159, response.Payload.Body.Length);
             Assert.False(string.IsNullOrEmpty(response.OauthScope));
             Assert.True(oauthScope.Contains(response.OauthScope));
@@ -381,7 +381,7 @@ namespace Grpc.IntegrationTesting
            
             var request = new SimpleRequest
             {
-                ResponseType = PayloadType.COMPRESSABLE,
+                ResponseType = PayloadType.Compressable,
                 ResponseSize = 314159,
                 Payload = CreateZerosPayload(271828),
                 FillUsername = true,
@@ -390,7 +390,7 @@ namespace Grpc.IntegrationTesting
             // not setting credentials here because they were set on channel already
             var response = client.UnaryCall(request);
 
-            Assert.AreEqual(PayloadType.COMPRESSABLE, response.Payload.Type);
+            Assert.AreEqual(PayloadType.Compressable, response.Payload.Type);
             Assert.AreEqual(314159, response.Payload.Body.Length);
             Assert.AreEqual(GetEmailFromServiceAccountFile(), response.Username);
             Console.WriteLine("Passed!");
@@ -460,19 +460,27 @@ namespace Grpc.IntegrationTesting
             {
                 await call.RequestStream.WriteAsync(new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 31415 } },
                     Payload = CreateZerosPayload(27182)
                 });
 
                 Assert.IsTrue(await call.ResponseStream.MoveNext());
-                Assert.AreEqual(PayloadType.COMPRESSABLE, call.ResponseStream.Current.Payload.Type);
+                Assert.AreEqual(PayloadType.Compressable, call.ResponseStream.Current.Payload.Type);
                 Assert.AreEqual(31415, call.ResponseStream.Current.Payload.Body.Length);
 
                 cts.Cancel();
 
-                var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.MoveNext());
-                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await call.ResponseStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (RpcException ex)
+                {
+                    Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+                }
             }
             Console.WriteLine("Passed!");
         }
@@ -497,9 +505,16 @@ namespace Grpc.IntegrationTesting
                     // Deadline was reached before write has started. Eat the exception and continue.
                 }
 
-                var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.MoveNext());
-                // We can't guarantee the status code always DeadlineExceeded. See issue #2685.
-                Assert.Contains(ex.Status.StatusCode, new[] { StatusCode.DeadlineExceeded, StatusCode.Internal });
+                try
+                {
+                    await call.ResponseStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (RpcException ex)
+                {
+                    // We can't guarantee the status code always DeadlineExceeded. See issue #2685.
+                    Assert.Contains(ex.Status.StatusCode, new[] { StatusCode.DeadlineExceeded, StatusCode.Internal });
+                }
             }
             Console.WriteLine("Passed!");
         }
@@ -511,7 +526,7 @@ namespace Grpc.IntegrationTesting
                 // step 1: test unary call
                 var request = new SimpleRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseSize = 314159,
                     Payload = CreateZerosPayload(271828)
                 };
@@ -530,7 +545,7 @@ namespace Grpc.IntegrationTesting
                 // step 2: test full duplex call
                 var request = new StreamingOutputCallRequest
                 {
-                    ResponseType = PayloadType.COMPRESSABLE,
+                    ResponseType = PayloadType.Compressable,
                     ResponseParameters = { new ResponseParameters { Size = 31415 } },
                     Payload = CreateZerosPayload(27182)
                 };
@@ -577,9 +592,17 @@ namespace Grpc.IntegrationTesting
                 await call.RequestStream.WriteAsync(request);
                 await call.RequestStream.CompleteAsync();
 
-                var e = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.ToListAsync());
-                Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
-                Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await call.ResponseStream.ToListAsync();
+                    Assert.Fail();
+                }
+                catch (RpcException e)
+                {
+                    Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
+                    Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+                }
             }
 
             Console.WriteLine("Passed!");
diff --git a/src/csharp/Grpc.IntegrationTesting/Messages.cs b/src/csharp/Grpc.IntegrationTesting/Messages.cs
index fcff475941..d42501aa5b 100644
--- a/src/csharp/Grpc.IntegrationTesting/Messages.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Messages.cs
@@ -55,18 +55,18 @@ namespace Grpc.Testing {
             "TkUQABIICgRHWklQEAESCwoHREVGTEFURRACYgZwcm90bzM="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(new[] {typeof(global::Grpc.Testing.PayloadType), typeof(global::Grpc.Testing.CompressionType), }, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Payload), global::Grpc.Testing.Payload.Parser, new[]{ "Type", "Body" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.EchoStatus), global::Grpc.Testing.EchoStatus.Parser, new[]{ "Code", "Message" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.SimpleRequest), global::Grpc.Testing.SimpleRequest.Parser, new[]{ "ResponseType", "ResponseSize", "Payload", "FillUsername", "FillOauthScope", "ResponseCompression", "ResponseStatus" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.SimpleResponse), global::Grpc.Testing.SimpleResponse.Parser, new[]{ "Payload", "Username", "OauthScope" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.StreamingInputCallRequest), global::Grpc.Testing.StreamingInputCallRequest.Parser, new[]{ "Payload" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.StreamingInputCallResponse), global::Grpc.Testing.StreamingInputCallResponse.Parser, new[]{ "AggregatedPayloadSize" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ResponseParameters), global::Grpc.Testing.ResponseParameters.Parser, new[]{ "Size", "IntervalUs" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.StreamingOutputCallRequest), global::Grpc.Testing.StreamingOutputCallRequest.Parser, new[]{ "ResponseType", "ResponseParameters", "Payload", "ResponseCompression", "ResponseStatus" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.StreamingOutputCallResponse), global::Grpc.Testing.StreamingOutputCallResponse.Parser, new[]{ "Payload" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ReconnectParams), global::Grpc.Testing.ReconnectParams.Parser, new[]{ "MaxReconnectBackoffMs" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ReconnectInfo), global::Grpc.Testing.ReconnectInfo.Parser, new[]{ "Passed", "BackoffMs" }, null, null, null)
+          new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Grpc.Testing.PayloadType), typeof(global::Grpc.Testing.CompressionType), }, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.Payload), global::Grpc.Testing.Payload.Parser, new[]{ "Type", "Body" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoStatus), global::Grpc.Testing.EchoStatus.Parser, new[]{ "Code", "Message" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SimpleRequest), global::Grpc.Testing.SimpleRequest.Parser, new[]{ "ResponseType", "ResponseSize", "Payload", "FillUsername", "FillOauthScope", "ResponseCompression", "ResponseStatus" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SimpleResponse), global::Grpc.Testing.SimpleResponse.Parser, new[]{ "Payload", "Username", "OauthScope" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingInputCallRequest), global::Grpc.Testing.StreamingInputCallRequest.Parser, new[]{ "Payload" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingInputCallResponse), global::Grpc.Testing.StreamingInputCallResponse.Parser, new[]{ "AggregatedPayloadSize" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParameters), global::Grpc.Testing.ResponseParameters.Parser, new[]{ "Size", "IntervalUs" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingOutputCallRequest), global::Grpc.Testing.StreamingOutputCallRequest.Parser, new[]{ "ResponseType", "ResponseParameters", "Payload", "ResponseCompression", "ResponseStatus" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.StreamingOutputCallResponse), global::Grpc.Testing.StreamingOutputCallResponse.Parser, new[]{ "Payload" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ReconnectParams), global::Grpc.Testing.ReconnectParams.Parser, new[]{ "MaxReconnectBackoffMs" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ReconnectInfo), global::Grpc.Testing.ReconnectInfo.Parser, new[]{ "Passed", "BackoffMs" }, null, null, null)
           }));
     }
     #endregion
@@ -80,15 +80,15 @@ namespace Grpc.Testing {
     /// <summary>
     ///  Compressable text format.
     /// </summary>
-    COMPRESSABLE = 0,
+    [pbr::OriginalName("COMPRESSABLE")] Compressable = 0,
     /// <summary>
     ///  Uncompressable binary format.
     /// </summary>
-    UNCOMPRESSABLE = 1,
+    [pbr::OriginalName("UNCOMPRESSABLE")] Uncompressable = 1,
     /// <summary>
     ///  Randomly chosen from all other formats defined in this enum.
     /// </summary>
-    RANDOM = 2,
+    [pbr::OriginalName("RANDOM")] Random = 2,
   }
 
   /// <summary>
@@ -98,9 +98,9 @@ namespace Grpc.Testing {
     /// <summary>
     ///  No compression
     /// </summary>
-    NONE = 0,
-    GZIP = 1,
-    DEFLATE = 2,
+    [pbr::OriginalName("NONE")] None = 0,
+    [pbr::OriginalName("GZIP")] Gzip = 1,
+    [pbr::OriginalName("DEFLATE")] Deflate = 2,
   }
 
   #endregion
@@ -139,7 +139,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "type" field.</summary>
     public const int TypeFieldNumber = 1;
-    private global::Grpc.Testing.PayloadType type_ = global::Grpc.Testing.PayloadType.COMPRESSABLE;
+    private global::Grpc.Testing.PayloadType type_ = 0;
     /// <summary>
     ///  The type of data in body.
     /// </summary>
@@ -159,7 +159,7 @@ namespace Grpc.Testing {
     public pb::ByteString Body {
       get { return body_; }
       set {
-        body_ = pb::Preconditions.CheckNotNull(value, "value");
+        body_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -181,7 +181,7 @@ namespace Grpc.Testing {
 
     public override int GetHashCode() {
       int hash = 1;
-      if (Type != global::Grpc.Testing.PayloadType.COMPRESSABLE) hash ^= Type.GetHashCode();
+      if (Type != 0) hash ^= Type.GetHashCode();
       if (Body.Length != 0) hash ^= Body.GetHashCode();
       return hash;
     }
@@ -191,7 +191,7 @@ namespace Grpc.Testing {
     }
 
     public void WriteTo(pb::CodedOutputStream output) {
-      if (Type != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (Type != 0) {
         output.WriteRawTag(8);
         output.WriteEnum((int) Type);
       }
@@ -203,7 +203,7 @@ namespace Grpc.Testing {
 
     public int CalculateSize() {
       int size = 0;
-      if (Type != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (Type != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) Type);
       }
       if (Body.Length != 0) {
@@ -216,7 +216,7 @@ namespace Grpc.Testing {
       if (other == null) {
         return;
       }
-      if (other.Type != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (other.Type != 0) {
         Type = other.Type;
       }
       if (other.Body.Length != 0) {
@@ -293,7 +293,7 @@ namespace Grpc.Testing {
     public string Message {
       get { return message_; }
       set {
-        message_ = pb::Preconditions.CheckNotNull(value, "value");
+        message_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -417,7 +417,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "response_type" field.</summary>
     public const int ResponseTypeFieldNumber = 1;
-    private global::Grpc.Testing.PayloadType responseType_ = global::Grpc.Testing.PayloadType.COMPRESSABLE;
+    private global::Grpc.Testing.PayloadType responseType_ = 0;
     /// <summary>
     ///  Desired payload type in the response from the server.
     ///  If response_type is RANDOM, server randomly chooses one from other formats.
@@ -484,7 +484,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "response_compression" field.</summary>
     public const int ResponseCompressionFieldNumber = 6;
-    private global::Grpc.Testing.CompressionType responseCompression_ = global::Grpc.Testing.CompressionType.NONE;
+    private global::Grpc.Testing.CompressionType responseCompression_ = 0;
     /// <summary>
     ///  Compression algorithm to be used by the server for the response (stream)
     /// </summary>
@@ -531,12 +531,12 @@ namespace Grpc.Testing {
 
     public override int GetHashCode() {
       int hash = 1;
-      if (ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) hash ^= ResponseType.GetHashCode();
+      if (ResponseType != 0) hash ^= ResponseType.GetHashCode();
       if (ResponseSize != 0) hash ^= ResponseSize.GetHashCode();
       if (payload_ != null) hash ^= Payload.GetHashCode();
       if (FillUsername != false) hash ^= FillUsername.GetHashCode();
       if (FillOauthScope != false) hash ^= FillOauthScope.GetHashCode();
-      if (ResponseCompression != global::Grpc.Testing.CompressionType.NONE) hash ^= ResponseCompression.GetHashCode();
+      if (ResponseCompression != 0) hash ^= ResponseCompression.GetHashCode();
       if (responseStatus_ != null) hash ^= ResponseStatus.GetHashCode();
       return hash;
     }
@@ -546,7 +546,7 @@ namespace Grpc.Testing {
     }
 
     public void WriteTo(pb::CodedOutputStream output) {
-      if (ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (ResponseType != 0) {
         output.WriteRawTag(8);
         output.WriteEnum((int) ResponseType);
       }
@@ -566,7 +566,7 @@ namespace Grpc.Testing {
         output.WriteRawTag(40);
         output.WriteBool(FillOauthScope);
       }
-      if (ResponseCompression != global::Grpc.Testing.CompressionType.NONE) {
+      if (ResponseCompression != 0) {
         output.WriteRawTag(48);
         output.WriteEnum((int) ResponseCompression);
       }
@@ -578,7 +578,7 @@ namespace Grpc.Testing {
 
     public int CalculateSize() {
       int size = 0;
-      if (ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (ResponseType != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ResponseType);
       }
       if (ResponseSize != 0) {
@@ -593,7 +593,7 @@ namespace Grpc.Testing {
       if (FillOauthScope != false) {
         size += 1 + 1;
       }
-      if (ResponseCompression != global::Grpc.Testing.CompressionType.NONE) {
+      if (ResponseCompression != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ResponseCompression);
       }
       if (responseStatus_ != null) {
@@ -606,7 +606,7 @@ namespace Grpc.Testing {
       if (other == null) {
         return;
       }
-      if (other.ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (other.ResponseType != 0) {
         ResponseType = other.ResponseType;
       }
       if (other.ResponseSize != 0) {
@@ -624,7 +624,7 @@ namespace Grpc.Testing {
       if (other.FillOauthScope != false) {
         FillOauthScope = other.FillOauthScope;
       }
-      if (other.ResponseCompression != global::Grpc.Testing.CompressionType.NONE) {
+      if (other.ResponseCompression != 0) {
         ResponseCompression = other.ResponseCompression;
       }
       if (other.responseStatus_ != null) {
@@ -737,7 +737,7 @@ namespace Grpc.Testing {
     public string Username {
       get { return username_; }
       set {
-        username_ = pb::Preconditions.CheckNotNull(value, "value");
+        username_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -750,7 +750,7 @@ namespace Grpc.Testing {
     public string OauthScope {
       get { return oauthScope_; }
       set {
-        oauthScope_ = pb::Preconditions.CheckNotNull(value, "value");
+        oauthScope_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -1259,7 +1259,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "response_type" field.</summary>
     public const int ResponseTypeFieldNumber = 1;
-    private global::Grpc.Testing.PayloadType responseType_ = global::Grpc.Testing.PayloadType.COMPRESSABLE;
+    private global::Grpc.Testing.PayloadType responseType_ = 0;
     /// <summary>
     ///  Desired payload type in the response from the server.
     ///  If response_type is RANDOM, the payload from each response in the stream
@@ -1300,7 +1300,7 @@ namespace Grpc.Testing {
 
     /// <summary>Field number for the "response_compression" field.</summary>
     public const int ResponseCompressionFieldNumber = 6;
-    private global::Grpc.Testing.CompressionType responseCompression_ = global::Grpc.Testing.CompressionType.NONE;
+    private global::Grpc.Testing.CompressionType responseCompression_ = 0;
     /// <summary>
     ///  Compression algorithm to be used by the server for the response (stream)
     /// </summary>
@@ -1345,10 +1345,10 @@ namespace Grpc.Testing {
 
     public override int GetHashCode() {
       int hash = 1;
-      if (ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) hash ^= ResponseType.GetHashCode();
+      if (ResponseType != 0) hash ^= ResponseType.GetHashCode();
       hash ^= responseParameters_.GetHashCode();
       if (payload_ != null) hash ^= Payload.GetHashCode();
-      if (ResponseCompression != global::Grpc.Testing.CompressionType.NONE) hash ^= ResponseCompression.GetHashCode();
+      if (ResponseCompression != 0) hash ^= ResponseCompression.GetHashCode();
       if (responseStatus_ != null) hash ^= ResponseStatus.GetHashCode();
       return hash;
     }
@@ -1358,7 +1358,7 @@ namespace Grpc.Testing {
     }
 
     public void WriteTo(pb::CodedOutputStream output) {
-      if (ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (ResponseType != 0) {
         output.WriteRawTag(8);
         output.WriteEnum((int) ResponseType);
       }
@@ -1367,7 +1367,7 @@ namespace Grpc.Testing {
         output.WriteRawTag(26);
         output.WriteMessage(Payload);
       }
-      if (ResponseCompression != global::Grpc.Testing.CompressionType.NONE) {
+      if (ResponseCompression != 0) {
         output.WriteRawTag(48);
         output.WriteEnum((int) ResponseCompression);
       }
@@ -1379,14 +1379,14 @@ namespace Grpc.Testing {
 
     public int CalculateSize() {
       int size = 0;
-      if (ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (ResponseType != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ResponseType);
       }
       size += responseParameters_.CalculateSize(_repeated_responseParameters_codec);
       if (payload_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(Payload);
       }
-      if (ResponseCompression != global::Grpc.Testing.CompressionType.NONE) {
+      if (ResponseCompression != 0) {
         size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ResponseCompression);
       }
       if (responseStatus_ != null) {
@@ -1399,7 +1399,7 @@ namespace Grpc.Testing {
       if (other == null) {
         return;
       }
-      if (other.ResponseType != global::Grpc.Testing.PayloadType.COMPRESSABLE) {
+      if (other.ResponseType != 0) {
         ResponseType = other.ResponseType;
       }
       responseParameters_.Add(other.responseParameters_);
@@ -1409,7 +1409,7 @@ namespace Grpc.Testing {
         }
         Payload.MergeFrom(other.Payload);
       }
-      if (other.ResponseCompression != global::Grpc.Testing.CompressionType.NONE) {
+      if (other.ResponseCompression != 0) {
         ResponseCompression = other.ResponseCompression;
       }
       if (other.responseStatus_ != null) {
diff --git a/src/csharp/Grpc.IntegrationTesting/Metrics.cs b/src/csharp/Grpc.IntegrationTesting/Metrics.cs
index 3163949d32..8f31fbc2a9 100644
--- a/src/csharp/Grpc.IntegrationTesting/Metrics.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Metrics.cs
@@ -34,10 +34,10 @@ namespace Grpc.Testing {
             "dWdlUmVzcG9uc2ViBnByb3RvMw=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.GaugeResponse), global::Grpc.Testing.GaugeResponse.Parser, new[]{ "Name", "LongValue", "DoubleValue", "StringValue" }, new[]{ "Value" }, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.GaugeRequest), global::Grpc.Testing.GaugeRequest.Parser, new[]{ "Name" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.EmptyMessage), global::Grpc.Testing.EmptyMessage.Parser, null, null, null, null)
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.GaugeResponse), global::Grpc.Testing.GaugeResponse.Parser, new[]{ "Name", "LongValue", "DoubleValue", "StringValue" }, new[]{ "Value" }, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.GaugeRequest), global::Grpc.Testing.GaugeRequest.Parser, new[]{ "Name" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EmptyMessage), global::Grpc.Testing.EmptyMessage.Parser, null, null, null, null)
           }));
     }
     #endregion
@@ -92,7 +92,7 @@ namespace Grpc.Testing {
     public string Name {
       get { return name_; }
       set {
-        name_ = pb::Preconditions.CheckNotNull(value, "value");
+        name_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
@@ -121,7 +121,7 @@ namespace Grpc.Testing {
     public string StringValue {
       get { return valueCase_ == ValueOneofCase.StringValue ? (string) value_ : ""; }
       set {
-        value_ = pb::Preconditions.CheckNotNull(value, "value");
+        value_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
         valueCase_ = ValueOneofCase.StringValue;
       }
     }
@@ -299,7 +299,7 @@ namespace Grpc.Testing {
     public string Name {
       get { return name_; }
       set {
-        name_ = pb::Preconditions.CheckNotNull(value, "value");
+        name_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
       }
     }
 
diff --git a/src/csharp/Grpc.IntegrationTesting/Payloads.cs b/src/csharp/Grpc.IntegrationTesting/Payloads.cs
index 663f625aa7..3ad7a44f4b 100644
--- a/src/csharp/Grpc.IntegrationTesting/Payloads.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Payloads.cs
@@ -34,11 +34,11 @@ namespace Grpc.Testing {
             "aW5nLkNvbXBsZXhQcm90b1BhcmFtc0gAQgkKB3BheWxvYWRiBnByb3RvMw=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ByteBufferParams), global::Grpc.Testing.ByteBufferParams.Parser, new[]{ "ReqSize", "RespSize" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.SimpleProtoParams), global::Grpc.Testing.SimpleProtoParams.Parser, new[]{ "ReqSize", "RespSize" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ComplexProtoParams), global::Grpc.Testing.ComplexProtoParams.Parser, null, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.PayloadConfig), global::Grpc.Testing.PayloadConfig.Parser, new[]{ "BytebufParams", "SimpleParams", "ComplexParams" }, new[]{ "Payload" }, null, null)
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ByteBufferParams), global::Grpc.Testing.ByteBufferParams.Parser, new[]{ "ReqSize", "RespSize" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.SimpleProtoParams), global::Grpc.Testing.SimpleProtoParams.Parser, new[]{ "ReqSize", "RespSize" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ComplexProtoParams), global::Grpc.Testing.ComplexProtoParams.Parser, null, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.PayloadConfig), global::Grpc.Testing.PayloadConfig.Parser, new[]{ "BytebufParams", "SimpleParams", "ComplexParams" }, new[]{ "Payload" }, null, null)
           }));
     }
     #endregion
diff --git a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
index 13ab5a25ab..b2f2e4d691 100644
--- a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
@@ -55,7 +55,7 @@ namespace Grpc.IntegrationTesting
         {
             var serverConfig = new ServerConfig
             {
-                ServerType = ServerType.ASYNC_SERVER
+                ServerType = ServerType.AsyncServer
             };
             serverRunner = ServerRunners.CreateStarted(serverConfig);
         }
@@ -75,7 +75,7 @@ namespace Grpc.IntegrationTesting
             var config = new ClientConfig
             {
                 ServerTargets = { string.Format("{0}:{1}", "localhost", serverRunner.BoundPort) },
-                RpcType = RpcType.UNARY,
+                RpcType = RpcType.Unary,
                 LoadParams = new LoadParams { ClosedLoop = new ClosedLoopParams() },
                 PayloadConfig = new PayloadConfig
                 {
diff --git a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs
index d7859443e0..8689d188ae 100644
--- a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs
@@ -77,13 +77,13 @@ namespace Grpc.IntegrationTesting
             }
 
             ServerServiceDefinition service = null;
-            if (config.ServerType == ServerType.ASYNC_SERVER)
+            if (config.ServerType == ServerType.AsyncServer)
             {
                 GrpcPreconditions.CheckArgument(config.PayloadConfig == null,
                     "ServerConfig.PayloadConfig shouldn't be set for BenchmarkService based server.");    
                 service = BenchmarkService.BindService(new BenchmarkServiceImpl());
             }
-            else if (config.ServerType == ServerType.ASYNC_GENERIC_SERVER)
+            else if (config.ServerType == ServerType.AsyncGenericServer)
             {
                 var genericService = new GenericServiceImpl(config.PayloadConfig.BytebufParams.RespSize);
                 service = GenericService.BindHandler(genericService.StreamingCall);
diff --git a/src/csharp/Grpc.IntegrationTesting/Services.cs b/src/csharp/Grpc.IntegrationTesting/Services.cs
index a8475c1817..e10b45c9a2 100644
--- a/src/csharp/Grpc.IntegrationTesting/Services.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Services.cs
@@ -39,7 +39,7 @@ namespace Grpc.Testing {
             "YgZwcm90bzM="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { global::Grpc.Testing.MessagesReflection.Descriptor, global::Grpc.Testing.ControlReflection.Descriptor, },
-          new pbr::GeneratedCodeInfo(null, null));
+          new pbr::GeneratedClrTypeInfo(null, null));
     }
     #endregion
 
diff --git a/src/csharp/Grpc.IntegrationTesting/Stats.cs b/src/csharp/Grpc.IntegrationTesting/Stats.cs
index 39c00ea88c..304d676113 100644
--- a/src/csharp/Grpc.IntegrationTesting/Stats.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Stats.cs
@@ -35,11 +35,11 @@ namespace Grpc.Testing {
             "ZXIYAyABKAESEwoLdGltZV9zeXN0ZW0YBCABKAFiBnByb3RvMw=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
-          new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] {
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerStats), global::Grpc.Testing.ServerStats.Parser, new[]{ "TimeElapsed", "TimeUser", "TimeSystem" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.HistogramParams), global::Grpc.Testing.HistogramParams.Parser, new[]{ "Resolution", "MaxPossible" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.HistogramData), global::Grpc.Testing.HistogramData.Parser, new[]{ "Bucket", "MinSeen", "MaxSeen", "Sum", "SumOfSquares", "Count" }, null, null, null),
-            new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientStats), global::Grpc.Testing.ClientStats.Parser, new[]{ "Latencies", "TimeElapsed", "TimeUser", "TimeSystem" }, null, null, null)
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ServerStats), global::Grpc.Testing.ServerStats.Parser, new[]{ "TimeElapsed", "TimeUser", "TimeSystem" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.HistogramParams), global::Grpc.Testing.HistogramParams.Parser, new[]{ "Resolution", "MaxPossible" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.HistogramData), global::Grpc.Testing.HistogramData.Parser, new[]{ "Bucket", "MinSeen", "MaxSeen", "Sum", "SumOfSquares", "Count" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ClientStats), global::Grpc.Testing.ClientStats.Parser, new[]{ "Latencies", "TimeElapsed", "TimeUser", "TimeSystem" }, null, null, null)
           }));
     }
     #endregion
diff --git a/src/csharp/Grpc.IntegrationTesting/Test.cs b/src/csharp/Grpc.IntegrationTesting/Test.cs
index 363f6444ec..9258dc185d 100644
--- a/src/csharp/Grpc.IntegrationTesting/Test.cs
+++ b/src/csharp/Grpc.IntegrationTesting/Test.cs
@@ -46,7 +46,7 @@ namespace Grpc.Testing {
             "cnBjLnRlc3RpbmcuUmVjb25uZWN0SW5mb2IGcHJvdG8z"));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { global::Grpc.Testing.EmptyReflection.Descriptor, global::Grpc.Testing.MessagesReflection.Descriptor, },
-          new pbr::GeneratedCodeInfo(null, null));
+          new pbr::GeneratedClrTypeInfo(null, null));
     }
     #endregion
 
diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
index 80dad9fdd9..c9eca73452 100644
--- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
+++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
@@ -64,7 +64,7 @@ namespace Grpc.Testing
             {
                 Stats = runner.GetStats(false),
                 Port = runner.BoundPort,
-                Cores = 0,  // TODO: set number of cores
+                Cores = Environment.ProcessorCount,
             });
                 
             while (await requestStream.MoveNext())
diff --git a/src/csharp/Grpc.IntegrationTesting/packages.config b/src/csharp/Grpc.IntegrationTesting/packages.config
index 3fef67dca4..3161c5b755 100644
--- a/src/csharp/Grpc.IntegrationTesting/packages.config
+++ b/src/csharp/Grpc.IntegrationTesting/packages.config
@@ -4,7 +4,7 @@
   <package id="CommandLineParser" version="1.9.71" targetFramework="net45" />
   <package id="Google.Apis.Auth" version="1.11.1" targetFramework="net45" />
   <package id="Google.Apis.Core" version="1.11.1" targetFramework="net45" />
-  <package id="Google.Protobuf" version="3.0.0-beta2" targetFramework="net45" />
+  <package id="Google.Protobuf" version="3.0.0-beta3" targetFramework="net45" />
   <package id="Ix-Async" version="1.2.5" targetFramework="net45" />
   <package id="Moq" version="4.2.1510.2205" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat
index 28e4262121..1cc63da970 100644
--- a/src/csharp/build_packages.bat
+++ b/src/csharp/build_packages.bat
@@ -31,7 +31,7 @@
 
 @rem Current package versions
 set VERSION=0.15.0-dev
-set PROTOBUF_VERSION=3.0.0-beta2
+set PROTOBUF_VERSION=3.0.0-beta3
 
 @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well.
 set VERSION_WITH_BETA=%VERSION%-beta
diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c
index 5b8ff9b819..4beef9ded8 100644
--- a/src/csharp/ext/grpc_csharp_ext.c
+++ b/src/csharp/ext/grpc_csharp_ext.c
@@ -806,11 +806,14 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_set_credentials(
 /* Server */
 
 GPR_EXPORT grpc_server *GPR_CALLTYPE
-grpcsharp_server_create(grpc_completion_queue *cq,
-                        const grpc_channel_args *args) {
-  grpc_server *server = grpc_server_create(args, NULL);
+grpcsharp_server_create(const grpc_channel_args *args) {
+  return grpc_server_create(args, NULL);
+}
+
+GPR_EXPORT void GPR_CALLTYPE
+grpcsharp_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq) {
   grpc_server_register_completion_queue(server, cq, NULL);
-  return server;
 }
 
 GPR_EXPORT int32_t GPR_CALLTYPE
diff --git a/src/node/ext/server.cc b/src/node/ext/server.cc
index b9e1fe9160..dd1b777ac8 100644
--- a/src/node/ext/server.cc
+++ b/src/node/ext/server.cc
@@ -35,15 +35,15 @@
 
 #include "server.h"
 
-#include <node.h>
 #include <nan.h>
+#include <node.h>
 
 #include <vector>
+#include "call.h"
+#include "completion_queue_async_worker.h"
 #include "grpc/grpc.h"
 #include "grpc/grpc_security.h"
 #include "grpc/support/log.h"
-#include "call.h"
-#include "completion_queue_async_worker.h"
 #include "server_credentials.h"
 #include "timeval.h"
 
@@ -100,8 +100,8 @@ class NewCallOp : public Op {
     Nan::Set(obj, Nan::New("host").ToLocalChecked(),
              Nan::New(details.host).ToLocalChecked());
     Nan::Set(obj, Nan::New("deadline").ToLocalChecked(),
-             Nan::New<Date>(
-                 TimespecToMilliseconds(details.deadline)).ToLocalChecked());
+             Nan::New<Date>(TimespecToMilliseconds(details.deadline))
+                 .ToLocalChecked());
     Nan::Set(obj, Nan::New("metadata").ToLocalChecked(),
              ParseMetadata(&request_metadata));
     return scope.Escape(obj);
@@ -117,14 +117,13 @@ class NewCallOp : public Op {
   grpc_metadata_array request_metadata;
 
  protected:
-  std::string GetTypeString() const {
-    return "new_call";
-  }
+  std::string GetTypeString() const { return "new_call"; }
 };
 
 Server::Server(grpc_server *server) : wrapped_server(server) {
   shutdown_queue = grpc_completion_queue_create(NULL);
-  grpc_server_register_completion_queue(server, shutdown_queue, NULL);
+  grpc_server_register_non_listening_completion_queue(server, shutdown_queue,
+                                                      NULL);
 }
 
 Server::~Server() {
@@ -156,8 +155,7 @@ bool Server::HasInstance(Local<Value> val) {
 }
 
 void Server::ShutdownServer() {
-  grpc_server_shutdown_and_notify(this->wrapped_server,
-                                  this->shutdown_queue,
+  grpc_server_shutdown_and_notify(this->wrapped_server, this->shutdown_queue,
                                   NULL);
   grpc_server_cancel_all_calls(this->wrapped_server);
   grpc_completion_queue_pluck(this->shutdown_queue, NULL,
@@ -170,8 +168,8 @@ NAN_METHOD(Server::New) {
   if (!info.IsConstructCall()) {
     const int argc = 1;
     Local<Value> argv[argc] = {info[0]};
-    MaybeLocal<Object> maybe_instance = constructor->GetFunction()->NewInstance(
-        argc, argv);
+    MaybeLocal<Object> maybe_instance =
+        constructor->GetFunction()->NewInstance(argc, argv);
     if (maybe_instance.IsEmpty()) {
       // There's probably a pending exception
       return;
@@ -185,8 +183,9 @@ NAN_METHOD(Server::New) {
   grpc_channel_args *channel_args;
   if (!ParseChannelArgs(info[0], &channel_args)) {
     DeallocateChannelArgs(channel_args);
-    return Nan::ThrowTypeError("Server options must be an object with "
-                               "string keys and integer or string values");
+    return Nan::ThrowTypeError(
+        "Server options must be an object with "
+        "string keys and integer or string values");
   }
   wrapped_server = grpc_server_create(channel_args, NULL);
   DeallocateChannelArgs(channel_args);
@@ -218,8 +217,7 @@ NAN_METHOD(Server::RequestCall) {
 
 NAN_METHOD(Server::AddHttp2Port) {
   if (!HasInstance(info.This())) {
-    return Nan::ThrowTypeError(
-        "addHttp2Port can only be called on a Server");
+    return Nan::ThrowTypeError("addHttp2Port can only be called on a Server");
   }
   if (!info[0]->IsString()) {
     return Nan::ThrowTypeError(
@@ -239,8 +237,7 @@ NAN_METHOD(Server::AddHttp2Port) {
                                                *Utf8String(info[0]));
   } else {
     port = grpc_server_add_secure_http2_port(server->wrapped_server,
-                                             *Utf8String(info[0]),
-                                             creds);
+                                             *Utf8String(info[0]), creds);
   }
   info.GetReturnValue().Set(Nan::New<Number>(port));
 }
@@ -262,8 +259,7 @@ NAN_METHOD(Server::TryShutdown) {
   Server *server = ObjectWrap::Unwrap<Server>(info.This());
   unique_ptr<OpVec> ops(new OpVec());
   grpc_server_shutdown_and_notify(
-      server->wrapped_server,
-      CompletionQueueAsyncWorker::GetQueue(),
+      server->wrapped_server, CompletionQueueAsyncWorker::GetQueue(),
       new struct tag(new Nan::Callback(info[0].As<Function>()), ops.release(),
                      shared_ptr<Resources>(nullptr)));
   CompletionQueueAsyncWorker::Next();
diff --git a/src/ruby/lib/grpc/signals.rb b/src/objective-c/CronetFramework.podspec
index 2ab85c8bb1..20af7647f7 100644
--- a/src/ruby/lib/grpc/signals.rb
+++ b/src/objective-c/CronetFramework.podspec
@@ -27,43 +27,17 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-require 'thread'
-require_relative 'grpc'
 
-# GRPC contains the General RPC module.
-module GRPC
-  # Signals contains gRPC functions related to signal handling
-  module Signals
-    @interpreter_exiting = false
-    @signal_handlers = []
-    @handlers_mutex = Mutex.new
-
-    def register_handler(&handler)
-      @handlers_mutex.synchronize do
-        @signal_handlers.push(handler)
-        handler.call if @exit_signal_received
-      end
-      # Returns a function to remove the handler
-      lambda do
-        @handlers_mutex.synchronize { @signal_handlers.delete(handler) }
-      end
-    end
-    module_function :register_handler
-
-    def wait_for_signals
-      t = Thread.new do
-        sleep 0.1 until GRPC::Core.signal_received? || @interpreter_exiting
-        unless @interpreter_exiting
-          @handlers_mutex.synchronize do
-            @signal_handlers.each(&:call)
-          end
-        end
-      end
-      at_exit do
-        @interpreter_exiting = true
-        t.join
-      end
-    end
-    module_function :wait_for_signals
-  end
+Pod::Spec.new do |s|
+  s.name         = "CronetFramework"
+  s.version      = "0.0.2"
+  s.summary      = "Cronet, precompiled and used as a framework."
+  s.homepage     = "http://chromium.org"
+  s.license      = { :type => 'BSD' }
+  s.vendored_framework = "Cronet.framework"
+  s.author             = "The Chromium Authors"
+  s.ios.deployment_target = "8.0"
+  s.source       = { :http => 'https://storage.googleapis.com/grpc-precompiled-binaries/cronet/Cronet.framework.zip' }
+  s.preserve_paths = "Cronet.framework"
+  s.public_header_files = "Cronet.framework/Headers/**/*{.h}"
 end
diff --git a/src/objective-c/GRPCClient/GRPCCall+Cronet.h b/src/objective-c/GRPCClient/GRPCCall+Cronet.h
new file mode 100644
index 0000000000..7f7fc6c2a9
--- /dev/null
+++ b/src/objective-c/GRPCClient/GRPCCall+Cronet.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#import <Cronet/Cronet.h>
+
+#import "GRPCCall.h"
+
+/**
+ * Methods for using cronet transport.
+ */
+@interface GRPCCall (Cronet)
+
+/**
+ * This method should be called before issuing the first RPC. It should be
+ * called only once. Create an instance of Cronet engine in your app elsewhere
+ * and pass the instance pointer in the cronet_engine parameter. Once set,
+ * all subsequent RPCs will use Cronet transport. The method is not thread
+ * safe.
+ */
++(void)useCronetWithEngine:(cronet_engine *)engine;
+
++(cronet_engine *)cronetEngine;
+
++(BOOL)isUsingCronet;
+
+@end
diff --git a/src/ruby/ext/grpc/rb_signal.h b/src/objective-c/GRPCClient/GRPCCall+Cronet.m
index 07e49c0a8b..69a410e95a 100644
--- a/src/ruby/ext/grpc/rb_signal.h
+++ b/src/objective-c/GRPCClient/GRPCCall+Cronet.m
@@ -31,9 +31,24 @@
  *
  */
 
-#ifndef GRPC_RB_SIGNAL_H_
-#define GRPC_RB_SIGNAL_H_
+#import "GRPCCall+Cronet.h"
 
-void Init_grpc_signals();
+static BOOL useCronet = NO;
+static cronet_engine *globalCronetEngine;
 
-#endif  /* GRPC_RB_SIGNAL_H_ */
+@implementation GRPCCall (Cronet)
+
++ (void)useCronetWithEngine:(cronet_engine *)engine {
+  useCronet = YES;
+  globalCronetEngine = engine;
+}
+
++ (cronet_engine *)cronetEngine {
+  return globalCronetEngine;
+}
+
++ (BOOL)isUsingCronet {
+  return useCronet;
+}
+
+@end
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h
index 70d1a9bd2f..3219835d02 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.h
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.h
@@ -56,6 +56,12 @@ struct grpc_channel_credentials;
 + (nullable GRPCChannel *)secureChannelWithHost:(nonnull NSString *)host;
 
 /**
+ * Creates a secure channel to the specified @c host using Cronet as a transport mechanism.
+ */
++ (nullable GRPCChannel *)secureCronetChannelWithHost:(NSString *)host
+                                          channelArgs:(NSDictionary *)channelArgs;
+
+/**
  * Creates a secure channel to the specified @c host using the specified @c credentials and
  * @c channelArgs. Only in tests should @c GRPC_SSL_TARGET_NAME_OVERRIDE_ARG channel arg be set.
  */
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m
index 203ef58c0d..e4e0dbe6d2 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.m
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.m
@@ -34,10 +34,13 @@
 #import "GRPCChannel.h"
 
 #include <grpc/grpc_security.h>
+#include <grpc/grpc_cronet.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#import <Cronet/Cronet.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 #import "GRPCCompletionQueue.h"
 
 void freeChannelArgs(grpc_channel_args *channel_args) {
@@ -99,6 +102,22 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
   grpc_channel_args *_channelArgs;
 }
 
+- (instancetype)initWithHost:(NSString *)host
+                cronetEngine:(cronet_engine *)cronetEngine
+                 channelArgs:(NSDictionary *)channelArgs {
+  if (!host) {
+    [NSException raise:NSInvalidArgumentException format:@"host argument missing"];
+  }
+
+  if (self = [super init]) {
+    _channelArgs = buildChannelArgs(channelArgs);
+    _host = [host copy];
+    _unmanagedChannel = grpc_cronet_secure_channel_create(cronetEngine, _host.UTF8String, _channelArgs,
+                                                     NULL);
+  }
+
+  return self;
+}
 
 - (instancetype)initWithHost:(NSString *)host
                       secure:(BOOL)secure
@@ -133,6 +152,17 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
   freeChannelArgs(_channelArgs);
 }
 
++ (GRPCChannel *)secureCronetChannelWithHost:(NSString *)host
+                                 channelArgs:(NSDictionary *)channelArgs {
+  cronet_engine *engine = [GRPCCall cronetEngine];
+  if (!engine) {
+    [NSException raise:NSInvalidArgumentException
+                format:@"cronet_engine is NULL. Set it first."];
+    return nil;
+  }
+  return [[GRPCChannel alloc] initWithHost:host cronetEngine:engine channelArgs:channelArgs];
+}
+
 + (GRPCChannel *)secureChannelWithHost:(NSString *)host {
   return [[GRPCChannel alloc] initWithHost:host secure:YES credentials:NULL channelArgs:NULL];
 }
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m
index 43166cbb52..7da508810c 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.m
+++ b/src/objective-c/GRPCClient/private/GRPCHost.m
@@ -37,6 +37,7 @@
 #include <grpc/grpc_security.h>
 #import <GRPCClient/GRPCCall.h>
 #import <GRPCClient/GRPCCall+ChannelArg.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 
 #import "GRPCChannel.h"
 #import "GRPCCompletionQueue.h"
@@ -200,15 +201,21 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (GRPCChannel *)newChannel {
   NSDictionary *args = [self channelArgs];
+  BOOL useCronet = [GRPCCall isUsingCronet];
   if (_secure) {
       GRPCChannel *channel;
       @synchronized(self) {
         if (_channelCreds == nil) {
           [self setTLSPEMRootCerts:nil withPrivateKey:nil withCertChain:nil error:nil];
         }
-        channel = [GRPCChannel secureChannelWithHost:_address
-                                          credentials:_channelCreds
-                                          channelArgs:args];
+        if (useCronet) {
+          channel = [GRPCChannel secureCronetChannelWithHost:_address
+                                                 channelArgs:args];
+        } else {
+          channel = [GRPCChannel secureChannelWithHost:_address
+                                            credentials:_channelCreds
+                                            channelArgs:args];
+        }
       }
       return channel;
   } else {
diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m
index 4f096b9efa..781c500f73 100644
--- a/src/objective-c/tests/InteropTests.m
+++ b/src/objective-c/tests/InteropTests.m
@@ -35,7 +35,9 @@
 
 #include <grpc/status.h>
 
+#import <Cronet/Cronet.h>
 #import <GRPCClient/GRPCCall+Tests.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 #import <ProtoRPC/ProtoRPC.h>
 #import <RemoteTest/Empty.pbobjc.h>
 #import <RemoteTest/Messages.pbobjc.h>
@@ -78,6 +80,8 @@
 
 #pragma mark Tests
 
+static cronet_engine *cronetEngine = NULL;
+
 @implementation InteropTests {
   RMTTestService *_service;
 }
@@ -88,6 +92,15 @@
 
 - (void)setUp {
   _service = self.class.host ? [RMTTestService serviceWithHost:self.class.host] : nil;
+#ifdef GRPC_COMPILE_WITH_CRONET
+  if (cronetEngine == NULL) {
+    // Cronet setup
+    [Cronet setHttp2Enabled:YES];
+    [Cronet start];
+    cronetEngine = [Cronet getGlobalEngine];
+    [GRPCCall useCronetWithEngine:cronetEngine];
+  }
+#endif
 }
 
 - (void)testEmptyUnaryRPC {
@@ -245,6 +258,8 @@
   [self waitForExpectationsWithTimeout:4 handler:nil];
 }
 
+#ifndef GRPC_COMPILE_WITH_CRONET
+// TODO(makdharma@): Fix this test
 - (void)testEmptyStreamRPC {
   XCTAssertNotNil(self.class.host);
   __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyStream"];
@@ -258,6 +273,7 @@
   }];
   [self waitForExpectationsWithTimeout:2 handler:nil];
 }
+#endif
 
 - (void)testCancelAfterBeginRPC {
   XCTAssertNotNil(self.class.host);
diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile
index 7ec7a25898..508641d681 100644
--- a/src/objective-c/tests/Podfile
+++ b/src/objective-c/tests/Podfile
@@ -3,6 +3,7 @@ platform :ios, '8.0'
 
 pod 'Protobuf', :path => "../../../third_party/protobuf"
 pod 'BoringSSL', :podspec => ".."
+pod 'CronetFramework', :podspec => ".."
 pod 'gRPC', :path => "../../.."
 pod 'RemoteTest', :path => "RemoteTestClient"
 
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index 66e6e6b549..d42c58050f 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -336,6 +336,8 @@ cdef extern from "grpc/_cython/loader.h":
   void grpc_server_register_completion_queue(grpc_server *server,
                                              grpc_completion_queue *cq,
                                              void *reserved) nogil
+  void grpc_server_register_non_listening_completion_queue(
+      grpc_server *server, grpc_completion_queue *cq, void *reserved) nogil
   int grpc_server_add_insecure_http2_port(
       grpc_server *server, const char *addr) nogil
   void grpc_server_start(grpc_server *server) nogil
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
index 8419a59068..55948755b2 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
@@ -81,11 +81,20 @@ cdef class Server:
           self.c_server, queue.c_completion_queue, NULL)
     self.registered_completion_queues.append(queue)
 
+  def register_non_listening_completion_queue(
+      self, CompletionQueue queue not None):
+    if self.is_started:
+      raise ValueError("cannot register completion queues after start")
+    with nogil:
+      grpc_server_register_non_listening_completion_queue(
+          self.c_server, queue.c_completion_queue, NULL)
+    self.registered_completion_queues.append(queue)
+
   def start(self):
     if self.is_started:
       raise ValueError("the server has already started")
     self.backup_shutdown_queue = CompletionQueue()
-    self.register_completion_queue(self.backup_shutdown_queue)
+    self.register_non_listening_completion_queue(self.backup_shutdown_queue)
     self.is_started = True
     with nogil:
       grpc_server_start(self.c_server)
@@ -169,4 +178,3 @@ cdef class Server:
           time.sleep(0)
       with nogil:
         grpc_server_destroy(self.c_server)
-
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c
index 09551472b5..f71cf12844 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.c
+++ b/src/python/grpcio/grpc/_cython/imports.generated.c
@@ -115,6 +115,7 @@ grpc_server_register_method_type grpc_server_register_method_import;
 grpc_server_request_registered_call_type grpc_server_request_registered_call_import;
 grpc_server_create_type grpc_server_create_import;
 grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
+grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
 grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 grpc_server_start_type grpc_server_start_import;
 grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import;
@@ -386,6 +387,7 @@ void pygrpc_load_imports(HMODULE library) {
   grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call");
   grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create");
   grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue");
+  grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue");
   grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port");
   grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start");
   grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify");
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
index 6de295414a..a364075e9e 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.h
+++ b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -296,6 +296,9 @@ extern grpc_server_create_type grpc_server_create_import;
 typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved);
 extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
 #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import
+typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved);
+extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
+#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import
 typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr);
 extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import
diff --git a/src/python/grpcio/grpc/framework/foundation/future.py b/src/python/grpcio/grpc/framework/foundation/future.py
index 9210616150..6fb58eadb6 100644
--- a/src/python/grpcio/grpc/framework/foundation/future.py
+++ b/src/python/grpcio/grpc/framework/foundation/future.py
@@ -232,6 +232,6 @@ class Future(six.with_metaclass(abc.ABCMeta)):
     immediately.
 
     Args:
-      fn: A callable taking a this Future object as its single parameter.
+      fn: A callable taking this Future object as its single parameter.
     """
     raise NotImplementedError()
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index cb18e5a5e3..1427396d8b 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -94,7 +94,6 @@ CORE_SOURCE_FILES = [
   'src/core/lib/iomgr/endpoint.c',
   'src/core/lib/iomgr/endpoint_pair_posix.c',
   'src/core/lib/iomgr/endpoint_pair_windows.c',
-  'src/core/lib/iomgr/ev_poll_and_epoll_posix.c',
   'src/core/lib/iomgr/ev_poll_posix.c',
   'src/core/lib/iomgr/ev_posix.c',
   'src/core/lib/iomgr/exec_ctx.c',
@@ -184,20 +183,28 @@ CORE_SOURCE_FILES = [
   'src/core/ext/transport/chttp2/transport/writing.c',
   'src/core/ext/transport/chttp2/alpn/alpn.c',
   'src/core/lib/http/httpcli_security_connector.c',
-  'src/core/lib/security/b64.c',
-  'src/core/lib/security/client_auth_filter.c',
-  'src/core/lib/security/credentials.c',
-  'src/core/lib/security/credentials_metadata.c',
-  'src/core/lib/security/credentials_posix.c',
-  'src/core/lib/security/credentials_win32.c',
-  'src/core/lib/security/google_default_credentials.c',
-  'src/core/lib/security/handshake.c',
-  'src/core/lib/security/json_token.c',
-  'src/core/lib/security/jwt_verifier.c',
-  'src/core/lib/security/secure_endpoint.c',
-  'src/core/lib/security/security_connector.c',
-  'src/core/lib/security/security_context.c',
-  'src/core/lib/security/server_auth_filter.c',
+  'src/core/lib/security/context/security_context.c',
+  'src/core/lib/security/credentials/composite/composite_credentials.c',
+  'src/core/lib/security/credentials/credentials.c',
+  'src/core/lib/security/credentials/credentials_metadata.c',
+  'src/core/lib/security/credentials/fake/fake_credentials.c',
+  'src/core/lib/security/credentials/google_default/credentials_posix.c',
+  'src/core/lib/security/credentials/google_default/credentials_win32.c',
+  'src/core/lib/security/credentials/google_default/google_default_credentials.c',
+  'src/core/lib/security/credentials/iam/iam_credentials.c',
+  'src/core/lib/security/credentials/jwt/json_token.c',
+  'src/core/lib/security/credentials/jwt/jwt_credentials.c',
+  'src/core/lib/security/credentials/jwt/jwt_verifier.c',
+  'src/core/lib/security/credentials/oauth2/oauth2_credentials.c',
+  'src/core/lib/security/credentials/plugin/plugin_credentials.c',
+  'src/core/lib/security/credentials/ssl/ssl_credentials.c',
+  'src/core/lib/security/transport/client_auth_filter.c',
+  'src/core/lib/security/transport/handshake.c',
+  'src/core/lib/security/transport/secure_endpoint.c',
+  'src/core/lib/security/transport/security_connector.c',
+  'src/core/lib/security/transport/server_auth_filter.c',
+  'src/core/lib/security/util/b64.c',
+  'src/core/lib/security/util/json_util.c',
   'src/core/lib/surface/init_secure.c',
   'src/core/lib/tsi/fake_transport_security.c',
   'src/core/lib/tsi/ssl_transport_security.c',
diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json
index 691062f25a..1beb619f87 100644
--- a/src/python/grpcio/tests/tests.json
+++ b/src/python/grpcio/tests/tests.json
@@ -5,6 +5,7 @@
   "_base_interface_test.SyncPeasyTest", 
   "_beta_features_test.BetaFeaturesTest", 
   "_beta_features_test.ContextManagementAndLifecycleTest", 
+  "_cancel_many_calls_test.CancelManyCallsTest",
   "_channel_test.ChannelTest", 
   "_connectivity_channel_test.ChannelConnectivityTest", 
   "_core_over_links_base_interface_test.AsyncEasyTest", 
diff --git a/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py
new file mode 100644
index 0000000000..c1de779014
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py
@@ -0,0 +1,222 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test making many calls and immediately cancelling most of them."""
+
+import threading
+import unittest
+
+from grpc._cython import cygrpc
+from grpc.framework.foundation import logging_pool
+from tests.unit.framework.common import test_constants
+
+_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
+_EMPTY_FLAGS = 0
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+_SERVER_SHUTDOWN_TAG = 'server_shutdown'
+_REQUEST_CALL_TAG = 'request_call'
+_RECEIVE_CLOSE_ON_SERVER_TAG = 'receive_close_on_server'
+_RECEIVE_MESSAGE_TAG = 'receive_message'
+_SERVER_COMPLETE_CALL_TAG = 'server_complete_call'
+
+_SUCCESS_CALL_FRACTION = 1.0 / 8.0
+
+
+class _State(object):
+
+  def __init__(self):
+    self.condition = threading.Condition()
+    self.handlers_released = False
+    self.parked_handlers = 0
+    self.handled_rpcs = 0
+
+
+def _is_cancellation_event(event):
+  return (
+      event.tag is _RECEIVE_CLOSE_ON_SERVER_TAG and
+      event.batch_operations[0].received_cancelled)
+
+
+class _Handler(object):
+
+  def __init__(self, state, completion_queue, rpc_event):
+    self._state = state
+    self._lock = threading.Lock()
+    self._completion_queue = completion_queue
+    self._call = rpc_event.operation_call
+
+  def __call__(self):
+    with self._state.condition:
+      self._state.parked_handlers += 1
+      if self._state.parked_handlers == test_constants.THREAD_CONCURRENCY:
+        self._state.condition.notify_all()
+      while not self._state.handlers_released:
+        self._state.condition.wait()
+
+    with self._lock:
+      self._call.start_batch(
+          cygrpc.Operations(
+              (cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),)),
+          _RECEIVE_CLOSE_ON_SERVER_TAG)
+      self._call.start_batch(
+          cygrpc.Operations((cygrpc.operation_receive_message(_EMPTY_FLAGS),)),
+          _RECEIVE_MESSAGE_TAG)
+    first_event = self._completion_queue.poll()
+    if _is_cancellation_event(first_event):
+      self._completion_queue.poll()
+    else:
+      with self._lock:
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(b'\x79\x57', _EMPTY_FLAGS),
+            cygrpc.operation_send_status_from_server(
+                _EMPTY_METADATA, cygrpc.StatusCode.ok, b'test details!',
+                _EMPTY_FLAGS),
+        )
+        self._call.start_batch(
+            cygrpc.Operations(operations), _SERVER_COMPLETE_CALL_TAG)
+      self._completion_queue.poll()
+      self._completion_queue.poll()
+
+
+def _serve(state, server, server_completion_queue, thread_pool):
+  for _ in range(test_constants.RPC_CONCURRENCY):
+    call_completion_queue = cygrpc.CompletionQueue()
+    server.request_call(
+        call_completion_queue, server_completion_queue, _REQUEST_CALL_TAG)
+    rpc_event = server_completion_queue.poll()
+    thread_pool.submit(_Handler(state, call_completion_queue, rpc_event))
+    with state.condition:
+      state.handled_rpcs += 1
+      if test_constants.RPC_CONCURRENCY <= state.handled_rpcs:
+        state.condition.notify_all()
+  server_completion_queue.poll()
+
+
+class _QueueDriver(object):
+
+  def __init__(self, condition, completion_queue, due):
+    self._condition = condition
+    self._completion_queue = completion_queue
+    self._due = due
+    self._events = []
+    self._returned = False
+
+  def start(self):
+    def in_thread():
+      while True:
+        event = self._completion_queue.poll()
+        with self._condition:
+          self._events.append(event)
+          self._due.remove(event.tag)
+          self._condition.notify_all()
+          if not self._due:
+            self._returned = True
+            return
+    thread = threading.Thread(target=in_thread)
+    thread.start()
+
+  def events(self, at_least):
+    with self._condition:
+      while len(self._events) < at_least:
+        self._condition.wait()
+      return tuple(self._events)
+
+
+class CancelManyCallsTest(unittest.TestCase):
+
+  def testCancelManyCalls(self):
+    server_thread_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+
+    server_completion_queue = cygrpc.CompletionQueue()
+    server = cygrpc.Server()
+    server.register_completion_queue(server_completion_queue)
+    port = server.add_http2_port('[::]:0')
+    server.start()
+    channel = cygrpc.Channel('localhost:{}'.format(port))
+
+    state = _State()
+
+    server_thread_args = (
+        state, server, server_completion_queue, server_thread_pool,)
+    server_thread = threading.Thread(target=_serve, args=server_thread_args)
+    server_thread.start()
+
+    client_condition = threading.Condition()
+    client_due = set()
+    client_completion_queue = cygrpc.CompletionQueue()
+    client_driver = _QueueDriver(
+        client_condition, client_completion_queue, client_due)
+    client_driver.start()
+
+    with client_condition:
+      client_calls = []
+      for index in range(test_constants.RPC_CONCURRENCY):
+        client_call = channel.create_call(
+            None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', None,
+            _INFINITE_FUTURE)
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(b'\x45\x56', _EMPTY_FLAGS),
+            cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+            cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+            cygrpc.operation_receive_message(_EMPTY_FLAGS),
+            cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+        )
+        tag = 'client_complete_call_{0:04d}_tag'.format(index)
+        client_call.start_batch(cygrpc.Operations(operations), tag)
+        client_due.add(tag)
+        client_calls.append(client_call)
+
+    with state.condition:
+      while True:
+        if state.parked_handlers < test_constants.THREAD_CONCURRENCY:
+          state.condition.wait()
+        elif state.handled_rpcs < test_constants.RPC_CONCURRENCY:
+          state.condition.wait()
+        else:
+          state.handlers_released = True
+          state.condition.notify_all()
+          break
+
+    client_driver.events(
+        test_constants.RPC_CONCURRENCY * _SUCCESS_CALL_FRACTION)
+    with client_condition:
+      for client_call in client_calls:
+        client_call.cancel()
+
+    with state.condition:
+      server.shutdown(server_completion_queue, _SERVER_SHUTDOWN_TAG)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/ruby/ext/grpc/rb_completion_queue.c b/src/ruby/ext/grpc/rb_completion_queue.c
index 4bb615f8be..b6ddbe88dc 100644
--- a/src/ruby/ext/grpc/rb_completion_queue.c
+++ b/src/ruby/ext/grpc/rb_completion_queue.c
@@ -52,21 +52,41 @@ typedef struct next_call_stack {
   grpc_event event;
   gpr_timespec timeout;
   void *tag;
+  volatile int interrupted;
 } next_call_stack;
 
 /* Calls grpc_completion_queue_next without holding the ruby GIL */
 static void *grpc_rb_completion_queue_next_no_gil(void *param) {
   next_call_stack *const next_call = (next_call_stack*)param;
-  next_call->event =
-      grpc_completion_queue_next(next_call->cq, next_call->timeout, NULL);
+  gpr_timespec increment = gpr_time_from_millis(20, GPR_TIMESPAN);
+  gpr_timespec deadline;
+  do {
+    deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), increment);
+    next_call->event = grpc_completion_queue_next(next_call->cq,
+                                                  deadline, NULL);
+    if (next_call->event.type != GRPC_QUEUE_TIMEOUT ||
+        gpr_time_cmp(deadline, next_call->timeout) > 0) {
+      break;
+    }
+  } while (!next_call->interrupted);
   return NULL;
 }
 
 /* Calls grpc_completion_queue_pluck without holding the ruby GIL */
 static void *grpc_rb_completion_queue_pluck_no_gil(void *param) {
   next_call_stack *const next_call = (next_call_stack*)param;
-  next_call->event = grpc_completion_queue_pluck(next_call->cq, next_call->tag,
-                                                 next_call->timeout, NULL);
+  gpr_timespec increment = gpr_time_from_millis(20, GPR_TIMESPAN);
+  gpr_timespec deadline;
+  do {
+    deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), increment);
+    next_call->event = grpc_completion_queue_pluck(next_call->cq,
+                                                   next_call->tag,
+                                                   deadline, NULL);
+    if (next_call->event.type != GRPC_QUEUE_TIMEOUT ||
+        gpr_time_cmp(deadline, next_call->timeout) > 0) {
+      break;
+    }
+  } while (!next_call->interrupted);
   return NULL;
 }
 
@@ -139,6 +159,11 @@ static VALUE grpc_rb_completion_queue_alloc(VALUE cls) {
   return TypedData_Wrap_Struct(cls, &grpc_rb_completion_queue_data_type, cq);
 }
 
+static void unblock_func(void *param) {
+  next_call_stack *const next_call = (next_call_stack*)param;
+  next_call->interrupted = 1;
+}
+
 /* Blocks until the next event for given tag is available, and returns the
  * event. */
 grpc_event grpc_rb_completion_queue_pluck_event(VALUE self, VALUE tag,
@@ -158,8 +183,23 @@ grpc_event grpc_rb_completion_queue_pluck_event(VALUE self, VALUE tag,
     next_call.tag = ROBJECT(tag);
   }
   next_call.event.type = GRPC_QUEUE_TIMEOUT;
-  rb_thread_call_without_gvl(grpc_rb_completion_queue_pluck_no_gil,
-                             (void *)&next_call, NULL, NULL);
+  /* Loop until we finish a pluck without an interruption. The internal
+     pluck function runs either until it is interrupted or it gets an
+     event, or time runs out.
+
+     The basic reason we need this relatively complicated construction is that
+     we need to re-acquire the GVL when an interrupt comes in, so that the ruby
+     interpreter can do what it needs to do with the interrupt. But we also need
+     to get back to plucking when the interrupt has been handled. */
+  do {
+    next_call.interrupted = 0;
+    rb_thread_call_without_gvl(grpc_rb_completion_queue_pluck_no_gil,
+                               (void *)&next_call, unblock_func,
+                               (void *)&next_call);
+    /* If an interrupt prevented pluck from returning useful information, then
+       any plucks that did complete must have timed out */
+  } while (next_call.interrupted &&
+           next_call.event.type == GRPC_QUEUE_TIMEOUT);
   return next_call.event;
 }
 
diff --git a/src/ruby/ext/grpc/rb_completion_queue.h b/src/ruby/ext/grpc/rb_completion_queue.h
index 6cc4e96589..42de43c3fb 100644
--- a/src/ruby/ext/grpc/rb_completion_queue.h
+++ b/src/ruby/ext/grpc/rb_completion_queue.h
@@ -46,7 +46,7 @@ grpc_completion_queue *grpc_rb_get_wrapped_completion_queue(VALUE v);
  *
  * This avoids having code that holds the GIL repeated at multiple sites.
  */
-grpc_event grpc_rb_completion_queue_pluck_event(VALUE cqueue, VALUE tag,
+grpc_event grpc_rb_completion_queue_pluck_event(VALUE self, VALUE tag,
                                                 VALUE timeout);
 
 /* Initializes the CompletionQueue class. */
diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c
index 5277148fc9..06a07ac646 100644
--- a/src/ruby/ext/grpc/rb_grpc.c
+++ b/src/ruby/ext/grpc/rb_grpc.c
@@ -50,7 +50,6 @@
 #include "rb_loader.h"
 #include "rb_server.h"
 #include "rb_server_credentials.h"
-#include "rb_signal.h"
 
 static VALUE grpc_rb_cTimeVal = Qnil;
 
@@ -333,7 +332,6 @@ void Init_grpc_c() {
   Init_grpc_channel_credentials();
   Init_grpc_server();
   Init_grpc_server_credentials();
-  Init_grpc_signals();
   Init_grpc_status_codes();
   Init_grpc_time_consts();
 }
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
index cebbe8c40f..3b62984def 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
@@ -115,6 +115,7 @@ grpc_server_register_method_type grpc_server_register_method_import;
 grpc_server_request_registered_call_type grpc_server_request_registered_call_import;
 grpc_server_create_type grpc_server_create_import;
 grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
+grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
 grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 grpc_server_start_type grpc_server_start_import;
 grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import;
@@ -382,6 +383,7 @@ void grpc_rb_load_imports(HMODULE library) {
   grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call");
   grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create");
   grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue");
+  grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue");
   grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port");
   grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start");
   grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify");
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
index d7ea6c574c..1428e6d71c 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -296,6 +296,9 @@ extern grpc_server_create_type grpc_server_create_import;
 typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved);
 extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
 #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import
+typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved);
+extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
+#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import
 typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr);
 extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import
diff --git a/src/ruby/ext/grpc/rb_server.c b/src/ruby/ext/grpc/rb_server.c
index 2b3acaaf59..0899feb685 100644
--- a/src/ruby/ext/grpc/rb_server.c
+++ b/src/ruby/ext/grpc/rb_server.c
@@ -60,6 +60,7 @@ typedef struct grpc_rb_server {
   VALUE mark;
   /* The actual server */
   grpc_server *wrapped;
+  grpc_completion_queue *queue;
 } grpc_rb_server;
 
 /* Destroys server instances. */
@@ -145,6 +146,7 @@ static VALUE grpc_rb_server_init(VALUE self, VALUE cqueue, VALUE channel_args) {
   }
   grpc_server_register_completion_queue(srv, cq, NULL);
   wrapper->wrapped = srv;
+  wrapper->queue = cq;
 
   /* Add the cq as the server's mark object. This ensures the ruby cq can't be
      GCed before the server */
diff --git a/src/ruby/lib/grpc.rb b/src/ruby/lib/grpc.rb
index 7c9aae30e9..79fa705b1c 100644
--- a/src/ruby/lib/grpc.rb
+++ b/src/ruby/lib/grpc.rb
@@ -33,7 +33,6 @@ require_relative 'grpc/errors'
 require_relative 'grpc/grpc'
 require_relative 'grpc/logconfig'
 require_relative 'grpc/notifier'
-require_relative 'grpc/signals'
 require_relative 'grpc/version'
 require_relative 'grpc/core/time_consts'
 require_relative 'grpc/generic/active_call'
@@ -48,5 +47,3 @@ begin
 ensure
   file.close
 end
-
-GRPC::Signals.wait_for_signals
diff --git a/src/ruby/lib/grpc/errors.rb b/src/ruby/lib/grpc/errors.rb
index a1dd1e3e9d..23b2bb7e12 100644
--- a/src/ruby/lib/grpc/errors.rb
+++ b/src/ruby/lib/grpc/errors.rb
@@ -40,11 +40,12 @@ module GRPC
 
     # @param code [Numeric] the status code
     # @param details [String] the details of the exception
-    def initialize(code, details = 'unknown cause', **kw)
+    # @param metadata [Hash] the error's metadata
+    def initialize(code, details = 'unknown cause', metadata = {})
       super("#{code}:#{details}")
       @code = code
       @details = details
-      @metadata = kw
+      @metadata = metadata
     end
 
     # Converts the exception to a GRPC::Status for use in the networking
diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb
index fd20a86144..e449e89176 100644
--- a/src/ruby/lib/grpc/generic/active_call.rb
+++ b/src/ruby/lib/grpc/generic/active_call.rb
@@ -30,7 +30,6 @@
 require 'forwardable'
 require 'weakref'
 require_relative 'bidi_call'
-require_relative '../signals'
 
 class Struct
   # BatchResult is the struct returned by calls to call#start_batch.
@@ -45,7 +44,7 @@ class Struct
         # raise BadStatus, propagating the metadata if present.
         md = status.metadata
         with_sym_keys = Hash[md.each_pair.collect { |x, y| [x.to_sym, y] }]
-        fail GRPC::BadStatus.new(status.code, status.details, **with_sym_keys)
+        fail GRPC::BadStatus.new(status.code, status.details, with_sym_keys)
       end
       status
     end
@@ -77,14 +76,15 @@ module GRPC
     #
     # @param call [Call] a call on which to start and invocation
     # @param q [CompletionQueue] the completion queue
-    def self.client_invoke(call, q, **kw)
+    # @param metadata [Hash] the metadata
+    def self.client_invoke(call, q, metadata = {})
       fail(TypeError, '!Core::Call') unless call.is_a? Core::Call
       unless q.is_a? Core::CompletionQueue
         fail(TypeError, '!Core::CompletionQueue')
       end
       metadata_tag = Object.new
       call.run_batch(q, metadata_tag, INFINITE_FUTURE,
-                     SEND_INITIAL_METADATA => kw)
+                     SEND_INITIAL_METADATA => metadata)
       metadata_tag
     end
 
@@ -123,10 +123,6 @@ module GRPC
       @unmarshal = unmarshal
       @metadata_tag = metadata_tag
       @op_notifier = nil
-      weak_self = WeakRef.new(self)
-      remove_handler = GRPC::Signals.register_handler(&weak_self
-                                                        .method(:cancel))
-      ObjectSpace.define_finalizer(self, remove_handler)
     end
 
     # output_metadata are provides access to hash that can be used to
@@ -216,13 +212,12 @@ module GRPC
     # @param details [String] details
     # @param assert_finished [true, false] when true(default), waits for
     # FINISHED.
-    #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    def send_status(code = OK, details = '', assert_finished = false, **kw)
+    # @param metadata [Hash] metadata to send to the server. If a value is a
+    # list, mulitple metadata for its key are sent
+    def send_status(code = OK, details = '', assert_finished = false,
+                    metadata: {})
       ops = {
-        SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, kw)
+        SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, metadata)
       }
       ops[RECV_CLOSE_ON_SERVER] = nil if assert_finished
       @call.run_batch(@cq, self, INFINITE_FUTURE, ops)
@@ -316,14 +311,12 @@ module GRPC
     # request_response sends a request to a GRPC server, and returns the
     # response.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param req [Object] the request sent to the server
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Object] the response received from the server
-    def request_response(req, **kw)
-      start_call(**kw) unless @started
+    def request_response(req, metadata: {})
+      start_call(metadata) unless @started
       remote_send(req)
       writes_done(false)
       response = remote_read
@@ -342,14 +335,12 @@ module GRPC
     # array of marshallable objects; in typical case it will be an Enumerable
     # that allows dynamic construction of the marshallable objects.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param requests [Object] an Enumerable of requests to send
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Object] the response received from the server
-    def client_streamer(requests, **kw)
-      start_call(**kw) unless @started
+    def client_streamer(requests, metadata: {})
+      start_call(metadata) unless @started
       requests.each { |r| remote_send(r) }
       writes_done(false)
       response = remote_read
@@ -370,15 +361,12 @@ module GRPC
     # it is executed with each response as the argument and no result is
     # returned.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    # any keyword arguments are treated as metadata to be sent to the server.
-    #
     # @param req [Object] the request sent to the server
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Enumerator|nil] a response Enumerator
-    def server_streamer(req, **kw)
-      start_call(**kw) unless @started
+    def server_streamer(req, metadata: {})
+      start_call(metadata) unless @started
       remote_send(req)
       writes_done(false)
       replies = enum_for(:each_remote_read_then_finish)
@@ -412,14 +400,12 @@ module GRPC
     # the_call#writes_done has been called, otherwise the block will loop
     # forever.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param requests [Object] an Enumerable of requests to send
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Enumerator, nil] a response Enumerator
-    def bidi_streamer(requests, **kw, &blk)
-      start_call(**kw) unless @started
+    def bidi_streamer(requests, metadata: {}, &blk)
+      start_call(metadata) unless @started
       bd = BidiCall.new(@call, @cq, @marshal, @unmarshal,
                         metadata_tag: @metadata_tag)
       @metadata_tag = nil  # run_on_client ensures metadata is read
@@ -458,9 +444,11 @@ module GRPC
     private
 
     # Starts the call if not already started
-    def start_call(**kw)
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
+    def start_call(metadata = {})
       return if @started
-      @metadata_tag = ActiveCall.client_invoke(@call, @cq, **kw)
+      @metadata_tag = ActiveCall.client_invoke(@call, @cq, metadata)
       @started = true
     end
 
diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb
index 12946fe819..cddca13d17 100644
--- a/src/ruby/lib/grpc/generic/client_stub.rb
+++ b/src/ruby/lib/grpc/generic/client_stub.rb
@@ -44,21 +44,21 @@ module GRPC
 
     # setup_channel is used by #initialize to constuct a channel from its
     # arguments.
-    def self.setup_channel(alt_chan, host, creds, **kw)
+    def self.setup_channel(alt_chan, host, creds, channel_args = {})
       unless alt_chan.nil?
         fail(TypeError, '!Channel') unless alt_chan.is_a?(Core::Channel)
         return alt_chan
       end
-      if kw['grpc.primary_user_agent'].nil?
-        kw['grpc.primary_user_agent'] = ''
+      if channel_args['grpc.primary_user_agent'].nil?
+        channel_args['grpc.primary_user_agent'] = ''
       else
-        kw['grpc.primary_user_agent'] += ' '
+        channel_args['grpc.primary_user_agent'] += ' '
       end
-      kw['grpc.primary_user_agent'] += "grpc-ruby/#{VERSION}"
+      channel_args['grpc.primary_user_agent'] += "grpc-ruby/#{VERSION}"
       unless creds.is_a?(Core::ChannelCredentials) || creds.is_a?(Symbol)
         fail(TypeError, '!ChannelCredentials or Symbol')
       end
-      Core::Channel.new(host, kw, creds)
+      Core::Channel.new(host, channel_args, creds)
     end
 
     # Allows users of the stub to modify the propagate mask.
@@ -96,15 +96,16 @@ module GRPC
     #     :this_channel_is_insecure
     # @param channel_override [Core::Channel] a pre-created channel
     # @param timeout [Number] the default timeout to use in requests
-    # @param kw [KeywordArgs]the channel arguments
+    # @param channel_args [Hash] the channel arguments
     def initialize(host, q, creds,
                    channel_override: nil,
                    timeout: nil,
                    propagate_mask: nil,
-                   **kw)
+                   channel_args: {})
       fail(TypeError, '!CompletionQueue') unless q.is_a?(Core::CompletionQueue)
-      @ch = ClientStub.setup_channel(channel_override, host, creds, **kw)
-      alt_host = kw[Core::Channel::SSL_TARGET]
+      @ch = ClientStub.setup_channel(channel_override, host, creds,
+                                     channel_args)
+      alt_host = channel_args[Core::Channel::SSL_TARGET]
       @host = alt_host.nil? ? host : alt_host
       @propagate_mask = propagate_mask
       @timeout = timeout.nil? ? DEFAULT_TIMEOUT : timeout
@@ -135,42 +136,35 @@ module GRPC
     # If return_op is true, the call returns an Operation, calling execute
     # on the Operation returns the response.
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # @param method [String] the RPC method to call on the GRPC server
     # @param req [Object] the request sent to the server
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
+    # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
-    # @param return_op [true|false] return an Operation if true
+    # @param metadata [Hash] metadata to be sent to the server
     # @return [Object] the response received from the server
     def request_response(method, req, marshal, unmarshal,
                          deadline: nil,
-                         timeout: nil,
                          return_op: false,
                          parent: nil,
                          credentials: nil,
-                         **kw)
+                         metadata: {})
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.request_response(req, **kw) unless return_op
+      return c.request_response(req, metadata: metadata) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #request_response.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.request_response(req, **kw)
+        c.request_response(req, metadata: metadata)
       end
       op
     end
@@ -205,42 +199,35 @@ module GRPC
     #
     # If return_op is true, the call returns the response.
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # @param method [String] the RPC method to call on the GRPC server
     # @param requests [Object] an Enumerable of requests to send
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
+    # @param metadata [Hash] metadata to be sent to the server
     # @return [Object|Operation] the response received from the server
     def client_streamer(method, requests, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         return_op: false,
                         parent: nil,
                         credentials: nil,
-                        **kw)
+                        metadata: {})
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.client_streamer(requests, **kw) unless return_op
+      return c.client_streamer(requests, metadata: metadata) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #client_streamer.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.client_streamer(requests, **kw)
+        c.client_streamer(requests, metadata: metadata)
       end
       op
     end
@@ -292,35 +279,33 @@ module GRPC
     # @param req [Object] the request sent to the server
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param return_op [true|false]return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
+    # @param metadata [Hash] metadata to be sent to the server
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|Operation|nil] as discussed above
     def server_streamer(method, req, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         return_op: false,
                         parent: nil,
                         credentials: nil,
-                        **kw,
+                        metadata: {},
                         &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.server_streamer(req, **kw, &blk) unless return_op
+      return c.server_streamer(req, metadata: metadata, &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #server_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.server_streamer(req, **kw, &blk)
+        c.server_streamer(req, metadata: metadata, &blk)
       end
       op
     end
@@ -391,11 +376,6 @@ module GRPC
     # * the deadline is exceeded
     #
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # == Return Value ==
     #
     # if the return_op is false, the return value is an Enumerator of the
@@ -411,36 +391,35 @@ module GRPC
     # @param requests [Object] an Enumerable of requests to send
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
     # @param return_op [true|false] return an Operation if true
+    # @param metadata [Hash] metadata to be sent to the server
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|nil|Operation] as discussed above
     def bidi_streamer(method, requests, marshal, unmarshal,
                       deadline: nil,
-                      timeout: nil,
                       return_op: false,
                       parent: nil,
                       credentials: nil,
-                      **kw,
+                      metadata: {},
                       &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
 
-      return c.bidi_streamer(requests, **kw, &blk) unless return_op
+      return c.bidi_streamer(requests, metadata: metadata,
+                             &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #bidi_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.bidi_streamer(requests, **kw, &blk)
+        c.bidi_streamer(requests, metadata: metadata, &blk)
       end
       op
     end
@@ -457,12 +436,10 @@ module GRPC
     # @param timeout [TimeConst]
     def new_active_call(method, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         parent: nil,
                         credentials: nil)
-      if deadline.nil?
-        deadline = from_relative_time(timeout.nil? ? @timeout : timeout)
-      end
+
+      deadline = from_relative_time(@timeout) if deadline.nil?
       # Provide each new client call with its own completion queue
       call_queue = Core::CompletionQueue.new
       call = @ch.create_call(call_queue,
diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb
index cc21ffd3c5..913f55d0d3 100644
--- a/src/ruby/lib/grpc/generic/rpc_desc.rb
+++ b/src/ruby/lib/grpc/generic/rpc_desc.rb
@@ -80,12 +80,12 @@ module GRPC
       else  # is a bidi_stream
         active_call.run_server_bidi(mth)
       end
-      send_status(active_call, OK, 'OK', **active_call.output_metadata)
+      send_status(active_call, OK, 'OK', active_call.output_metadata)
     rescue BadStatus => e
       # this is raised by handlers that want GRPC to send an application error
       # code and detail message and some additional app-specific metadata.
       GRPC.logger.debug("app err:#{active_call}, status:#{e.code}:#{e.details}")
-      send_status(active_call, e.code, e.details, **e.metadata)
+      send_status(active_call, e.code, e.details, e.metadata)
     rescue Core::CallError => e
       # This is raised by GRPC internals but should rarely, if ever happen.
       # Log it, but don't notify the other endpoint..
@@ -135,10 +135,10 @@ module GRPC
       "##{mth.name}: bad arg count; got:#{mth.arity}, want:#{want}, #{msg}"
     end
 
-    def send_status(active_client, code, details, **kw)
+    def send_status(active_client, code, details, metadata = {})
       details = 'Not sure why' if details.nil?
       GRPC.logger.debug("Sending status  #{code}:#{details}")
-      active_client.send_status(code, details, code == OK, **kw)
+      active_client.send_status(code, details, code == OK, metadata: metadata)
     rescue StandardError => e
       GRPC.logger.warn("Could not send status #{code}:#{details}")
       GRPC.logger.warn(e)
diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb
index 238aaa9656..6b0b4ce557 100644
--- a/src/ruby/lib/grpc/generic/rpc_server.rb
+++ b/src/ruby/lib/grpc/generic/rpc_server.rb
@@ -28,7 +28,6 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 require_relative '../grpc'
-require_relative '../signals'
 require_relative 'active_call'
 require_relative 'service'
 require 'thread'
@@ -170,14 +169,6 @@ module GRPC
       alt_cq
     end
 
-    # setup_srv is used by #initialize to constuct a Core::Server from its
-    # arguments.
-    def self.setup_srv(alt_srv, cq, **kw)
-      return Core::Server.new(cq, kw) if alt_srv.nil?
-      fail(TypeError, '!Server') unless alt_srv.is_a? Core::Server
-      alt_srv
-    end
-
     # setup_connect_md_proc is used by #initialize to validate the
     # connect_md_proc.
     def self.setup_connect_md_proc(a_proc)
@@ -194,9 +185,6 @@ module GRPC
     # instance, however other arbitrary are allowed and when present are used
     # to configure the listeninng connection set up by the RpcServer.
     #
-    # * server_override: which if passed must be a [GRPC::Core::Server].  When
-    # present.
-    #
     # * poll_period: when present, the server polls for new events with this
     # period
     #
@@ -218,13 +206,15 @@ module GRPC
     # when non-nil is a proc for determining metadata to to send back the client
     # on receiving an invocation req.  The proc signature is:
     # {key: val, ..} func(method_name, {key: val, ...})
+    #
+    # * server_args:
+    # A server arguments hash to be passed down to the underlying core server
     def initialize(pool_size:DEFAULT_POOL_SIZE,
                    max_waiting_requests:DEFAULT_MAX_WAITING_REQUESTS,
                    poll_period:DEFAULT_POLL_PERIOD,
                    completion_queue_override:nil,
-                   server_override:nil,
                    connect_md_proc:nil,
-                   **kw)
+                   server_args:{})
       @connect_md_proc = RpcServer.setup_connect_md_proc(connect_md_proc)
       @cq = RpcServer.setup_cq(completion_queue_override)
       @max_waiting_requests = max_waiting_requests
@@ -236,7 +226,7 @@ module GRPC
       # running_state can take 4 values: :not_started, :running, :stopping, and
       # :stopped. State transitions can only proceed in that order.
       @running_state = :not_started
-      @server = RpcServer.setup_srv(server_override, @cq, **kw)
+      @server = Core::Server.new(@cq, server_args)
     end
 
     # stops a running server
@@ -353,10 +343,7 @@ module GRPC
         transition_running_state(:running)
         @run_cond.broadcast
       end
-      remove_signal_handler = GRPC::Signals.register_handler { stop }
       loop_handle_server_calls
-      # Remove signal handler when server stops
-      remove_signal_handler.call
     end
 
     alias_method :run_till_terminated, :run
diff --git a/src/ruby/lib/grpc/generic/service.rb b/src/ruby/lib/grpc/generic/service.rb
index 0a166e823e..f30242ee80 100644
--- a/src/ruby/lib/grpc/generic/service.rb
+++ b/src/ruby/lib/grpc/generic/service.rb
@@ -179,24 +179,24 @@ module GRPC
             unmarshal = desc.unmarshal_proc(:output)
             route = "/#{route_prefix}/#{name}"
             if desc.request_response?
-              define_method(mth_name) do |req, **kw|
+              define_method(mth_name) do |req, metadata = {}|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                request_response(route, req, marshal, unmarshal, **kw)
+                request_response(route, req, marshal, unmarshal, metadata)
               end
             elsif desc.client_streamer?
-              define_method(mth_name) do |reqs, **kw|
+              define_method(mth_name) do |reqs, metadata = {}|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                client_streamer(route, reqs, marshal, unmarshal, **kw)
+                client_streamer(route, reqs, marshal, unmarshal, metadata)
               end
             elsif desc.server_streamer?
-              define_method(mth_name) do |req, **kw, &blk|
+              define_method(mth_name) do |req, metadata = {}, &blk|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                server_streamer(route, req, marshal, unmarshal, **kw, &blk)
+                server_streamer(route, req, marshal, unmarshal, metadata, &blk)
               end
             else  # is a bidi_stream
-              define_method(mth_name) do |reqs, **kw, &blk|
+              define_method(mth_name) do |reqs, metadata = {}, &blk|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                bidi_streamer(route, reqs, marshal, unmarshal, **kw, &blk)
+                bidi_streamer(route, reqs, marshal, unmarshal, metadata, &blk)
               end
             end
           end
diff --git a/src/ruby/pb/test/client.rb b/src/ruby/pb/test/client.rb
index 95b059a18e..b6695482a2 100755
--- a/src/ruby/pb/test/client.rb
+++ b/src/ruby/pb/test/client.rb
@@ -114,7 +114,9 @@ def create_stub(opts)
   if opts.secure
     creds = ssl_creds(opts.use_test_ca)
     stub_opts = {
-      GRPC::Core::Channel::SSL_TARGET => opts.host_override
+      channel_args: {
+        GRPC::Core::Channel::SSL_TARGET => opts.host_override
+      }
     }
 
     # Add service account creds if specified
@@ -315,7 +317,8 @@ class NamedTests
   def timeout_on_sleeping_server
     msg_sizes = [[27_182, 31_415]]
     ppp = PingPongPlayer.new(msg_sizes)
-    resps = @stub.full_duplex_call(ppp.each_item, timeout: 0.001)
+    deadline = GRPC::Core::TimeConsts::from_relative_time(0.001)
+    resps = @stub.full_duplex_call(ppp.each_item, deadline: deadline)
     resps.each { |r| ppp.queue.push(r) }
     fail 'Should have raised GRPC::BadStatus(DEADLINE_EXCEEDED)'
   rescue GRPC::BadStatus => e
diff --git a/src/ruby/qps/client.rb b/src/ruby/qps/client.rb
index d04f707479..917b01271e 100644
--- a/src/ruby/qps/client.rb
+++ b/src/ruby/qps/client.rb
@@ -66,8 +66,10 @@ class BenchmarkClient
         cred = GRPC::Core::ChannelCredentials.new()
       end
       if config.security_params.server_host_override
-        opts[GRPC::Core::Channel::SSL_TARGET] =
+        channel_args = {}
+        channel_args[GRPC::Core::Channel::SSL_TARGET] =
           config.security_params.server_host_override
+        opts[:channel_args] = channel_args
       end
     else
       cred = :this_channel_is_insecure
diff --git a/src/ruby/qps/server.rb b/src/ruby/qps/server.rb
index f05fbbdaaf..52a89ce847 100644
--- a/src/ruby/qps/server.rb
+++ b/src/ruby/qps/server.rb
@@ -75,13 +75,14 @@ class BenchmarkServer
     @port = @server.add_http2_port("0.0.0.0:" + port.to_s, cred)
     @server.handle(BenchmarkServiceImpl.new)
     @start_time = Time.now
-    Thread.new {
+    t = Thread.new {
       @server.run
     }
+    t.abort_on_exception
   end
   def mark(reset)
     s = Grpc::Testing::ServerStats.new(time_elapsed:
-                                         (Time.now-@start_time).to_f)
+                                       (Time.now-@start_time).to_f)
     @start_time = Time.now if reset
     s
   end
diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb
index c0181e21fb..d9c9780c93 100644
--- a/src/ruby/spec/generic/active_call_spec.rb
+++ b/src/ruby/spec/generic/active_call_spec.rb
@@ -159,9 +159,10 @@ describe GRPC::ActiveCall do
   end
 
   describe '#client_invoke' do
-    it 'sends keywords as metadata to the server when the are present' do
+    it 'sends metadata to the server when present' do
       call = make_test_call
-      ActiveCall.client_invoke(call, @client_queue, k1: 'v1', k2: 'v2')
+      metadata = { k1: 'v1', k2: 'v2' }
+      ActiveCall.client_invoke(call, @client_queue, metadata)
       recvd_rpc =  @server.request_call(@server_queue, @server_tag, deadline)
       recvd_call = recvd_rpc.call
       expect(recvd_call).to_not be_nil
diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb
index dd8e2e9f7a..168e7fb791 100644
--- a/src/ruby/spec/generic/client_stub_spec.rb
+++ b/src/ruby/spec/generic/client_stub_spec.rb
@@ -68,15 +68,7 @@ describe 'ClientStub' do
   describe '#new' do
     let(:fake_host) { 'localhost:0' }
     it 'can be created from a host and args' do
-      opts = { a_channel_arg: 'an_arg' }
-      blk = proc do
-        GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'can be created with a default deadline' do
-      opts = { a_channel_arg: 'an_arg', deadline: 5 }
+      opts = { channel_args: { a_channel_arg: 'an_arg' } }
       blk = proc do
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
@@ -84,7 +76,10 @@ describe 'ClientStub' do
     end
 
     it 'can be created with an channel override' do
-      opts = { a_channel_arg: 'an_arg', channel_override: @ch }
+      opts = {
+        channel_args: { a_channel_arg: 'an_arg' },
+        channel_override: @ch
+      }
       blk = proc do
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
@@ -93,7 +88,10 @@ describe 'ClientStub' do
 
     it 'cannot be created with a bad channel override' do
       blk = proc do
-        opts = { a_channel_arg: 'an_arg', channel_override: Object.new }
+        opts = {
+          channel_args: { a_channel_arg: 'an_arg' },
+          channel_override: Object.new
+        }
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
       expect(&blk).to raise_error
@@ -101,7 +99,7 @@ describe 'ClientStub' do
 
     it 'cannot be created with bad credentials' do
       blk = proc do
-        opts = { a_channel_arg: 'an_arg' }
+        opts = { channel_args: { a_channel_arg: 'an_arg' } }
         GRPC::ClientStub.new(fake_host, @cq, Object.new, **opts)
       end
       expect(&blk).to raise_error
@@ -111,8 +109,10 @@ describe 'ClientStub' do
       certs = load_test_certs
       blk = proc do
         opts = {
-          GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr',
-          a_channel_arg: 'an_arg'
+          channel_args: {
+            GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr',
+            a_channel_arg: 'an_arg'
+          }
         }
         creds = GRPC::Core::ChannelCredentials.new(certs[0], nil, nil)
         GRPC::ClientStub.new(fake_host, @cq, creds,  **opts)
@@ -172,7 +172,7 @@ describe 'ClientStub' do
     describe 'without a call operation' do
       def get_response(stub)
         stub.request_response(@method, @sent_msg, noop, noop,
-                              k1: 'v1', k2: 'v2')
+                              metadata: { k1: 'v1', k2: 'v2' })
       end
 
       it_behaves_like 'request response'
@@ -181,7 +181,8 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_response(stub)
         op = stub.request_response(@method, @sent_msg, noop, noop,
-                                   return_op: true, k1: 'v1', k2: 'v2')
+                                   return_op: true,
+                                   metadata: { k1: 'v1', k2: 'v2' })
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         op.execute
       end
@@ -196,7 +197,7 @@ describe 'ClientStub' do
         server_port = create_test_server
         host = "localhost:#{server_port}"
         @stub = GRPC::ClientStub.new(host, @cq, :this_channel_is_insecure)
-        @options = { k1: 'v1', k2: 'v2' }
+        @metadata = { k1: 'v1', k2: 'v2' }
         @sent_msgs = Array.new(3) { |i| 'msg_' + (i + 1).to_s }
         @resp = 'a_reply'
       end
@@ -208,7 +209,7 @@ describe 'ClientStub' do
       end
 
       it 'should send metadata to the server ok' do
-        th = run_client_streamer(@sent_msgs, @resp, @pass, @options)
+        th = run_client_streamer(@sent_msgs, @resp, @pass, **@metadata)
         expect(get_response(@stub)).to eq(@resp)
         th.join
       end
@@ -221,7 +222,7 @@ describe 'ClientStub' do
       end
 
       it 'should raise ArgumentError if metadata contains invalid values' do
-        @options.merge!(k3: 3)
+        @metadata.merge!(k3: 3)
         expect do
           get_response(@stub)
         end.to raise_error(ArgumentError,
@@ -231,7 +232,8 @@ describe 'ClientStub' do
 
     describe 'without a call operation' do
       def get_response(stub)
-        stub.client_streamer(@method, @sent_msgs, noop, noop, @options)
+        stub.client_streamer(@method, @sent_msgs, noop, noop,
+                             metadata: @metadata)
       end
 
       it_behaves_like 'client streaming'
@@ -240,7 +242,7 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_response(stub)
         op = stub.client_streamer(@method, @sent_msgs, noop, noop,
-                                  @options.merge(return_op: true))
+                                  return_op: true, metadata: @metadata)
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         op.execute
       end
@@ -290,7 +292,7 @@ describe 'ClientStub' do
     describe 'without a call operation' do
       def get_responses(stub)
         e = stub.server_streamer(@method, @sent_msg, noop, noop,
-                                 k1: 'v1', k2: 'v2')
+                                 metadata: { k1: 'v1', k2: 'v2' })
         expect(e).to be_a(Enumerator)
         e
       end
@@ -301,7 +303,8 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_responses(stub)
         op = stub.server_streamer(@method, @sent_msg, noop, noop,
-                                  return_op: true, k1: 'v1', k2: 'v2')
+                                  return_op: true,
+                                  metadata: { k1: 'v1', k2: 'v2' })
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         e = op.execute
         expect(e).to be_a(Enumerator)
@@ -383,7 +386,7 @@ describe 'ClientStub' do
         stub = GRPC::ClientStub.new(@host, @cq, :this_channel_is_insecure)
         blk = proc do
           e = stub.bidi_streamer(@method, @sent_msgs, noop, noop,
-                                 timeout: 0.001)
+                                 deadline: from_relative_time(0.001))
           e.collect { |r| r }
         end
         expect(&blk).to raise_error GRPC::BadStatus, /Deadline Exceeded/
diff --git a/src/ruby/spec/generic/rpc_desc_spec.rb b/src/ruby/spec/generic/rpc_desc_spec.rb
index 083632a080..d2080b7ca2 100644
--- a/src/ruby/spec/generic/rpc_desc_spec.rb
+++ b/src/ruby/spec/generic/rpc_desc_spec.rb
@@ -56,14 +56,14 @@ describe GRPC::RpcDesc do
     it 'sends the specified status if BadStatus is raised' do
       expect(@call).to receive(:remote_read).once.and_return(Object.new)
       expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                       {})
+                                                       metadata: {})
       this_desc.run_server_method(@call, method(:bad_status))
     end
 
     it 'sends status UNKNOWN if other StandardErrors are raised' do
       expect(@call).to receive(:remote_read).once.and_return(Object.new)
       expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason,
-                                                        false, {})
+                                                        false, metadata: {})
       this_desc.run_server_method(@call, method(:other_error))
     end
 
@@ -93,7 +93,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).once.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         this_desc.run_server_method(@call, method(:fake_reqresp))
       end
     end
@@ -106,13 +106,13 @@ describe GRPC::RpcDesc do
 
       it 'sends the specified status if BadStatus is raised' do
         expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                         {})
+                                                         metadata: {})
         @client_streamer.run_server_method(@call, method(:bad_status_alt))
       end
 
       it 'sends status UNKNOWN if other StandardErrors are raised' do
-        expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason,
-                                                          false, {})
+        expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason,
+                                                         false, metadata: {})
         @client_streamer.run_server_method(@call, method(:other_error_alt))
       end
 
@@ -128,7 +128,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).once.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @client_streamer.run_server_method(@call, method(:fake_clstream))
       end
     end
@@ -148,7 +148,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).twice.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @server_streamer.run_server_method(@call, method(:fake_svstream))
       end
     end
@@ -165,14 +165,14 @@ describe GRPC::RpcDesc do
         e = GRPC::BadStatus.new(@bs_code, 'NOK')
         expect(@call).to receive(:run_server_bidi).and_raise(e)
         expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                         {})
+                                                         metadata: {})
         @bidi_streamer.run_server_method(@call, method(:bad_status_alt))
       end
 
       it 'sends status UNKNOWN if other StandardErrors are raised' do
         expect(@call).to receive(:run_server_bidi).and_raise(StandardError)
         expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason,
-                                                         false, {})
+                                                         false, metadata: {})
         @bidi_streamer.run_server_method(@call, method(:other_error_alt))
       end
 
@@ -180,7 +180,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:run_server_bidi)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @bidi_streamer.run_server_method(@call, method(:fake_bidistream))
       end
     end
diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb
index 2a42736237..943502cea2 100644
--- a/src/ruby/spec/generic/rpc_server_spec.rb
+++ b/src/ruby/spec/generic/rpc_server_spec.rb
@@ -99,7 +99,7 @@ class FailingService
   end
 
   def an_rpc(_req, _call)
-    fail GRPC::BadStatus.new(@code, @details, **@md)
+    fail GRPC::BadStatus.new(@code, @details, @md)
   end
 end
 
@@ -137,24 +137,11 @@ describe GRPC::RpcServer do
     @noop = proc { |x| x }
 
     @server_queue = GRPC::Core::CompletionQueue.new
-    server_host = '0.0.0.0:0'
-    @server = GRPC::Core::Server.new(@server_queue, nil)
-    server_port = @server.add_http2_port(server_host, :this_port_is_insecure)
-    @host = "localhost:#{server_port}"
-    @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
   end
 
   describe '#new' do
     it 'can be created with just some args' do
-      opts = { a_channel_arg: 'an_arg' }
-      blk = proc do
-        RpcServer.new(**opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'can be created with a default deadline' do
-      opts = { a_channel_arg: 'an_arg', deadline: 5 }
+      opts = { server_args: { a_channel_arg: 'an_arg' } }
       blk = proc do
         RpcServer.new(**opts)
       end
@@ -163,7 +150,7 @@ describe GRPC::RpcServer do
 
     it 'can be created with a completion queue override' do
       opts = {
-        a_channel_arg: 'an_arg',
+        server_args: { a_channel_arg: 'an_arg' },
         completion_queue_override: @server_queue
       }
       blk = proc do
@@ -175,7 +162,7 @@ describe GRPC::RpcServer do
     it 'cannot be created with a bad completion queue override' do
       blk = proc do
         opts = {
-          a_channel_arg: 'an_arg',
+          server_args: { a_channel_arg: 'an_arg' },
           completion_queue_override: Object.new
         }
         RpcServer.new(**opts)
@@ -186,38 +173,20 @@ describe GRPC::RpcServer do
     it 'cannot be created with invalid ServerCredentials' do
       blk = proc do
         opts = {
-          a_channel_arg: 'an_arg',
+          server_args: { a_channel_arg: 'an_arg' },
           creds: Object.new
         }
         RpcServer.new(**opts)
       end
       expect(&blk).to raise_error
     end
-
-    it 'can be created with a server override' do
-      opts = { a_channel_arg: 'an_arg', server_override: @server }
-      blk = proc do
-        RpcServer.new(**opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'cannot be created with a bad server override' do
-      blk = proc do
-        opts = {
-          a_channel_arg: 'an_arg',
-          server_override: Object.new
-        }
-        RpcServer.new(**opts)
-      end
-      expect(&blk).to raise_error
-    end
   end
 
   describe '#stopped?' do
     before(:each) do
-      opts = { a_channel_arg: 'an_arg', poll_period: 1.5 }
+      opts = { server_args: { a_channel_arg: 'an_arg' }, poll_period: 1.5 }
       @srv = RpcServer.new(**opts)
+      @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
     end
 
     it 'starts out false' do
@@ -245,28 +214,30 @@ describe GRPC::RpcServer do
 
   describe '#running?' do
     it 'starts out false' do
-      opts = { a_channel_arg: 'an_arg', server_override: @server }
+      opts = {
+        server_args: { a_channel_arg: 'an_arg' }
+      }
       r = RpcServer.new(**opts)
       expect(r.running?).to be(false)
     end
 
     it 'is false if run is called with no services registered', server: true do
       opts = {
-        a_channel_arg: 'an_arg',
-        poll_period: 2,
-        server_override: @server
+        server_args: { a_channel_arg: 'an_arg' },
+        poll_period: 2
       }
       r = RpcServer.new(**opts)
+      r.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
       expect { r.run }.to raise_error(RuntimeError)
     end
 
     it 'is true after run is called with a registered service' do
       opts = {
-        a_channel_arg: 'an_arg',
-        poll_period: 2.5,
-        server_override: @server
+        server_args: { a_channel_arg: 'an_arg' },
+        poll_period: 2.5
       }
       r = RpcServer.new(**opts)
+      r.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
       r.handle(EchoService)
       t = Thread.new { r.run }
       r.wait_till_running
@@ -278,8 +249,9 @@ describe GRPC::RpcServer do
 
   describe '#handle' do
     before(:each) do
-      @opts = { a_channel_arg: 'an_arg', poll_period: 1 }
+      @opts = { server_args: { a_channel_arg: 'an_arg' }, poll_period: 1 }
       @srv = RpcServer.new(**@opts)
+      @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
     end
 
     it 'raises if #run has already been called' do
@@ -322,11 +294,13 @@ describe GRPC::RpcServer do
     context 'with no connect_metadata' do
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1
         }
         @srv = RpcServer.new(**server_opts)
+        server_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @host = "localhost:#{server_port}"
+        @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
       end
 
       it 'should return NOT_FOUND status on unknown methods', server: true do
@@ -383,7 +357,8 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        expect(stub.an_rpc(req, k1: 'v1', k2: 'v2')).to be_a(EchoMsg)
+        expect(stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }))
+          .to be_a(EchoMsg)
         wanted_md = [{ 'k1' => 'v1', 'k2' => 'v2' }]
         check_md(wanted_md, service.received_md)
         @srv.stop
@@ -397,8 +372,11 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
-        timeout = service.delay + 1.0 # wait for long enough
-        resp = stub.an_rpc(req, timeout: timeout, k1: 'v1', k2: 'v2')
+        timeout = service.delay + 1.0
+        deadline = GRPC::Core::TimeConsts.from_relative_time(timeout)
+        resp = stub.an_rpc(req,
+                           deadline: deadline,
+                           metadata: { k1: 'v1', k2: 'v2' })
         expect(resp).to be_a(EchoMsg)
         wanted_md = [{ 'k1' => 'v1', 'k2' => 'v2' }]
         check_md(wanted_md, service.received_md)
@@ -413,7 +391,7 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        op = stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }, return_op: true)
         Thread.new do  # cancel the call
           sleep 0.1
           op.cancel
@@ -443,8 +421,7 @@ describe GRPC::RpcServer do
 
       it 'should return RESOURCE_EXHAUSTED on too many jobs', server: true do
         opts = {
-          a_channel_arg: 'an_arg',
-          server_override: @server,
+          server_args: { a_channel_arg: 'an_arg' },
           completion_queue_override: @server_queue,
           pool_size: 1,
           poll_period: 1,
@@ -452,6 +429,8 @@ describe GRPC::RpcServer do
         }
         alt_srv = RpcServer.new(**opts)
         alt_srv.handle(SlowService)
+        alt_port = alt_srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        alt_host = "0.0.0.0:#{alt_port}"
         t = Thread.new { alt_srv.run }
         alt_srv.wait_till_running
         req = EchoMsg.new
@@ -460,7 +439,7 @@ describe GRPC::RpcServer do
         one_failed_as_unavailable = false
         n.times do
           threads << Thread.new do
-            stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
+            stub = SlowStub.new(alt_host, :this_channel_is_insecure)
             begin
               stub.an_rpc(req)
             rescue GRPC::BadStatus => e
@@ -487,12 +466,13 @@ describe GRPC::RpcServer do
       end
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1,
           connect_md_proc: test_md_proc
         }
         @srv = RpcServer.new(**server_opts)
+        alt_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @alt_host = "0.0.0.0:#{alt_port}"
       end
 
       it 'should send connect metadata to the client', server: true do
@@ -501,8 +481,8 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        stub = EchoStub.new(@alt_host, :this_channel_is_insecure)
+        op = stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }, return_op: true)
         expect(op.metadata).to be nil
         expect(op.execute).to be_a(EchoMsg)
         wanted_md = {
@@ -522,11 +502,12 @@ describe GRPC::RpcServer do
     context 'with trailing metadata' do
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1
         }
         @srv = RpcServer.new(**server_opts)
+        alt_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @alt_host = "0.0.0.0:#{alt_port}"
       end
 
       it 'should be added to BadStatus when requests fail', server: true do
@@ -535,7 +516,7 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = FailingStub.new(@host, :this_channel_is_insecure, **client_opts)
+        stub = FailingStub.new(@alt_host, :this_channel_is_insecure)
         blk = proc { stub.an_rpc(req) }
 
         # confirm it raise the expected error
@@ -560,8 +541,8 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        stub = EchoStub.new(@alt_host, :this_channel_is_insecure)
+        op = stub.an_rpc(req, return_op: true, metadata: { k1: 'v1', k2: 'v2' })
         expect(op.metadata).to be nil
         expect(op.execute).to be_a(EchoMsg)
         expect(op.metadata).to eq(wanted_trailers)
diff --git a/src/ruby/spec/pb/health/checker_spec.rb b/src/ruby/spec/pb/health/checker_spec.rb
index 5523347f45..f3d121a31e 100644
--- a/src/ruby/spec/pb/health/checker_spec.rb
+++ b/src/ruby/spec/pb/health/checker_spec.rb
@@ -170,17 +170,15 @@ describe Grpc::Health::Checker do
     before(:each) do
       @server_queue = GRPC::Core::CompletionQueue.new
       server_host = '0.0.0.0:0'
-      @server = GRPC::Core::Server.new(@server_queue, nil)
-      server_port = @server.add_http2_port(server_host, :this_port_is_insecure)
-      @host = "localhost:#{server_port}"
-      @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
       @client_opts = { channel_override: @ch }
       server_opts = {
-        server_override: @server,
         completion_queue_override: @server_queue,
         poll_period: 1
       }
       @srv = RpcServer.new(**server_opts)
+      server_port = @srv.add_http2_port(server_host, :this_port_is_insecure)
+      @host = "localhost:#{server_port}"
+      @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
     end
 
     after(:each) do