Merge branch 'master' of github.com:grpc/grpc into lb_pollset_propagation

58 files changed, 2567 insertions, 3720 deletions

diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index 893cf0700e..569be4dc28 100644
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -132,7 +132,8 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
   for (size_t i = 0; i < a->num_args; i++) {
     args[i] = &a->args[i];
   }
-  qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
+  if (a->num_args > 1)
+    qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
 
   grpc_channel_args *b = gpr_malloc(sizeof(grpc_channel_args));
   b->num_args = a->num_args;
diff --git a/src/core/lib/compression/compression_algorithm.c b/src/core/lib/compression/compression_algorithm.c
index 7039364b7b..820871d579 100644
--- a/src/core/lib/compression/compression_algorithm.c
+++ b/src/core/lib/compression/compression_algorithm.c
@@ -199,5 +199,6 @@ void grpc_compression_options_disable_algorithm(
 int grpc_compression_options_is_algorithm_enabled(
     const grpc_compression_options *opts,
     grpc_compression_algorithm algorithm) {
+  if (algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) return 0;
   return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
 }
diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c
index ea4bff30d4..5590928968 100644
--- a/src/core/lib/http/httpcli_security_connector.c
+++ b/src/core/lib/http/httpcli_security_connector.c
@@ -38,7 +38,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
 
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
deleted file mode 100644
index aeb6e28665..0000000000
--- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
+++ /dev/null
@@ -1,1934 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/* This file will be removed shortly: it's here to keep refactoring
- * steps simple and auditable.
- * It's the combination of the old files:
- *  - fd_posix.{h,c}
- *  - pollset_posix.{h,c}
- *  - pullset_multipoller_with_{poll,epoll}.{h,c}
- * The new version will be split into:
- *  - ev_poll_posix.{h,c}
- *  - ev_epoll_posix.{h,c}
- */
-
-#include <grpc/support/port_platform.h>
-
-#ifdef GPR_POSIX_SOCKET
-
-#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <poll.h>
-#include <string.h>
-#include <sys/socket.h>
-#include <unistd.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/tls.h>
-#include <grpc/support/useful.h>
-
-#include "src/core/lib/iomgr/iomgr_internal.h"
-#include "src/core/lib/iomgr/wakeup_fd_posix.h"
-#include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/block_annotate.h"
-
-/*******************************************************************************
- * FD declarations
- */
-
-typedef struct grpc_fd_watcher {
-  struct grpc_fd_watcher *next;
-  struct grpc_fd_watcher *prev;
-  grpc_pollset *pollset;
-  grpc_pollset_worker *worker;
-  grpc_fd *fd;
-} grpc_fd_watcher;
-
-struct grpc_fd {
-  int fd;
-  /* refst format:
-     bit0:   1=active/0=orphaned
-     bit1-n: refcount
-     meaning that mostly we ref by two to avoid altering the orphaned bit,
-     and just unref by 1 when we're ready to flag the object as orphaned */
-  gpr_atm refst;
-
-  gpr_mu mu;
-  int shutdown;
-  int closed;
-  int released;
-
-  /* The watcher list.
-
-     The following watcher related fields are protected by watcher_mu.
-
-     An fd_watcher is an ephemeral object created when an fd wants to
-     begin polling, and destroyed after the poll.
-
-     It denotes the fd's interest in whether to read poll or write poll
-     or both or neither on this fd.
-
-     If a watcher is asked to poll for reads or writes, the read_watcher
-     or write_watcher fields are set respectively. A watcher may be asked
-     to poll for both, in which case both fields will be set.
-
-     read_watcher and write_watcher may be NULL if no watcher has been
-     asked to poll for reads or writes.
-
-     If an fd_watcher is not asked to poll for reads or writes, it's added
-     to a linked list of inactive watchers, rooted at inactive_watcher_root.
-     If at a later time there becomes need of a poller to poll, one of
-     the inactive pollers may be kicked out of their poll loops to take
-     that responsibility. */
-  grpc_fd_watcher inactive_watcher_root;
-  grpc_fd_watcher *read_watcher;
-  grpc_fd_watcher *write_watcher;
-
-  grpc_closure *read_closure;
-  grpc_closure *write_closure;
-
-  struct grpc_fd *freelist_next;
-
-  grpc_closure *on_done_closure;
-
-  grpc_iomgr_object iomgr_object;
-};
-
-/* Begin polling on an fd.
-   Registers that the given pollset is interested in this fd - so that if read
-   or writability interest changes, the pollset can be kicked to pick up that
-   new interest.
-   Return value is:
-     (fd_needs_read? read_mask : 0) | (fd_needs_write? write_mask : 0)
-   i.e. a combination of read_mask and write_mask determined by the fd's current
-   interest in said events.
-   Polling strategies that do not need to alter their behavior depending on the
-   fd's current interest (such as epoll) do not need to call this function.
-   MUST NOT be called with a pollset lock taken */
-static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
-                              grpc_pollset_worker *worker, uint32_t read_mask,
-                              uint32_t write_mask, grpc_fd_watcher *rec);
-/* Complete polling previously started with fd_begin_poll
-   MUST NOT be called with a pollset lock taken
-   if got_read or got_write are 1, also does the become_{readable,writable} as
-   appropriate. */
-static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
-
-/* Return 1 if this fd is orphaned, 0 otherwise */
-static bool fd_is_orphaned(grpc_fd *fd);
-
-/* Reference counting for fds */
-/*#define GRPC_FD_REF_COUNT_DEBUG*/
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line);
-static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
-                     int line);
-#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__)
-#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__)
-#else
-static void fd_ref(grpc_fd *fd);
-static void fd_unref(grpc_fd *fd);
-#define GRPC_FD_REF(fd, reason) fd_ref(fd)
-#define GRPC_FD_UNREF(fd, reason) fd_unref(fd)
-#endif
-
-static void fd_global_init(void);
-static void fd_global_shutdown(void);
-
-#define CLOSURE_NOT_READY ((grpc_closure *)0)
-#define CLOSURE_READY ((grpc_closure *)1)
-
-/*******************************************************************************
- * pollset declarations
- */
-
-typedef struct grpc_pollset_vtable grpc_pollset_vtable;
-
-typedef struct grpc_cached_wakeup_fd {
-  grpc_wakeup_fd fd;
-  struct grpc_cached_wakeup_fd *next;
-} grpc_cached_wakeup_fd;
-
-struct grpc_pollset_worker {
-  grpc_cached_wakeup_fd *wakeup_fd;
-  int reevaluate_polling_on_wakeup;
-  int kicked_specifically;
-  struct grpc_pollset_worker *next;
-  struct grpc_pollset_worker *prev;
-};
-
-struct grpc_pollset {
-  /* pollsets under posix can mutate representation as fds are added and
-     removed.
-     For example, we may choose a poll() based implementation on linux for
-     few fds, and an epoll() based implementation for many fds */
-  const grpc_pollset_vtable *vtable;
-  gpr_mu mu;
-  grpc_pollset_worker root_worker;
-  int in_flight_cbs;
-  int shutting_down;
-  int called_shutdown;
-  int kicked_without_pollers;
-  grpc_closure *shutdown_done;
-  grpc_closure_list idle_jobs;
-  union {
-    int fd;
-    void *ptr;
-  } data;
-  /* Local cache of eventfds for workers */
-  grpc_cached_wakeup_fd *local_wakeup_cache;
-};
-
-struct grpc_pollset_vtable {
-  void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                 struct grpc_fd *fd, int and_unlock_pollset);
-  void (*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                grpc_pollset_worker *worker,
-                                gpr_timespec deadline, gpr_timespec now);
-  void (*finish_shutdown)(grpc_pollset *pollset);
-  void (*destroy)(grpc_pollset *pollset);
-};
-
-/* Add an fd to a pollset */
-static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           struct grpc_fd *fd);
-
-static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd);
-
-/* Convert a timespec to milliseconds:
-   - very small or negative poll times are clamped to zero to do a
-     non-blocking poll (which becomes spin polling)
-   - other small values are rounded up to one millisecond
-   - longer than a millisecond polls are rounded up to the next nearest
-     millisecond to avoid spinning
-   - infinite timeouts are converted to -1 */
-static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
-                                           gpr_timespec now);
-
-/* Allow kick to wakeup the currently polling worker */
-#define GRPC_POLLSET_CAN_KICK_SELF 1
-/* Force the wakee to repoll when awoken */
-#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2
-/* As per pollset_kick, with an extended set of flags (defined above)
-   -- mostly for fd_posix's use. */
-static void pollset_kick_ext(grpc_pollset *p,
-                             grpc_pollset_worker *specific_worker,
-                             uint32_t flags);
-
-/* turn a pollset into a multipoller: platform specific */
-typedef void (*platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx,
-                                                 grpc_pollset *pollset,
-                                                 struct grpc_fd **fds,
-                                                 size_t fd_count);
-static platform_become_multipoller_type platform_become_multipoller;
-
-/* Return 1 if the pollset has active threads in pollset_work (pollset must
- * be locked) */
-static int pollset_has_workers(grpc_pollset *pollset);
-
-static void remove_fd_from_all_epoll_sets(int fd);
-
-/*******************************************************************************
- * pollset_set definitions
- */
-
-struct grpc_pollset_set {
-  gpr_mu mu;
-
-  size_t pollset_count;
-  size_t pollset_capacity;
-  grpc_pollset **pollsets;
-
-  size_t pollset_set_count;
-  size_t pollset_set_capacity;
-  struct grpc_pollset_set **pollset_sets;
-
-  size_t fd_count;
-  size_t fd_capacity;
-  grpc_fd **fds;
-};
-
-/*******************************************************************************
- * fd_posix.c
- */
-
-/* We need to keep a freelist not because of any concerns of malloc performance
- * but instead so that implementations with multiple threads in (for example)
- * epoll_wait deal with the race between pollset removal and incoming poll
- * notifications.
- *
- * The problem is that the poller ultimately holds a reference to this
- * object, so it is very difficult to know when is safe to free it, at least
- * without some expensive synchronization.
- *
- * If we keep the object freelisted, in the worst case losing this race just
- * becomes a spurious read notification on a reused fd.
- */
-/* TODO(klempner): We could use some form of polling generation count to know
- * when these are safe to free. */
-/* TODO(klempner): Consider disabling freelisting if we don't have multiple
- * threads in poll on the same fd */
-/* TODO(klempner): Batch these allocations to reduce fragmentation */
-static grpc_fd *fd_freelist = NULL;
-static gpr_mu fd_freelist_mu;
-
-static void freelist_fd(grpc_fd *fd) {
-  gpr_mu_lock(&fd_freelist_mu);
-  fd->freelist_next = fd_freelist;
-  fd_freelist = fd;
-  grpc_iomgr_unregister_object(&fd->iomgr_object);
-  gpr_mu_unlock(&fd_freelist_mu);
-}
-
-static grpc_fd *alloc_fd(int fd) {
-  grpc_fd *r = NULL;
-  gpr_mu_lock(&fd_freelist_mu);
-  if (fd_freelist != NULL) {
-    r = fd_freelist;
-    fd_freelist = fd_freelist->freelist_next;
-  }
-  gpr_mu_unlock(&fd_freelist_mu);
-  if (r == NULL) {
-    r = gpr_malloc(sizeof(grpc_fd));
-    gpr_mu_init(&r->mu);
-  }
-
-  gpr_mu_lock(&r->mu);
-  gpr_atm_rel_store(&r->refst, 1);
-  r->shutdown = 0;
-  r->read_closure = CLOSURE_NOT_READY;
-  r->write_closure = CLOSURE_NOT_READY;
-  r->fd = fd;
-  r->inactive_watcher_root.next = r->inactive_watcher_root.prev =
-      &r->inactive_watcher_root;
-  r->freelist_next = NULL;
-  r->read_watcher = r->write_watcher = NULL;
-  r->on_done_closure = NULL;
-  r->closed = 0;
-  r->released = 0;
-  gpr_mu_unlock(&r->mu);
-  return r;
-}
-
-static void destroy(grpc_fd *fd) {
-  gpr_mu_destroy(&fd->mu);
-  gpr_free(fd);
-}
-
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__)
-#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__)
-static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file,
-                   int line) {
-  gpr_log(GPR_DEBUG, "FD %d %p   ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
-          gpr_atm_no_barrier_load(&fd->refst),
-          gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line);
-#else
-#define REF_BY(fd, n, reason) ref_by(fd, n)
-#define UNREF_BY(fd, n, reason) unref_by(fd, n)
-static void ref_by(grpc_fd *fd, int n) {
-#endif
-  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
-}
-
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file,
-                     int line) {
-  gpr_atm old;
-  gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
-          gpr_atm_no_barrier_load(&fd->refst),
-          gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line);
-#else
-static void unref_by(grpc_fd *fd, int n) {
-  gpr_atm old;
-#endif
-  old = gpr_atm_full_fetch_add(&fd->refst, -n);
-  if (old == n) {
-    freelist_fd(fd);
-  } else {
-    GPR_ASSERT(old > n);
-  }
-}
-
-static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); }
-
-static void fd_global_shutdown(void) {
-  gpr_mu_lock(&fd_freelist_mu);
-  gpr_mu_unlock(&fd_freelist_mu);
-  while (fd_freelist != NULL) {
-    grpc_fd *fd = fd_freelist;
-    fd_freelist = fd_freelist->freelist_next;
-    destroy(fd);
-  }
-  gpr_mu_destroy(&fd_freelist_mu);
-}
-
-static grpc_fd *fd_create(int fd, const char *name) {
-  grpc_fd *r = alloc_fd(fd);
-  char *name2;
-  gpr_asprintf(&name2, "%s fd=%d", name, fd);
-  grpc_iomgr_register_object(&r->iomgr_object, name2);
-  gpr_free(name2);
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-  gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name);
-#endif
-  return r;
-}
-
-static bool fd_is_orphaned(grpc_fd *fd) {
-  return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
-}
-
-static void pollset_kick_locked(grpc_fd_watcher *watcher) {
-  gpr_mu_lock(&watcher->pollset->mu);
-  GPR_ASSERT(watcher->worker);
-  pollset_kick_ext(watcher->pollset, watcher->worker,
-                   GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
-  gpr_mu_unlock(&watcher->pollset->mu);
-}
-
-static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
-  if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) {
-    pollset_kick_locked(fd->inactive_watcher_root.next);
-  } else if (fd->read_watcher) {
-    pollset_kick_locked(fd->read_watcher);
-  } else if (fd->write_watcher) {
-    pollset_kick_locked(fd->write_watcher);
-  }
-}
-
-static void wake_all_watchers_locked(grpc_fd *fd) {
-  grpc_fd_watcher *watcher;
-  for (watcher = fd->inactive_watcher_root.next;
-       watcher != &fd->inactive_watcher_root; watcher = watcher->next) {
-    pollset_kick_locked(watcher);
-  }
-  if (fd->read_watcher) {
-    pollset_kick_locked(fd->read_watcher);
-  }
-  if (fd->write_watcher && fd->write_watcher != fd->read_watcher) {
-    pollset_kick_locked(fd->write_watcher);
-  }
-}
-
-static int has_watchers(grpc_fd *fd) {
-  return fd->read_watcher != NULL || fd->write_watcher != NULL ||
-         fd->inactive_watcher_root.next != &fd->inactive_watcher_root;
-}
-
-static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  fd->closed = 1;
-  if (!fd->released) {
-    close(fd->fd);
-  } else {
-    remove_fd_from_all_epoll_sets(fd->fd);
-  }
-  grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, true, NULL);
-}
-
-static int fd_wrapped_fd(grpc_fd *fd) {
-  if (fd->released || fd->closed) {
-    return -1;
-  } else {
-    return fd->fd;
-  }
-}
-
-static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                      grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  fd->on_done_closure = on_done;
-  fd->released = release_fd != NULL;
-  if (!fd->released) {
-    shutdown(fd->fd, SHUT_RDWR);
-  } else {
-    *release_fd = fd->fd;
-  }
-  gpr_mu_lock(&fd->mu);
-  REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
-  if (!has_watchers(fd)) {
-    close_fd_locked(exec_ctx, fd);
-  } else {
-    wake_all_watchers_locked(fd);
-  }
-  gpr_mu_unlock(&fd->mu);
-  UNREF_BY(fd, 2, reason); /* drop the reference */
-}
-
-/* increment refcount by two to avoid changing the orphan bit */
-#ifdef GRPC_FD_REF_COUNT_DEBUG
-static void fd_ref(grpc_fd *fd, const char *reason, const char *file,
-                   int line) {
-  ref_by(fd, 2, reason, file, line);
-}
-
-static void fd_unref(grpc_fd *fd, const char *reason, const char *file,
-                     int line) {
-  unref_by(fd, 2, reason, file, line);
-}
-#else
-static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
-
-static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
-#endif
-
-static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                             grpc_closure **st, grpc_closure *closure) {
-  if (*st == CLOSURE_NOT_READY) {
-    /* not ready ==> switch to a waiting state by setting the closure */
-    *st = closure;
-  } else if (*st == CLOSURE_READY) {
-    /* already ready ==> queue the closure to run immediately */
-    *st = CLOSURE_NOT_READY;
-    grpc_exec_ctx_enqueue(exec_ctx, closure, !fd->shutdown, NULL);
-    maybe_wake_one_watcher_locked(fd);
-  } else {
-    /* upcallptr was set to a different closure.  This is an error! */
-    gpr_log(GPR_ERROR,
-            "User called a notify_on function with a previous callback still "
-            "pending");
-    abort();
-  }
-}
-
-/* returns 1 if state becomes not ready */
-static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                            grpc_closure **st) {
-  if (*st == CLOSURE_READY) {
-    /* duplicate ready ==> ignore */
-    return 0;
-  } else if (*st == CLOSURE_NOT_READY) {
-    /* not ready, and not waiting ==> flag ready */
-    *st = CLOSURE_READY;
-    return 0;
-  } else {
-    /* waiting ==> queue closure */
-    grpc_exec_ctx_enqueue(exec_ctx, *st, !fd->shutdown, NULL);
-    *st = CLOSURE_NOT_READY;
-    return 1;
-  }
-}
-
-static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  gpr_mu_lock(&fd->mu);
-  GPR_ASSERT(!fd->shutdown);
-  fd->shutdown = 1;
-  set_ready_locked(exec_ctx, fd, &fd->read_closure);
-  set_ready_locked(exec_ctx, fd, &fd->write_closure);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                              grpc_closure *closure) {
-  gpr_mu_lock(&fd->mu);
-  notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
-                               grpc_closure *closure) {
-  gpr_mu_lock(&fd->mu);
-  notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
-                              grpc_pollset_worker *worker, uint32_t read_mask,
-                              uint32_t write_mask, grpc_fd_watcher *watcher) {
-  uint32_t mask = 0;
-  grpc_closure *cur;
-  int requested;
-  /* keep track of pollers that have requested our events, in case they change
-   */
-  GRPC_FD_REF(fd, "poll");
-
-  gpr_mu_lock(&fd->mu);
-
-  /* if we are shutdown, then don't add to the watcher set */
-  if (fd->shutdown) {
-    watcher->fd = NULL;
-    watcher->pollset = NULL;
-    watcher->worker = NULL;
-    gpr_mu_unlock(&fd->mu);
-    GRPC_FD_UNREF(fd, "poll");
-    return 0;
-  }
-
-  /* if there is nobody polling for read, but we need to, then start doing so */
-  cur = fd->read_closure;
-  requested = cur != CLOSURE_READY;
-  if (read_mask && fd->read_watcher == NULL && requested) {
-    fd->read_watcher = watcher;
-    mask |= read_mask;
-  }
-  /* if there is nobody polling for write, but we need to, then start doing so
-   */
-  cur = fd->write_closure;
-  requested = cur != CLOSURE_READY;
-  if (write_mask && fd->write_watcher == NULL && requested) {
-    fd->write_watcher = watcher;
-    mask |= write_mask;
-  }
-  /* if not polling, remember this watcher in case we need someone to later */
-  if (mask == 0 && worker != NULL) {
-    watcher->next = &fd->inactive_watcher_root;
-    watcher->prev = watcher->next->prev;
-    watcher->next->prev = watcher->prev->next = watcher;
-  }
-  watcher->pollset = pollset;
-  watcher->worker = worker;
-  watcher->fd = fd;
-  gpr_mu_unlock(&fd->mu);
-
-  return mask;
-}
-
-static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
-  int was_polling = 0;
-  int kick = 0;
-  grpc_fd *fd = watcher->fd;
-
-  if (fd == NULL) {
-    return;
-  }
-
-  gpr_mu_lock(&fd->mu);
-
-  if (watcher == fd->read_watcher) {
-    /* remove read watcher, kick if we still need a read */
-    was_polling = 1;
-    if (!got_read) {
-      kick = 1;
-    }
-    fd->read_watcher = NULL;
-  }
-  if (watcher == fd->write_watcher) {
-    /* remove write watcher, kick if we still need a write */
-    was_polling = 1;
-    if (!got_write) {
-      kick = 1;
-    }
-    fd->write_watcher = NULL;
-  }
-  if (!was_polling && watcher->worker != NULL) {
-    /* remove from inactive list */
-    watcher->next->prev = watcher->prev;
-    watcher->prev->next = watcher->next;
-  }
-  if (got_read) {
-    if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
-      kick = 1;
-    }
-  }
-  if (got_write) {
-    if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
-      kick = 1;
-    }
-  }
-  if (kick) {
-    maybe_wake_one_watcher_locked(fd);
-  }
-  if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) {
-    close_fd_locked(exec_ctx, fd);
-  }
-  gpr_mu_unlock(&fd->mu);
-
-  GRPC_FD_UNREF(fd, "poll");
-}
-
-/*******************************************************************************
- * pollset_posix.c
- */
-
-GPR_TLS_DECL(g_current_thread_poller);
-GPR_TLS_DECL(g_current_thread_worker);
-
-/** The alarm system needs to be able to wakeup 'some poller' sometimes
- *  (specifically when a new alarm needs to be triggered earlier than the next
- *  alarm 'epoch').
- *  This wakeup_fd gives us something to alert on when such a case occurs. */
-grpc_wakeup_fd grpc_global_wakeup_fd;
-
-static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
-  worker->prev->next = worker->next;
-  worker->next->prev = worker->prev;
-}
-
-static int pollset_has_workers(grpc_pollset *p) {
-  return p->root_worker.next != &p->root_worker;
-}
-
-static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) {
-  if (pollset_has_workers(p)) {
-    grpc_pollset_worker *w = p->root_worker.next;
-    remove_worker(p, w);
-    return w;
-  } else {
-    return NULL;
-  }
-}
-
-static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
-  worker->next = &p->root_worker;
-  worker->prev = worker->next->prev;
-  worker->prev->next = worker->next->prev = worker;
-}
-
-static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) {
-  worker->prev = &p->root_worker;
-  worker->next = worker->prev->next;
-  worker->prev->next = worker->next->prev = worker;
-}
-
-static void pollset_kick_ext(grpc_pollset *p,
-                             grpc_pollset_worker *specific_worker,
-                             uint32_t flags) {
-  GPR_TIMER_BEGIN("pollset_kick_ext", 0);
-
-  /* pollset->mu already held */
-  if (specific_worker != NULL) {
-    if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
-      GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0);
-      GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
-      for (specific_worker = p->root_worker.next;
-           specific_worker != &p->root_worker;
-           specific_worker = specific_worker->next) {
-        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-      }
-      p->kicked_without_pollers = 1;
-      GPR_TIMER_END("pollset_kick_ext.broadcast", 0);
-    } else if (gpr_tls_get(&g_current_thread_worker) !=
-               (intptr_t)specific_worker) {
-      GPR_TIMER_MARK("different_thread_worker", 0);
-      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
-        specific_worker->reevaluate_polling_on_wakeup = 1;
-      }
-      specific_worker->kicked_specifically = 1;
-      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-    } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) {
-      GPR_TIMER_MARK("kick_yoself", 0);
-      if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
-        specific_worker->reevaluate_polling_on_wakeup = 1;
-      }
-      specific_worker->kicked_specifically = 1;
-      grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-    }
-  } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) {
-    GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
-    GPR_TIMER_MARK("kick_anonymous", 0);
-    specific_worker = pop_front_worker(p);
-    if (specific_worker != NULL) {
-      if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) {
-        GPR_TIMER_MARK("kick_anonymous_not_self", 0);
-        push_back_worker(p, specific_worker);
-        specific_worker = pop_front_worker(p);
-        if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 &&
-            gpr_tls_get(&g_current_thread_worker) ==
-                (intptr_t)specific_worker) {
-          push_back_worker(p, specific_worker);
-          specific_worker = NULL;
-        }
-      }
-      if (specific_worker != NULL) {
-        GPR_TIMER_MARK("finally_kick", 0);
-        push_back_worker(p, specific_worker);
-        grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
-      }
-    } else {
-      GPR_TIMER_MARK("kicked_no_pollers", 0);
-      p->kicked_without_pollers = 1;
-    }
-  }
-
-  GPR_TIMER_END("pollset_kick_ext", 0);
-}
-
-static void pollset_kick(grpc_pollset *p,
-                         grpc_pollset_worker *specific_worker) {
-  pollset_kick_ext(p, specific_worker, 0);
-}
-
-/* global state management */
-
-static void pollset_global_init(void) {
-  gpr_tls_init(&g_current_thread_poller);
-  gpr_tls_init(&g_current_thread_worker);
-  grpc_wakeup_fd_init(&grpc_global_wakeup_fd);
-}
-
-static void pollset_global_shutdown(void) {
-  grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd);
-  gpr_tls_destroy(&g_current_thread_poller);
-  gpr_tls_destroy(&g_current_thread_worker);
-}
-
-static void kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); }
-
-/* main interface */
-
-static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null);
-
-static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
-  gpr_mu_init(&pollset->mu);
-  *mu = &pollset->mu;
-  pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
-  pollset->in_flight_cbs = 0;
-  pollset->shutting_down = 0;
-  pollset->called_shutdown = 0;
-  pollset->kicked_without_pollers = 0;
-  pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL;
-  pollset->local_wakeup_cache = NULL;
-  pollset->kicked_without_pollers = 0;
-  become_basic_pollset(pollset, NULL);
-}
-
-static void pollset_destroy(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
-  GPR_ASSERT(!pollset_has_workers(pollset));
-  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
-  pollset->vtable->destroy(pollset);
-  while (pollset->local_wakeup_cache) {
-    grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next;
-    grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd);
-    gpr_free(pollset->local_wakeup_cache);
-    pollset->local_wakeup_cache = next;
-  }
-  gpr_mu_destroy(&pollset->mu);
-}
-
-static void pollset_reset(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->shutting_down);
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
-  GPR_ASSERT(!pollset_has_workers(pollset));
-  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
-  pollset->vtable->destroy(pollset);
-  pollset->shutting_down = 0;
-  pollset->called_shutdown = 0;
-  pollset->kicked_without_pollers = 0;
-  become_basic_pollset(pollset, NULL);
-}
-
-static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           grpc_fd *fd) {
-  gpr_mu_lock(&pollset->mu);
-  pollset->vtable->add_fd(exec_ctx, pollset, fd, 1);
-/* the following (enabled only in debug) will reacquire and then release
-   our lock - meaning that if the unlocking flag passed to add_fd above is
-   not respected, the code will deadlock (in a way that we have a chance of
-   debugging) */
-#ifndef NDEBUG
-  gpr_mu_lock(&pollset->mu);
-  gpr_mu_unlock(&pollset->mu);
-#endif
-}
-
-static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
-  GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs));
-  pollset->vtable->finish_shutdown(pollset);
-  grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL);
-}
-
-static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                         grpc_pollset_worker **worker_hdl, gpr_timespec now,
-                         gpr_timespec deadline) {
-  grpc_pollset_worker worker;
-  *worker_hdl = &worker;
-
-  /* pollset->mu already held */
-  int added_worker = 0;
-  int locked = 1;
-  int queued_work = 0;
-  int keep_polling = 0;
-  GPR_TIMER_BEGIN("pollset_work", 0);
-  /* this must happen before we (potentially) drop pollset->mu */
-  worker.next = worker.prev = NULL;
-  worker.reevaluate_polling_on_wakeup = 0;
-  if (pollset->local_wakeup_cache != NULL) {
-    worker.wakeup_fd = pollset->local_wakeup_cache;
-    pollset->local_wakeup_cache = worker.wakeup_fd->next;
-  } else {
-    worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
-    grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
-  }
-  worker.kicked_specifically = 0;
-  /* If there's work waiting for the pollset to be idle, and the
-     pollset is idle, then do that work */
-  if (!pollset_has_workers(pollset) &&
-      !grpc_closure_list_empty(pollset->idle_jobs)) {
-    GPR_TIMER_MARK("pollset_work.idle_jobs", 0);
-    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
-    goto done;
-  }
-  /* If we're shutting down then we don't execute any extended work */
-  if (pollset->shutting_down) {
-    GPR_TIMER_MARK("pollset_work.shutting_down", 0);
-    goto done;
-  }
-  /* Give do_promote priority so we don't starve it out */
-  if (pollset->in_flight_cbs) {
-    GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0);
-    gpr_mu_unlock(&pollset->mu);
-    locked = 0;
-    goto done;
-  }
-  /* Start polling, and keep doing so while we're being asked to
-     re-evaluate our pollers (this allows poll() based pollers to
-     ensure they don't miss wakeups) */
-  keep_polling = 1;
-  while (keep_polling) {
-    keep_polling = 0;
-    if (!pollset->kicked_without_pollers) {
-      if (!added_worker) {
-        push_front_worker(pollset, &worker);
-        added_worker = 1;
-        gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
-      }
-      gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
-      GPR_TIMER_BEGIN("maybe_work_and_unlock", 0);
-      pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, &worker,
-                                             deadline, now);
-      GPR_TIMER_END("maybe_work_and_unlock", 0);
-      locked = 0;
-      gpr_tls_set(&g_current_thread_poller, 0);
-    } else {
-      GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0);
-      pollset->kicked_without_pollers = 0;
-    }
-  /* Finished execution - start cleaning up.
-     Note that we may arrive here from outside the enclosing while() loop.
-     In that case we won't loop though as we haven't added worker to the
-     worker list, which means nobody could ask us to re-evaluate polling). */
-  done:
-    if (!locked) {
-      queued_work |= grpc_exec_ctx_flush(exec_ctx);
-      gpr_mu_lock(&pollset->mu);
-      locked = 1;
-    }
-    /* If we're forced to re-evaluate polling (via pollset_kick with
-       GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force
-       a loop */
-    if (worker.reevaluate_polling_on_wakeup) {
-      worker.reevaluate_polling_on_wakeup = 0;
-      pollset->kicked_without_pollers = 0;
-      if (queued_work || worker.kicked_specifically) {
-        /* If there's queued work on the list, then set the deadline to be
-           immediate so we get back out of the polling loop quickly */
-        deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
-      }
-      keep_polling = 1;
-    }
-  }
-  if (added_worker) {
-    remove_worker(pollset, &worker);
-    gpr_tls_set(&g_current_thread_worker, 0);
-  }
-  /* release wakeup fd to the local pool */
-  worker.wakeup_fd->next = pollset->local_wakeup_cache;
-  pollset->local_wakeup_cache = worker.wakeup_fd;
-  /* check shutdown conditions */
-  if (pollset->shutting_down) {
-    if (pollset_has_workers(pollset)) {
-      pollset_kick(pollset, NULL);
-    } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
-      pollset->called_shutdown = 1;
-      gpr_mu_unlock(&pollset->mu);
-      finish_shutdown(exec_ctx, pollset);
-      grpc_exec_ctx_flush(exec_ctx);
-      /* Continuing to access pollset here is safe -- it is the caller's
-       * responsibility to not destroy when it has outstanding calls to
-       * pollset_work.
-       * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */
-      gpr_mu_lock(&pollset->mu);
-    } else if (!grpc_closure_list_empty(pollset->idle_jobs)) {
-      grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
-      gpr_mu_unlock(&pollset->mu);
-      grpc_exec_ctx_flush(exec_ctx);
-      gpr_mu_lock(&pollset->mu);
-    }
-  }
-  *worker_hdl = NULL;
-  GPR_TIMER_END("pollset_work", 0);
-}
-
-static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                             grpc_closure *closure) {
-  GPR_ASSERT(!pollset->shutting_down);
-  pollset->shutting_down = 1;
-  pollset->shutdown_done = closure;
-  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
-  if (!pollset_has_workers(pollset)) {
-    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
-  }
-  if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
-      !pollset_has_workers(pollset)) {
-    pollset->called_shutdown = 1;
-    finish_shutdown(exec_ctx, pollset);
-  }
-}
-
-static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
-                                           gpr_timespec now) {
-  gpr_timespec timeout;
-  static const int64_t max_spin_polling_us = 10;
-  if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) {
-    return -1;
-  }
-  if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros(
-                                                   max_spin_polling_us,
-                                                   GPR_TIMESPAN))) <= 0) {
-    return 0;
-  }
-  timeout = gpr_time_sub(deadline, now);
-  return gpr_time_to_millis(gpr_time_add(
-      timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN)));
-}
-
-/*
- * basic_pollset - a vtable that provides polling for zero or one file
- *                 descriptor via poll()
- */
-
-typedef struct grpc_unary_promote_args {
-  const grpc_pollset_vtable *original_vtable;
-  grpc_pollset *pollset;
-  grpc_fd *fd;
-  grpc_closure promotion_closure;
-} grpc_unary_promote_args;
-
-static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args,
-                             bool success) {
-  grpc_unary_promote_args *up_args = args;
-  const grpc_pollset_vtable *original_vtable = up_args->original_vtable;
-  grpc_pollset *pollset = up_args->pollset;
-  grpc_fd *fd = up_args->fd;
-
-  /*
-   * This is quite tricky. There are a number of cases to keep in mind here:
-   * 1. fd may have been orphaned
-   * 2. The pollset may no longer be a unary poller (and we can't let case #1
-   * leak to other pollset types!)
-   * 3. pollset's fd (which may have changed) may have been orphaned
-   * 4. The pollset may be shutting down.
-   */
-
-  gpr_mu_lock(&pollset->mu);
-  /* First we need to ensure that nobody is polling concurrently */
-  GPR_ASSERT(!pollset_has_workers(pollset));
-
-  gpr_free(up_args);
-  /* At this point the pollset may no longer be a unary poller. In that case
-   * we should just call the right add function and be done. */
-  /* TODO(klempner): If we're not careful this could cause infinite recursion.
-   * That's not a problem for now because empty_pollset has a trivial poller
-   * and we don't have any mechanism to unbecome multipoller. */
-  pollset->in_flight_cbs--;
-  if (pollset->shutting_down) {
-    /* We don't care about this pollset anymore. */
-    if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) {
-      pollset->called_shutdown = 1;
-      finish_shutdown(exec_ctx, pollset);
-    }
-  } else if (fd_is_orphaned(fd)) {
-    /* Don't try to add it to anything, we'll drop our ref on it below */
-  } else if (pollset->vtable != original_vtable) {
-    pollset->vtable->add_fd(exec_ctx, pollset, fd, 0);
-  } else if (fd != pollset->data.ptr) {
-    grpc_fd *fds[2];
-    fds[0] = pollset->data.ptr;
-    fds[1] = fd;
-
-    if (fds[0] && !fd_is_orphaned(fds[0])) {
-      platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds));
-      GRPC_FD_UNREF(fds[0], "basicpoll");
-    } else {
-      /* old fd is orphaned and we haven't cleaned it up until now, so remain a
-       * unary poller */
-      /* Note that it is possible that fds[1] is also orphaned at this point.
-       * That's okay, we'll correct it at the next add or poll. */
-      if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll");
-      pollset->data.ptr = fd;
-      GRPC_FD_REF(fd, "basicpoll");
-    }
-  }
-
-  gpr_mu_unlock(&pollset->mu);
-
-  /* Matching ref in basic_pollset_add_fd */
-  GRPC_FD_UNREF(fd, "basicpoll_add");
-}
-
-static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                 grpc_fd *fd, int and_unlock_pollset) {
-  grpc_unary_promote_args *up_args;
-  GPR_ASSERT(fd);
-  if (fd == pollset->data.ptr) goto exit;
-
-  if (!pollset_has_workers(pollset)) {
-    /* Fast path -- no in flight cbs */
-    /* TODO(klempner): Comment this out and fix any test failures or establish
-     * they are due to timing issues */
-    grpc_fd *fds[2];
-    fds[0] = pollset->data.ptr;
-    fds[1] = fd;
-
-    if (fds[0] == NULL) {
-      pollset->data.ptr = fd;
-      GRPC_FD_REF(fd, "basicpoll");
-    } else if (!fd_is_orphaned(fds[0])) {
-      platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds));
-      GRPC_FD_UNREF(fds[0], "basicpoll");
-    } else {
-      /* old fd is orphaned and we haven't cleaned it up until now, so remain a
-       * unary poller */
-      GRPC_FD_UNREF(fds[0], "basicpoll");
-      pollset->data.ptr = fd;
-      GRPC_FD_REF(fd, "basicpoll");
-    }
-    goto exit;
-  }
-
-  /* Now we need to promote. This needs to happen when we're not polling. Since
-   * this may be called from poll, the wait needs to happen asynchronously. */
-  GRPC_FD_REF(fd, "basicpoll_add");
-  pollset->in_flight_cbs++;
-  up_args = gpr_malloc(sizeof(*up_args));
-  up_args->fd = fd;
-  up_args->original_vtable = pollset->vtable;
-  up_args->pollset = pollset;
-  up_args->promotion_closure.cb = basic_do_promote;
-  up_args->promotion_closure.cb_arg = up_args;
-
-  grpc_closure_list_add(&pollset->idle_jobs, &up_args->promotion_closure, 1);
-  pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
-
-exit:
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-}
-
-static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
-                                                grpc_pollset *pollset,
-                                                grpc_pollset_worker *worker,
-                                                gpr_timespec deadline,
-                                                gpr_timespec now) {
-#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
-#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
-
-  struct pollfd pfd[3];
-  grpc_fd *fd;
-  grpc_fd_watcher fd_watcher;
-  int timeout;
-  int r;
-  nfds_t nfds;
-
-  fd = pollset->data.ptr;
-  if (fd && fd_is_orphaned(fd)) {
-    GRPC_FD_UNREF(fd, "basicpoll");
-    fd = pollset->data.ptr = NULL;
-  }
-  timeout = poll_deadline_to_millis_timeout(deadline, now);
-  pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
-  pfd[0].events = POLLIN;
-  pfd[0].revents = 0;
-  pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
-  pfd[1].events = POLLIN;
-  pfd[1].revents = 0;
-  nfds = 2;
-  if (fd) {
-    pfd[2].fd = fd->fd;
-    pfd[2].revents = 0;
-    GRPC_FD_REF(fd, "basicpoll_begin");
-    gpr_mu_unlock(&pollset->mu);
-    pfd[2].events =
-        (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher);
-    if (pfd[2].events != 0) {
-      nfds++;
-    }
-  } else {
-    gpr_mu_unlock(&pollset->mu);
-  }
-
-  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
-     even going into the blocking annotation if possible */
-  /* poll fd count (argument 2) is shortened by one if we have no events
-     to poll on - such that it only includes the kicker */
-  GPR_TIMER_BEGIN("poll", 0);
-  GRPC_SCHEDULING_START_BLOCKING_REGION;
-  r = grpc_poll_function(pfd, nfds, timeout);
-  GRPC_SCHEDULING_END_BLOCKING_REGION;
-  GPR_TIMER_END("poll", 0);
-
-  if (r < 0) {
-    if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
-    }
-    if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
-    }
-  } else if (r == 0) {
-    if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
-    }
-  } else {
-    if (pfd[0].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
-    }
-    if (pfd[1].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
-    }
-    if (nfds > 2) {
-      fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK,
-                  pfd[2].revents & POLLOUT_CHECK);
-    } else if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
-    }
-  }
-
-  if (fd) {
-    GRPC_FD_UNREF(fd, "basicpoll_begin");
-  }
-}
-
-static void basic_pollset_destroy(grpc_pollset *pollset) {
-  if (pollset->data.ptr != NULL) {
-    GRPC_FD_UNREF(pollset->data.ptr, "basicpoll");
-    pollset->data.ptr = NULL;
-  }
-}
-
-static const grpc_pollset_vtable basic_pollset = {
-    basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock,
-    basic_pollset_destroy, basic_pollset_destroy};
-
-static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) {
-  pollset->vtable = &basic_pollset;
-  pollset->data.ptr = fd_or_null;
-  if (fd_or_null != NULL) {
-    GRPC_FD_REF(fd_or_null, "basicpoll");
-  }
-}
-
-/*******************************************************************************
- * pollset_multipoller_with_poll_posix.c
- */
-
-#ifndef GPR_LINUX_MULTIPOLL_WITH_EPOLL
-
-typedef struct {
-  /* all polled fds */
-  size_t fd_count;
-  size_t fd_capacity;
-  grpc_fd **fds;
-  /* fds that have been removed from the pollset explicitly */
-  size_t del_count;
-  size_t del_capacity;
-  grpc_fd **dels;
-} poll_hdr;
-
-static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
-                                               grpc_pollset *pollset,
-                                               grpc_fd *fd,
-                                               int and_unlock_pollset) {
-  size_t i;
-  poll_hdr *h = pollset->data.ptr;
-  /* TODO(ctiller): this is O(num_fds^2); maybe switch to a hash set here */
-  for (i = 0; i < h->fd_count; i++) {
-    if (h->fds[i] == fd) goto exit;
-  }
-  if (h->fd_count == h->fd_capacity) {
-    h->fd_capacity = GPR_MAX(h->fd_capacity + 8, h->fd_count * 3 / 2);
-    h->fds = gpr_realloc(h->fds, sizeof(grpc_fd *) * h->fd_capacity);
-  }
-  h->fds[h->fd_count++] = fd;
-  GRPC_FD_REF(fd, "multipoller");
-exit:
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-  }
-}
-
-static void multipoll_with_poll_pollset_maybe_work_and_unlock(
-    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
-    gpr_timespec deadline, gpr_timespec now) {
-#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR)
-#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR)
-
-  int timeout;
-  int r;
-  size_t i, j, fd_count;
-  nfds_t pfd_count;
-  poll_hdr *h;
-  /* TODO(ctiller): inline some elements to avoid an allocation */
-  grpc_fd_watcher *watchers;
-  struct pollfd *pfds;
-
-  h = pollset->data.ptr;
-  timeout = poll_deadline_to_millis_timeout(deadline, now);
-  /* TODO(ctiller): perform just one malloc here if we exceed the inline case */
-  pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 2));
-  watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 2));
-  fd_count = 0;
-  pfd_count = 2;
-  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
-  pfds[0].events = POLLIN;
-  pfds[0].revents = 0;
-  pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
-  pfds[1].events = POLLIN;
-  pfds[1].revents = 0;
-  for (i = 0; i < h->fd_count; i++) {
-    int remove = fd_is_orphaned(h->fds[i]);
-    for (j = 0; !remove && j < h->del_count; j++) {
-      if (h->fds[i] == h->dels[j]) remove = 1;
-    }
-    if (remove) {
-      GRPC_FD_UNREF(h->fds[i], "multipoller");
-    } else {
-      h->fds[fd_count++] = h->fds[i];
-      watchers[pfd_count].fd = h->fds[i];
-      GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start");
-      pfds[pfd_count].fd = h->fds[i]->fd;
-      pfds[pfd_count].revents = 0;
-      pfd_count++;
-    }
-  }
-  for (j = 0; j < h->del_count; j++) {
-    GRPC_FD_UNREF(h->dels[j], "multipoller_del");
-  }
-  h->del_count = 0;
-  h->fd_count = fd_count;
-  gpr_mu_unlock(&pollset->mu);
-
-  for (i = 2; i < pfd_count; i++) {
-    grpc_fd *fd = watchers[i].fd;
-    pfds[i].events = (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT,
-                                          &watchers[i]);
-    GRPC_FD_UNREF(fd, "multipoller_start");
-  }
-
-  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
-     even going into the blocking annotation if possible */
-  GRPC_SCHEDULING_START_BLOCKING_REGION;
-  r = grpc_poll_function(pfds, pfd_count, timeout);
-  GRPC_SCHEDULING_END_BLOCKING_REGION;
-
-  if (r < 0) {
-    if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
-    }
-    for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
-    }
-  } else if (r == 0) {
-    for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
-    }
-  } else {
-    if (pfds[0].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
-    }
-    if (pfds[1].revents & POLLIN_CHECK) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
-    }
-    for (i = 2; i < pfd_count; i++) {
-      if (watchers[i].fd == NULL) {
-        fd_end_poll(exec_ctx, &watchers[i], 0, 0);
-        continue;
-      }
-      fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                  pfds[i].revents & POLLOUT_CHECK);
-    }
-  }
-
-  gpr_free(pfds);
-  gpr_free(watchers);
-}
-
-static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) {
-  size_t i;
-  poll_hdr *h = pollset->data.ptr;
-  for (i = 0; i < h->fd_count; i++) {
-    GRPC_FD_UNREF(h->fds[i], "multipoller");
-  }
-  for (i = 0; i < h->del_count; i++) {
-    GRPC_FD_UNREF(h->dels[i], "multipoller_del");
-  }
-  h->fd_count = 0;
-  h->del_count = 0;
-}
-
-static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
-  poll_hdr *h = pollset->data.ptr;
-  multipoll_with_poll_pollset_finish_shutdown(pollset);
-  gpr_free(h->fds);
-  gpr_free(h->dels);
-  gpr_free(h);
-}
-
-static const grpc_pollset_vtable multipoll_with_poll_pollset = {
-    multipoll_with_poll_pollset_add_fd,
-    multipoll_with_poll_pollset_maybe_work_and_unlock,
-    multipoll_with_poll_pollset_finish_shutdown,
-    multipoll_with_poll_pollset_destroy};
-
-static void poll_become_multipoller(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset *pollset, grpc_fd **fds,
-                                    size_t nfds) {
-  size_t i;
-  poll_hdr *h = gpr_malloc(sizeof(poll_hdr));
-  pollset->vtable = &multipoll_with_poll_pollset;
-  pollset->data.ptr = h;
-  h->fd_count = nfds;
-  h->fd_capacity = nfds;
-  h->fds = gpr_malloc(nfds * sizeof(grpc_fd *));
-  h->del_count = 0;
-  h->del_capacity = 0;
-  h->dels = NULL;
-  for (i = 0; i < nfds; i++) {
-    h->fds[i] = fds[i];
-    GRPC_FD_REF(fds[i], "multipoller");
-  }
-}
-
-#endif /* !GPR_LINUX_MULTIPOLL_WITH_EPOLL */
-
-/*******************************************************************************
- * pollset_multipoller_with_epoll_posix.c
- */
-
-#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
-
-#include <errno.h>
-#include <poll.h>
-#include <string.h>
-#include <sys/epoll.h>
-#include <unistd.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/useful.h>
-
-#include "src/core/lib/iomgr/ev_posix.h"
-#include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/block_annotate.h"
-
-static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st) {
-  /* only one set_ready can be active at once (but there may be a racing
-     notify_on) */
-  gpr_mu_lock(&fd->mu);
-  set_ready_locked(exec_ctx, fd, st);
-  gpr_mu_unlock(&fd->mu);
-}
-
-static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->read_closure);
-}
-
-static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->write_closure);
-}
-
-struct epoll_fd_list {
-  int *epoll_fds;
-  size_t count;
-  size_t capacity;
-};
-
-static struct epoll_fd_list epoll_fd_global_list;
-static gpr_once init_epoll_fd_list_mu = GPR_ONCE_INIT;
-static gpr_mu epoll_fd_list_mu;
-
-static void init_mu(void) { gpr_mu_init(&epoll_fd_list_mu); }
-
-static void add_epoll_fd_to_global_list(int epoll_fd) {
-  gpr_once_init(&init_epoll_fd_list_mu, init_mu);
-
-  gpr_mu_lock(&epoll_fd_list_mu);
-  if (epoll_fd_global_list.count == epoll_fd_global_list.capacity) {
-    epoll_fd_global_list.capacity =
-        GPR_MAX((size_t)8, epoll_fd_global_list.capacity * 2);
-    epoll_fd_global_list.epoll_fds =
-        gpr_realloc(epoll_fd_global_list.epoll_fds,
-                    epoll_fd_global_list.capacity * sizeof(int));
-  }
-  epoll_fd_global_list.epoll_fds[epoll_fd_global_list.count++] = epoll_fd;
-  gpr_mu_unlock(&epoll_fd_list_mu);
-}
-
-static void remove_epoll_fd_from_global_list(int epoll_fd) {
-  gpr_mu_lock(&epoll_fd_list_mu);
-  GPR_ASSERT(epoll_fd_global_list.count > 0);
-  for (size_t i = 0; i < epoll_fd_global_list.count; i++) {
-    if (epoll_fd == epoll_fd_global_list.epoll_fds[i]) {
-      epoll_fd_global_list.epoll_fds[i] =
-          epoll_fd_global_list.epoll_fds[--(epoll_fd_global_list.count)];
-      break;
-    }
-  }
-  gpr_mu_unlock(&epoll_fd_list_mu);
-}
-
-static void remove_fd_from_all_epoll_sets(int fd) {
-  int err;
-  gpr_once_init(&init_epoll_fd_list_mu, init_mu);
-  gpr_mu_lock(&epoll_fd_list_mu);
-  if (epoll_fd_global_list.count == 0) {
-    gpr_mu_unlock(&epoll_fd_list_mu);
-    return;
-  }
-  for (size_t i = 0; i < epoll_fd_global_list.count; i++) {
-    err = epoll_ctl(epoll_fd_global_list.epoll_fds[i], EPOLL_CTL_DEL, fd, NULL);
-    if (err < 0 && errno != ENOENT) {
-      gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd,
-              strerror(errno));
-    }
-  }
-  gpr_mu_unlock(&epoll_fd_list_mu);
-}
-
-typedef struct {
-  grpc_pollset *pollset;
-  grpc_fd *fd;
-  grpc_closure closure;
-} delayed_add;
-
-typedef struct { int epoll_fd; } epoll_hdr;
-
-static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           grpc_fd *fd) {
-  epoll_hdr *h = pollset->data.ptr;
-  struct epoll_event ev;
-  int err;
-  grpc_fd_watcher watcher;
-
-  /* We pretend to be polling whilst adding an fd to keep the fd from being
-     closed during the add. This may result in a spurious wakeup being assigned
-     to this pollset whilst adding, but that should be benign. */
-  GPR_ASSERT(fd_begin_poll(fd, pollset, NULL, 0, 0, &watcher) == 0);
-  if (watcher.fd != NULL) {
-    ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET);
-    ev.data.ptr = fd;
-    err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev);
-    if (err < 0) {
-      /* FDs may be added to a pollset multiple times, so EEXIST is normal. */
-      if (errno != EEXIST) {
-        gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd,
-                strerror(errno));
-      }
-    }
-  }
-  fd_end_poll(exec_ctx, &watcher, 0, 0);
-}
-
-static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg,
-                                bool iomgr_status) {
-  delayed_add *da = arg;
-
-  if (!fd_is_orphaned(da->fd)) {
-    finally_add_fd(exec_ctx, da->pollset, da->fd);
-  }
-
-  gpr_mu_lock(&da->pollset->mu);
-  da->pollset->in_flight_cbs--;
-  if (da->pollset->shutting_down) {
-    /* We don't care about this pollset anymore. */
-    if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) {
-      da->pollset->called_shutdown = 1;
-      grpc_exec_ctx_enqueue(exec_ctx, da->pollset->shutdown_done, true, NULL);
-    }
-  }
-  gpr_mu_unlock(&da->pollset->mu);
-
-  GRPC_FD_UNREF(da->fd, "delayed_add");
-
-  gpr_free(da);
-}
-
-static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx,
-                                                grpc_pollset *pollset,
-                                                grpc_fd *fd,
-                                                int and_unlock_pollset) {
-  if (and_unlock_pollset) {
-    gpr_mu_unlock(&pollset->mu);
-    finally_add_fd(exec_ctx, pollset, fd);
-  } else {
-    delayed_add *da = gpr_malloc(sizeof(*da));
-    da->pollset = pollset;
-    da->fd = fd;
-    GRPC_FD_REF(fd, "delayed_add");
-    grpc_closure_init(&da->closure, perform_delayed_add, da);
-    pollset->in_flight_cbs++;
-    grpc_exec_ctx_enqueue(exec_ctx, &da->closure, true, NULL);
-  }
-}
-
-/* TODO(klempner): We probably want to turn this down a bit */
-#define GRPC_EPOLL_MAX_EVENTS 1000
-
-static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
-    grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker,
-    gpr_timespec deadline, gpr_timespec now) {
-  struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
-  int ep_rv;
-  int poll_rv;
-  epoll_hdr *h = pollset->data.ptr;
-  int timeout_ms;
-  struct pollfd pfds[2];
-
-  /* If you want to ignore epoll's ability to sanely handle parallel pollers,
-   * for a more apples-to-apples performance comparison with poll, add a
-   * if (pollset->counter != 0) { return 0; }
-   * here.
-   */
-
-  gpr_mu_unlock(&pollset->mu);
-
-  timeout_ms = poll_deadline_to_millis_timeout(deadline, now);
-
-  pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd);
-  pfds[0].events = POLLIN;
-  pfds[0].revents = 0;
-  pfds[1].fd = h->epoll_fd;
-  pfds[1].events = POLLIN;
-  pfds[1].revents = 0;
-
-  /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
-     even going into the blocking annotation if possible */
-  GPR_TIMER_BEGIN("poll", 0);
-  GRPC_SCHEDULING_START_BLOCKING_REGION;
-  poll_rv = grpc_poll_function(pfds, 2, timeout_ms);
-  GRPC_SCHEDULING_END_BLOCKING_REGION;
-  GPR_TIMER_END("poll", 0);
-
-  if (poll_rv < 0) {
-    if (errno != EINTR) {
-      gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
-    }
-  } else if (poll_rv == 0) {
-    /* do nothing */
-  } else {
-    if (pfds[0].revents) {
-      grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd);
-    }
-    if (pfds[1].revents) {
-      do {
-        /* The following epoll_wait never blocks; it has a timeout of 0 */
-        ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
-        if (ep_rv < 0) {
-          if (errno != EINTR) {
-            gpr_log(GPR_ERROR, "epoll_wait() failed: %s", strerror(errno));
-          }
-        } else {
-          int i;
-          for (i = 0; i < ep_rv; ++i) {
-            grpc_fd *fd = ep_ev[i].data.ptr;
-            /* TODO(klempner): We might want to consider making err and pri
-             * separate events */
-            int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
-            int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
-            int write_ev = ep_ev[i].events & EPOLLOUT;
-            if (fd == NULL) {
-              grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
-            } else {
-              if (read_ev || cancel) {
-                fd_become_readable(exec_ctx, fd);
-              }
-              if (write_ev || cancel) {
-                fd_become_writable(exec_ctx, fd);
-              }
-            }
-          }
-        }
-      } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
-    }
-  }
-}
-
-static void multipoll_with_epoll_pollset_finish_shutdown(
-    grpc_pollset *pollset) {}
-
-static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) {
-  epoll_hdr *h = pollset->data.ptr;
-  close(h->epoll_fd);
-  remove_epoll_fd_from_global_list(h->epoll_fd);
-  gpr_free(h);
-}
-
-static const grpc_pollset_vtable multipoll_with_epoll_pollset = {
-    multipoll_with_epoll_pollset_add_fd,
-    multipoll_with_epoll_pollset_maybe_work_and_unlock,
-    multipoll_with_epoll_pollset_finish_shutdown,
-    multipoll_with_epoll_pollset_destroy};
-
-static void epoll_become_multipoller(grpc_exec_ctx *exec_ctx,
-                                     grpc_pollset *pollset, grpc_fd **fds,
-                                     size_t nfds) {
-  size_t i;
-  epoll_hdr *h = gpr_malloc(sizeof(epoll_hdr));
-  struct epoll_event ev;
-  int err;
-
-  pollset->vtable = &multipoll_with_epoll_pollset;
-  pollset->data.ptr = h;
-  h->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
-  if (h->epoll_fd < 0) {
-    /* TODO(klempner): Fall back to poll here, especially on ENOSYS */
-    gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno));
-    abort();
-  }
-  add_epoll_fd_to_global_list(h->epoll_fd);
-
-  ev.events = (uint32_t)(EPOLLIN | EPOLLET);
-  ev.data.ptr = NULL;
-  err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD,
-                  GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), &ev);
-  if (err < 0) {
-    gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s",
-            GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd),
-            strerror(errno));
-  }
-
-  for (i = 0; i < nfds; i++) {
-    multipoll_with_epoll_pollset_add_fd(exec_ctx, pollset, fds[i], 0);
-  }
-}
-
-#else /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
-
-static void remove_fd_from_all_epoll_sets(int fd) {}
-
-#endif /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
-
-/*******************************************************************************
- * pollset_set_posix.c
- */
-
-static grpc_pollset_set *pollset_set_create(void) {
-  grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set));
-  memset(pollset_set, 0, sizeof(*pollset_set));
-  gpr_mu_init(&pollset_set->mu);
-  return pollset_set;
-}
-
-static void pollset_set_destroy(grpc_pollset_set *pollset_set) {
-  size_t i;
-  gpr_mu_destroy(&pollset_set->mu);
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
-  }
-  gpr_free(pollset_set->pollsets);
-  gpr_free(pollset_set->pollset_sets);
-  gpr_free(pollset_set->fds);
-  gpr_free(pollset_set);
-}
-
-static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset_set *pollset_set,
-                                    grpc_pollset *pollset) {
-  size_t i, j;
-  gpr_mu_lock(&pollset_set->mu);
-  if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
-    pollset_set->pollset_capacity =
-        GPR_MAX(8, 2 * pollset_set->pollset_capacity);
-    pollset_set->pollsets =
-        gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity *
-                                               sizeof(*pollset_set->pollsets));
-  }
-  pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
-  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
-    if (fd_is_orphaned(pollset_set->fds[i])) {
-      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
-    } else {
-      pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
-      pollset_set->fds[j++] = pollset_set->fds[i];
-    }
-  }
-  pollset_set->fd_count = j;
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset_set *pollset_set,
-                                    grpc_pollset *pollset) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  for (i = 0; i < pollset_set->pollset_count; i++) {
-    if (pollset_set->pollsets[i] == pollset) {
-      pollset_set->pollset_count--;
-      GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i],
-               pollset_set->pollsets[pollset_set->pollset_count]);
-      break;
-    }
-  }
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
-                                        grpc_pollset_set *bag,
-                                        grpc_pollset_set *item) {
-  size_t i, j;
-  gpr_mu_lock(&bag->mu);
-  if (bag->pollset_set_count == bag->pollset_set_capacity) {
-    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
-    bag->pollset_sets =
-        gpr_realloc(bag->pollset_sets,
-                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
-  }
-  bag->pollset_sets[bag->pollset_set_count++] = item;
-  for (i = 0, j = 0; i < bag->fd_count; i++) {
-    if (fd_is_orphaned(bag->fds[i])) {
-      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
-    } else {
-      pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
-      bag->fds[j++] = bag->fds[i];
-    }
-  }
-  bag->fd_count = j;
-  gpr_mu_unlock(&bag->mu);
-}
-
-static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
-                                        grpc_pollset_set *bag,
-                                        grpc_pollset_set *item) {
-  size_t i;
-  gpr_mu_lock(&bag->mu);
-  for (i = 0; i < bag->pollset_set_count; i++) {
-    if (bag->pollset_sets[i] == item) {
-      bag->pollset_set_count--;
-      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
-               bag->pollset_sets[bag->pollset_set_count]);
-      break;
-    }
-  }
-  gpr_mu_unlock(&bag->mu);
-}
-
-static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  if (pollset_set->fd_count == pollset_set->fd_capacity) {
-    pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity);
-    pollset_set->fds = gpr_realloc(
-        pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds));
-  }
-  GRPC_FD_REF(fd, "pollset_set");
-  pollset_set->fds[pollset_set->fd_count++] = fd;
-  for (i = 0; i < pollset_set->pollset_count; i++) {
-    pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
-  }
-  for (i = 0; i < pollset_set->pollset_set_count; i++) {
-    pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
-  }
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    if (pollset_set->fds[i] == fd) {
-      pollset_set->fd_count--;
-      GPR_SWAP(grpc_fd *, pollset_set->fds[i],
-               pollset_set->fds[pollset_set->fd_count]);
-      GRPC_FD_UNREF(fd, "pollset_set");
-      break;
-    }
-  }
-  for (i = 0; i < pollset_set->pollset_set_count; i++) {
-    pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
-  }
-  gpr_mu_unlock(&pollset_set->mu);
-}
-
-/*******************************************************************************
- * event engine binding
- */
-
-static void shutdown_engine(void) {
-  fd_global_shutdown();
-  pollset_global_shutdown();
-}
-
-static const grpc_event_engine_vtable vtable = {
-    .pollset_size = sizeof(grpc_pollset),
-
-    .fd_create = fd_create,
-    .fd_wrapped_fd = fd_wrapped_fd,
-    .fd_orphan = fd_orphan,
-    .fd_shutdown = fd_shutdown,
-    .fd_notify_on_read = fd_notify_on_read,
-    .fd_notify_on_write = fd_notify_on_write,
-
-    .pollset_init = pollset_init,
-    .pollset_shutdown = pollset_shutdown,
-    .pollset_reset = pollset_reset,
-    .pollset_destroy = pollset_destroy,
-    .pollset_work = pollset_work,
-    .pollset_kick = pollset_kick,
-    .pollset_add_fd = pollset_add_fd,
-
-    .pollset_set_create = pollset_set_create,
-    .pollset_set_destroy = pollset_set_destroy,
-    .pollset_set_add_pollset = pollset_set_add_pollset,
-    .pollset_set_del_pollset = pollset_set_del_pollset,
-    .pollset_set_add_pollset_set = pollset_set_add_pollset_set,
-    .pollset_set_del_pollset_set = pollset_set_del_pollset_set,
-    .pollset_set_add_fd = pollset_set_add_fd,
-    .pollset_set_del_fd = pollset_set_del_fd,
-
-    .kick_poller = kick_poller,
-
-    .shutdown_engine = shutdown_engine,
-};
-
-const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void) {
-#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
-  platform_become_multipoller = epoll_become_multipoller;
-#else
-  platform_become_multipoller = poll_become_multipoller;
-#endif
-  fd_global_init();
-  pollset_global_init();
-  return &vtable;
-}
-
-#endif
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index e91ae40212..e2a21230b9 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -59,6 +59,8 @@
  * FD declarations
  */
 
+grpc_wakeup_fd grpc_global_wakeup_fd;
+
 typedef struct grpc_fd_watcher {
   struct grpc_fd_watcher *next;
   struct grpc_fd_watcher *prev;
@@ -113,6 +115,9 @@ struct grpc_fd {
   grpc_closure *on_done_closure;
 
   grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
 };
 
 /* Begin polling on an fd.
@@ -134,7 +139,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
    if got_read or got_write are 1, also does the become_{readable,writable} as
    appropriate. */
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
 
 /* Return 1 if this fd is orphaned, 0 otherwise */
 static bool fd_is_orphaned(grpc_fd *fd);
@@ -177,7 +183,6 @@ struct grpc_pollset_worker {
 struct grpc_pollset {
   gpr_mu mu;
   grpc_pollset_worker root_worker;
-  int in_flight_cbs;
   int shutting_down;
   int called_shutdown;
   int kicked_without_pollers;
@@ -187,10 +192,6 @@ struct grpc_pollset {
   size_t fd_count;
   size_t fd_capacity;
   grpc_fd **fds;
-  /* fds that have been removed from the pollset explicitly */
-  size_t del_count;
-  size_t del_capacity;
-  grpc_fd **dels;
   /* Local cache of eventfds for workers */
   grpc_cached_wakeup_fd *local_wakeup_cache;
 };
@@ -301,6 +302,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
   r->on_done_closure = NULL;
   r->closed = 0;
   r->released = 0;
+  r->read_notifier_pollset = NULL;
 
   char *name2;
   gpr_asprintf(&name2, "%s fd=%d", name, fd);
@@ -316,6 +318,18 @@ static bool fd_is_orphaned(grpc_fd *fd) {
   return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
 }
 
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
 static void pollset_kick_locked(grpc_fd_watcher *watcher) {
   gpr_mu_lock(&watcher->pollset->mu);
   GPR_ASSERT(watcher->worker);
@@ -444,6 +458,11 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   }
 }
 
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_lock(&fd->mu);
   GPR_ASSERT(!fd->shutdown);
@@ -519,7 +538,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
 }
 
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
   int was_polling = 0;
   int kick = 0;
   grpc_fd *fd = watcher->fd;
@@ -555,6 +575,9 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
     if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
       kick = 1;
     }
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
   }
   if (got_write) {
     if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
@@ -700,7 +723,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->mu);
   *mu = &pollset->mu;
   pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
-  pollset->in_flight_cbs = 0;
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
@@ -709,14 +731,10 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   pollset->kicked_without_pollers = 0;
   pollset->fd_count = 0;
   pollset->fd_capacity = 0;
-  pollset->del_count = 0;
-  pollset->del_capacity = 0;
   pollset->fds = NULL;
-  pollset->dels = NULL;
 }
 
 static void pollset_destroy(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
   GPR_ASSERT(!pollset_has_workers(pollset));
   GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   while (pollset->local_wakeup_cache) {
@@ -726,17 +744,14 @@ static void pollset_destroy(grpc_pollset *pollset) {
     pollset->local_wakeup_cache = next;
   }
   gpr_free(pollset->fds);
-  gpr_free(pollset->dels);
   gpr_mu_destroy(&pollset->mu);
 }
 
 static void pollset_reset(grpc_pollset *pollset) {
   GPR_ASSERT(pollset->shutting_down);
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
   GPR_ASSERT(!pollset_has_workers(pollset));
   GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   GPR_ASSERT(pollset->fd_count == 0);
-  GPR_ASSERT(pollset->del_count == 0);
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
@@ -769,11 +784,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
   for (i = 0; i < pollset->fd_count; i++) {
     GRPC_FD_UNREF(pollset->fds[i], "multipoller");
   }
-  for (i = 0; i < pollset->del_count; i++) {
-    GRPC_FD_UNREF(pollset->dels[i], "multipoller_del");
-  }
   pollset->fd_count = 0;
-  pollset->del_count = 0;
   grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL);
 }
 
@@ -813,13 +824,6 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     GPR_TIMER_MARK("pollset_work.shutting_down", 0);
     goto done;
   }
-  /* Give do_promote priority so we don't starve it out */
-  if (pollset->in_flight_cbs) {
-    GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0);
-    gpr_mu_unlock(&pollset->mu);
-    locked = 0;
-    goto done;
-  }
   /* Start polling, and keep doing so while we're being asked to
      re-evaluate our pollers (this allows poll() based pollers to
      ensure they don't miss wakeups) */
@@ -839,7 +843,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
       int timeout;
       int r;
-      size_t i, j, fd_count;
+      size_t i, fd_count;
       nfds_t pfd_count;
       /* TODO(ctiller): inline some elements to avoid an allocation */
       grpc_fd_watcher *watchers;
@@ -859,11 +863,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       pfds[1].events = POLLIN;
       pfds[1].revents = 0;
       for (i = 0; i < pollset->fd_count; i++) {
-        int remove = fd_is_orphaned(pollset->fds[i]);
-        for (j = 0; !remove && j < pollset->del_count; j++) {
-          if (pollset->fds[i] == pollset->dels[j]) remove = 1;
-        }
-        if (remove) {
+        if (fd_is_orphaned(pollset->fds[i])) {
           GRPC_FD_UNREF(pollset->fds[i], "multipoller");
         } else {
           pollset->fds[fd_count++] = pollset->fds[i];
@@ -874,10 +874,6 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           pfd_count++;
         }
       }
-      for (j = 0; j < pollset->del_count; j++) {
-        GRPC_FD_UNREF(pollset->dels[j], "multipoller_del");
-      }
-      pollset->del_count = 0;
       pollset->fd_count = fd_count;
       gpr_mu_unlock(&pollset->mu);
 
@@ -899,11 +895,11 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
         }
         for (i = 2; i < pfd_count; i++) {
-          fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else if (r == 0) {
         for (i = 2; i < pfd_count; i++) {
-          fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else {
         if (pfds[0].revents & POLLIN_CHECK) {
@@ -914,10 +910,10 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
         }
         for (i = 2; i < pfd_count; i++) {
           if (watchers[i].fd == NULL) {
-            fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+            fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
           } else {
             fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                        pfds[i].revents & POLLOUT_CHECK);
+                        pfds[i].revents & POLLOUT_CHECK, pollset);
           }
         }
       }
@@ -969,7 +965,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (pollset->shutting_down) {
     if (pollset_has_workers(pollset)) {
       pollset_kick(pollset, NULL);
-    } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
+    } else if (!pollset->called_shutdown) {
       pollset->called_shutdown = 1;
       gpr_mu_unlock(&pollset->mu);
       finish_shutdown(exec_ctx, pollset);
@@ -999,8 +995,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (!pollset_has_workers(pollset)) {
     grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
   }
-  if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
-      !pollset_has_workers(pollset)) {
+  if (!pollset->called_shutdown && !pollset_has_workers(pollset)) {
     pollset->called_shutdown = 1;
     finish_shutdown(exec_ctx, pollset);
   }
@@ -1181,6 +1176,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_shutdown = fd_shutdown,
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index a7dfc9552d..95520b01d3 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -44,7 +44,6 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h"
 #include "src/core/lib/iomgr/ev_poll_posix.h"
 #include "src/core/lib/support/env.h"
 
@@ -62,7 +61,7 @@ typedef struct {
 } event_engine_factory;
 
 static const event_engine_factory g_factories[] = {
-    {"poll", grpc_init_poll_posix}, {"legacy", grpc_init_poll_and_epoll_posix},
+    {"poll", grpc_init_poll_posix},
 };
 
 static void add(const char *beg, const char *end, char ***ss, size_t *ns) {
@@ -163,6 +162,11 @@ void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   g_event_engine->fd_notify_on_write(exec_ctx, fd, closure);
 }
 
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd) {
+  return g_event_engine->fd_get_read_notifier_pollset(exec_ctx, fd);
+}
+
 size_t grpc_pollset_size(void) { return g_event_engine->pollset_size; }
 
 void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 1fa9f5ef2d..344bf63438 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -55,6 +55,8 @@ typedef struct grpc_event_engine_vtable {
                             grpc_closure *closure);
   void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
+  grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
 
   void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu);
   void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -137,6 +139,10 @@ void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
 
+/* Return the read notifier pollset from the fd */
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
+
 /* pollset_posix functions */
 
 /* Add an fd to a pollset */
diff --git a/src/core/lib/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h
index 99b9f29729..fee14ae661 100644
--- a/src/core/lib/iomgr/tcp_server.h
+++ b/src/core/lib/iomgr/tcp_server.h
@@ -52,6 +52,7 @@ typedef struct grpc_tcp_server_acceptor {
 /* Called for newly connected TCP connections. */
 typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_endpoint *ep,
+                                   grpc_pollset *accepting_pollset,
                                    grpc_tcp_server_acceptor *acceptor);
 
 /* Create a server, initially not bound to any ports. The caller owns one ref.
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index aaeb384f6e..909e34abc7 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -128,6 +128,9 @@ struct grpc_tcp_server {
   grpc_pollset **pollsets;
   /* number of pollsets in the pollsets array */
   size_t pollset_count;
+
+  /* next pollset to assign a channel to */
+  size_t next_pollset_to_assign;
 };
 
 grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
@@ -145,6 +148,7 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
   s->head = NULL;
   s->tail = NULL;
   s->nports = 0;
+  s->next_pollset_to_assign = 0;
   return s;
 }
 
@@ -310,13 +314,17 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_tcp_listener *sp = arg;
   grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index,
                                        sp->fd_index};
+  grpc_pollset *read_notifier_pollset = NULL;
   grpc_fd *fdobj;
-  size_t i;
 
   if (!success) {
     goto error;
   }
 
+  read_notifier_pollset =
+      sp->server->pollsets[(sp->server->next_pollset_to_assign++) %
+                           sp->server->pollset_count];
+
   /* loop until accept4 returns EAGAIN, and then re-arm notification */
   for (;;) {
     struct sockaddr_storage addr;
@@ -349,16 +357,18 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     }
 
     fdobj = grpc_fd_create(fd, name);
-    /* TODO(ctiller): revise this when we have server-side sharding
-       of channels -- we certainly should not be automatically adding every
-       incoming channel to every pollset owned by the server */
-    for (i = 0; i < sp->server->pollset_count; i++) {
-      grpc_pollset_add_fd(exec_ctx, sp->server->pollsets[i], fdobj);
+
+    if (read_notifier_pollset == NULL) {
+      gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd");
+      goto error;
     }
+
+    grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj);
+
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
         grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
-        &acceptor);
+        read_notifier_pollset, &acceptor);
 
     gpr_free(name);
     gpr_free(addr_str);
diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 125f521d87..e15f8b0cdf 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -379,9 +379,10 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, bool from_iocp) {
 
   /* The only time we should call our callback, is where we successfully
      managed to accept a connection, and created an endpoint. */
-  if (ep)
-    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep,
+  if (ep) {
+    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
                              &acceptor);
+  }
   /* As we were notified from the IOCP of one and exactly one accept,
      the former socked we created has now either been destroy or assigned
      to the new connection. We need to create a new one for the next
diff --git a/src/core/lib/security/security_context.c b/src/core/lib/security/context/security_context.c
index 343e0b5b8b..127b13ee50 100644
--- a/src/core/lib/security/security_context.c
+++ b/src/core/lib/security/context/security_context.c
@@ -33,7 +33,7 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
diff --git a/src/core/lib/security/security_context.h b/src/core/lib/security/context/security_context.h
index 81161ec47d..ef0c06b1fb 100644
--- a/src/core/lib/security/security_context.h
+++ b/src/core/lib/security/context/security_context.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
+#ifndef GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
+#define GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
 
 #include "src/core/lib/iomgr/pollset.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 /* --- grpc_auth_context ---
 
@@ -111,4 +111,4 @@ grpc_auth_context *grpc_auth_context_from_arg(const grpc_arg *arg);
 grpc_auth_context *grpc_find_auth_context_in_args(
     const grpc_channel_args *args);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H */
diff --git a/src/core/lib/security/credentials.c b/src/core/lib/security/credentials.c
deleted file mode 100644
index 43c061a265..0000000000
--- a/src/core/lib/security/credentials.c
+++ /dev/null
@@ -1,1296 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/lib/security/credentials.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/http_client_filter.h"
-#include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/http/parser.h"
-#include "src/core/lib/iomgr/executor.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/surface/api_trace.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/time.h>
-
-/* -- Common. -- */
-
-struct grpc_credentials_metadata_request {
-  grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
-  void *user_data;
-};
-
-static grpc_credentials_metadata_request *
-grpc_credentials_metadata_request_create(grpc_call_credentials *creds,
-                                         grpc_credentials_metadata_cb cb,
-                                         void *user_data) {
-  grpc_credentials_metadata_request *r =
-      gpr_malloc(sizeof(grpc_credentials_metadata_request));
-  r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
-  return r;
-}
-
-static void grpc_credentials_metadata_request_destroy(
-    grpc_credentials_metadata_request *r) {
-  grpc_call_credentials_unref(r->creds);
-  gpr_free(r);
-}
-
-grpc_channel_credentials *grpc_channel_credentials_ref(
-    grpc_channel_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
-  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
-  grpc_channel_credentials_unref(creds);
-}
-
-grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_call_credentials_unref(grpc_call_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_call_credentials_release(grpc_call_credentials *creds) {
-  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
-  grpc_call_credentials_unref(creds);
-}
-
-void grpc_call_credentials_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-    }
-    return;
-  }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pops, context, cb,
-                                      user_data);
-}
-
-grpc_security_status grpc_channel_credentials_create_security_connector(
-    grpc_channel_credentials *channel_creds, const char *target,
-    const grpc_channel_args *args, grpc_channel_security_connector **sc,
-    grpc_channel_args **new_args) {
-  *new_args = NULL;
-  if (channel_creds == NULL) {
-    return GRPC_SECURITY_ERROR;
-  }
-  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
-  return channel_creds->vtable->create_security_connector(
-      channel_creds, NULL, target, args, sc, new_args);
-}
-
-grpc_server_credentials *grpc_server_credentials_ref(
-    grpc_server_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_server_credentials_unref(grpc_server_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-      creds->processor.destroy(creds->processor.state);
-    }
-    gpr_free(creds);
-  }
-}
-
-void grpc_server_credentials_release(grpc_server_credentials *creds) {
-  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
-  grpc_server_credentials_unref(creds);
-}
-
-grpc_security_status grpc_server_credentials_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
-    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
-    return GRPC_SECURITY_ERROR;
-  }
-  return creds->vtable->create_security_connector(creds, sc);
-}
-
-void grpc_server_credentials_set_auth_metadata_processor(
-    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
-  GRPC_API_TRACE(
-      "grpc_server_credentials_set_auth_metadata_processor("
-      "creds=%p, "
-      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
-      3, (creds, (void *)(intptr_t)processor.process, processor.state));
-  if (creds == NULL) return;
-  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-    creds->processor.destroy(creds->processor.state);
-  }
-  creds->processor = processor;
-}
-
-static void server_credentials_pointer_arg_destroy(void *p) {
-  grpc_server_credentials_unref(p);
-}
-
-static void *server_credentials_pointer_arg_copy(void *p) {
-  return grpc_server_credentials_ref(p);
-}
-
-static int server_credentials_pointer_cmp(void *a, void *b) {
-  return GPR_ICMP(a, b);
-}
-
-static const grpc_arg_pointer_vtable cred_ptr_vtable = {
-    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
-    server_credentials_pointer_cmp};
-
-grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
-  grpc_arg arg;
-  memset(&arg, 0, sizeof(grpc_arg));
-  arg.type = GRPC_ARG_POINTER;
-  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
-  arg.value.pointer.p = p;
-  arg.value.pointer.vtable = &cred_ptr_vtable;
-  return arg;
-}
-
-grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
-  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
-  if (arg->type != GRPC_ARG_POINTER) {
-    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
-            GRPC_SERVER_CREDENTIALS_ARG);
-    return NULL;
-  }
-  return arg->value.pointer.p;
-}
-
-grpc_server_credentials *grpc_find_server_credentials_in_args(
-    const grpc_channel_args *args) {
-  size_t i;
-  if (args == NULL) return NULL;
-  for (i = 0; i < args->num_args; i++) {
-    grpc_server_credentials *p =
-        grpc_server_credentials_from_arg(&args->args[i]);
-    if (p != NULL) return p;
-  }
-  return NULL;
-}
-
-/* -- Ssl credentials. -- */
-
-static void ssl_destruct(grpc_channel_credentials *creds) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
-  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
-}
-
-static void ssl_server_destruct(grpc_server_credentials *creds) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
-    if (c->config.pem_private_keys[i] != NULL) {
-      gpr_free(c->config.pem_private_keys[i]);
-    }
-    if (c->config.pem_cert_chains[i] != NULL) {
-      gpr_free(c->config.pem_cert_chains[i]);
-    }
-  }
-  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
-  if (c->config.pem_private_keys_sizes != NULL) {
-    gpr_free(c->config.pem_private_keys_sizes);
-  }
-  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
-  if (c->config.pem_cert_chains_sizes != NULL) {
-    gpr_free(c->config.pem_cert_chains_sizes);
-  }
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-}
-
-static grpc_security_status ssl_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_OK;
-  size_t i = 0;
-  const char *overridden_target_name = NULL;
-  grpc_arg new_arg;
-
-  for (i = 0; args && i < args->num_args; i++) {
-    grpc_arg *arg = &args->args[i];
-    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
-        arg->type == GRPC_ARG_STRING) {
-      overridden_target_name = arg->value.string;
-      break;
-    }
-  }
-  status = grpc_ssl_channel_security_connector_create(
-      call_creds, &c->config, target, overridden_target_name, sc);
-  if (status != GRPC_SECURITY_OK) {
-    return status;
-  }
-  new_arg.type = GRPC_ARG_STRING;
-  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
-  new_arg.value.string = "https";
-  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
-  return status;
-}
-
-static grpc_security_status ssl_server_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  return grpc_ssl_server_security_connector_create(&c->config, sc);
-}
-
-static grpc_channel_credentials_vtable ssl_vtable = {
-    ssl_destruct, ssl_create_security_connector};
-
-static grpc_server_credentials_vtable ssl_server_vtable = {
-    ssl_server_destruct, ssl_server_create_security_connector};
-
-static void ssl_copy_key_material(const char *input, unsigned char **output,
-                                  size_t *output_size) {
-  *output_size = strlen(input);
-  *output = gpr_malloc(*output_size);
-  memcpy(*output, input, *output_size);
-}
-
-static void ssl_build_config(const char *pem_root_certs,
-                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-                             grpc_ssl_config *config) {
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (pem_key_cert_pair != NULL) {
-    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pair->private_key,
-                          &config->pem_private_key,
-                          &config->pem_private_key_size);
-    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
-                          &config->pem_cert_chain,
-                          &config->pem_cert_chain_size);
-  }
-}
-
-static void ssl_build_server_config(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    grpc_ssl_server_config *config) {
-  size_t i;
-  config->client_certificate_request = client_certificate_request;
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (num_key_cert_pairs > 0) {
-    GPR_ASSERT(pem_key_cert_pairs != NULL);
-    config->pem_private_keys =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_cert_chains =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_private_keys_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-    config->pem_cert_chains_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-  }
-  config->num_key_cert_pairs = num_key_cert_pairs;
-  for (i = 0; i < num_key_cert_pairs; i++) {
-    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
-                          &config->pem_private_keys[i],
-                          &config->pem_private_keys_sizes[i]);
-    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
-                          &config->pem_cert_chains[i],
-                          &config->pem_cert_chains_sizes[i]);
-  }
-}
-
-grpc_channel_credentials *grpc_ssl_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-    void *reserved) {
-  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_credentials_create(pem_root_certs=%s, "
-      "pem_key_cert_pair=%p, "
-      "reserved=%p)",
-      3, (pem_root_certs, pem_key_cert_pair, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  c->base.vtable = &ssl_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
-  return &c->base;
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
-  return grpc_ssl_server_credentials_create_ex(
-      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
-      force_client_auth
-          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
-          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
-      reserved);
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    void *reserved) {
-  grpc_ssl_server_credentials *c =
-      gpr_malloc(sizeof(grpc_ssl_server_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_server_credentials_create_ex("
-      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
-      "client_certificate_request=%d, reserved=%p)",
-      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
-          client_certificate_request, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_server_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &ssl_server_vtable;
-  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
-                          num_key_cert_pairs, client_certificate_request,
-                          &c->config);
-  return &c->base;
-}
-
-/* -- Jwt credentials -- */
-
-static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
-  if (c->cached.service_url != NULL) {
-    gpr_free(c->cached.service_url);
-    c->cached.service_url = NULL;
-  }
-  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-}
-
-static void jwt_destruct(grpc_call_credentials *creds) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  grpc_auth_json_key_destruct(&c->key);
-  jwt_reset_cache(c);
-  gpr_mu_destroy(&c->cache_mu);
-}
-
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pops *pops,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-
-  /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
-  {
-    gpr_mu_lock(&c->cache_mu);
-    if (c->cached.service_url != NULL &&
-        strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
-        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
-                                   gpr_now(GPR_CLOCK_REALTIME)),
-                      refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md == NULL) {
-    char *jwt = NULL;
-    /* Generate a new jwt. */
-    gpr_mu_lock(&c->cache_mu);
-    jwt_reset_cache(c);
-    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
-                                   c->jwt_lifetime, NULL);
-    if (jwt != NULL) {
-      char *md_value;
-      gpr_asprintf(&md_value, "Bearer %s", jwt);
-      gpr_free(jwt);
-      c->cached.jwt_expiration =
-          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
-      c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
-      gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(jwt_md);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  }
-}
-
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
-
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime) {
-  grpc_service_account_jwt_access_credentials *c;
-  if (!grpc_auth_json_key_is_valid(&key)) {
-    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
-  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &jwt_vtable;
-  c->key = key;
-  c->jwt_lifetime = token_lifetime;
-  gpr_mu_init(&c->cache_mu);
-  jwt_reset_cache(c);
-  return &c->base;
-}
-
-grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
-    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_service_account_jwt_access_credentials_create("
-      "json_key=%s, "
-      "token_lifetime="
-      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
-      "reserved=%p)",
-      5,
-      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
-       (int)token_lifetime.clock_type, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
-}
-
-/* -- Oauth2TokenFetcher credentials -- */
-
-static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-  gpr_mu_destroy(&c->mu);
-  grpc_httpcli_context_destroy(&c->httpcli_context);
-}
-
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const grpc_http_response *response, grpc_credentials_md_store **token_md,
-    gpr_timespec *token_lifetime) {
-  char *null_terminated_body = NULL;
-  char *new_access_token = NULL;
-  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
-  grpc_json *json = NULL;
-
-  if (response == NULL) {
-    gpr_log(GPR_ERROR, "Received NULL response.");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  }
-
-  if (response->body_length > 0) {
-    null_terminated_body = gpr_malloc(response->body_length + 1);
-    null_terminated_body[response->body_length] = '\0';
-    memcpy(null_terminated_body, response->body, response->body_length);
-  }
-
-  if (response->status != 200) {
-    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
-            response->status,
-            null_terminated_body != NULL ? null_terminated_body : "");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  } else {
-    grpc_json *access_token = NULL;
-    grpc_json *token_type = NULL;
-    grpc_json *expires_in = NULL;
-    grpc_json *ptr;
-    json = grpc_json_parse_string(null_terminated_body);
-    if (json == NULL) {
-      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (json->type != GRPC_JSON_OBJECT) {
-      gpr_log(GPR_ERROR, "Response should be a JSON object");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    for (ptr = json->child; ptr; ptr = ptr->next) {
-      if (strcmp(ptr->key, "access_token") == 0) {
-        access_token = ptr;
-      } else if (strcmp(ptr->key, "token_type") == 0) {
-        token_type = ptr;
-      } else if (strcmp(ptr->key, "expires_in") == 0) {
-        expires_in = ptr;
-      }
-    }
-    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
-      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
-                 access_token->value);
-    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
-    token_lifetime->tv_nsec = 0;
-    token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
-    status = GRPC_CREDENTIALS_OK;
-  }
-
-end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(*token_md);
-    *token_md = NULL;
-  }
-  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
-  if (new_access_token != NULL) gpr_free(new_access_token);
-  if (json != NULL) grpc_json_destroy(json);
-  return status;
-}
-
-static void on_oauth2_token_fetcher_http_response(
-    grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_http_response *response) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)r->creds;
-  gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, status);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status);
-  }
-  gpr_mu_unlock(&c->mu);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void oauth2_token_fetcher_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
-    gpr_mu_unlock(&c->mu);
-  }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(cached_access_token_md);
-  } else {
-    c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pops, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
-  }
-}
-
-static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
-                                      grpc_fetch_oauth2_func fetch_func) {
-  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  gpr_ref_init(&c->base.refcount, 1);
-  gpr_mu_init(&c->mu);
-  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-  c->fetch_func = fetch_func;
-  grpc_httpcli_context_init(&c->httpcli_context);
-}
-
-/* -- GoogleComputeEngine credentials. -- */
-
-static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void compute_engine_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pops *pops,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_http_header header = {"Metadata-Flavor", "Google"};
-  grpc_httpcli_request request;
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
-  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  grpc_httpcli_get(exec_ctx, httpcli_context, pops, &request, deadline,
-                   response_cb, metadata_req);
-}
-
-grpc_call_credentials *grpc_google_compute_engine_credentials_create(
-    void *reserved) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
-  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
-  c->base.vtable = &compute_engine_vtable;
-  return &c->base;
-}
-
-/* -- GoogleRefreshToken credentials. -- */
-
-static void refresh_token_destruct(grpc_call_credentials *creds) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)creds;
-  grpc_auth_refresh_token_destruct(&c->refresh_token);
-  oauth2_token_fetcher_destruct(&c->base.base);
-}
-
-static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void refresh_token_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pops *pops,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)metadata_req->creds;
-  grpc_http_header header = {"Content-Type",
-                             "application/x-www-form-urlencoded"};
-  grpc_httpcli_request request;
-  char *body = NULL;
-  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
-               c->refresh_token.client_id, c->refresh_token.client_secret,
-               c->refresh_token.refresh_token);
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
-  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  request.handshaker = &grpc_httpcli_ssl;
-  grpc_httpcli_post(exec_ctx, httpcli_context, pops, &request, body,
-                    strlen(body), deadline, response_cb, metadata_req);
-  gpr_free(body);
-}
-
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token refresh_token) {
-  grpc_google_refresh_token_credentials *c;
-  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
-    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
-  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
-  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
-  c->base.base.vtable = &refresh_token_vtable;
-  c->refresh_token = refresh_token;
-  return &c->base.base;
-}
-
-grpc_call_credentials *grpc_google_refresh_token_credentials_create(
-    const char *json_refresh_token, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
-      "reserved=%p)",
-      2, (json_refresh_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
-      grpc_auth_refresh_token_create_from_string(json_refresh_token));
-}
-
-/* -- Metadata-only credentials. -- */
-
-static void md_only_test_destruct(grpc_call_credentials *creds) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(c->md_store);
-}
-
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, bool success) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void md_only_test_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
-  if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    grpc_executor_enqueue(
-        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
-
-grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
-  grpc_md_only_test_credentials *c =
-      gpr_malloc(sizeof(grpc_md_only_test_credentials));
-  memset(c, 0, sizeof(grpc_md_only_test_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &md_only_test_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
-  c->is_async = is_async;
-  return &c->base;
-}
-
-/* -- Oauth2 Access Token credentials. -- */
-
-static void access_token_destruct(grpc_call_credentials *creds) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-}
-
-static void access_token_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
-
-grpc_call_credentials *grpc_access_token_credentials_create(
-    const char *access_token, void *reserved) {
-  grpc_access_token_credentials *c =
-      gpr_malloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
-  GRPC_API_TRACE(
-      "grpc_access_token_credentials_create(access_token=%s, "
-      "reserved=%p)",
-      2, (access_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_access_token_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &access_token_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
-  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
-  gpr_free(token_md_value);
-  return &c->base;
-}
-
-/* -- Fake transport security credentials. -- */
-
-static grpc_security_status fake_transport_security_create_security_connector(
-    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  *sc = grpc_fake_channel_security_connector_create(call_creds);
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_security_status
-fake_transport_security_server_create_security_connector(
-    grpc_server_credentials *c, grpc_server_security_connector **sc) {
-  *sc = grpc_fake_server_security_connector_create();
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_channel_credentials_vtable
-    fake_transport_security_credentials_vtable = {
-        NULL, fake_transport_security_create_security_connector};
-
-static grpc_server_credentials_vtable
-    fake_transport_security_server_credentials_vtable = {
-        NULL, fake_transport_security_server_create_security_connector};
-
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
-    void) {
-  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
-  memset(c, 0, sizeof(grpc_channel_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  c->vtable = &fake_transport_security_credentials_vtable;
-  gpr_ref_init(&c->refcount, 1);
-  return c;
-}
-
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void) {
-  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
-  memset(c, 0, sizeof(grpc_server_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  gpr_ref_init(&c->refcount, 1);
-  c->vtable = &fake_transport_security_server_credentials_vtable;
-  return c;
-}
-
-/* -- Composite call credentials. -- */
-
-typedef struct {
-  grpc_composite_call_credentials *composite_creds;
-  size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
-  grpc_pops *pops;
-  grpc_credentials_metadata_cb cb;
-} grpc_composite_call_credentials_metadata_context;
-
-static void composite_call_destruct(grpc_call_credentials *creds) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
-    grpc_call_credentials_unref(c->inner.creds_array[i]);
-  }
-  gpr_free(c->inner.creds_array);
-}
-
-static void composite_call_md_context_destroy(
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status) {
-  grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
-    }
-  }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(exec_ctx, inner_creds, ctx->pops,
-                                               ctx->auth_md_context,
-                                               composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
-  composite_call_md_context_destroy(ctx);
-}
-
-static void composite_call_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
-    grpc_auth_metadata_context auth_md_context, grpc_credentials_metadata_cb cb,
-    void *user_data) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  grpc_composite_call_credentials_metadata_context *ctx;
-
-  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
-  ctx->composite_creds = c;
-  ctx->pops = pops;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], pops, auth_md_context,
-      composite_call_metadata_cb, ctx);
-}
-
-static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
-
-static grpc_call_credentials_array get_creds_array(
-    grpc_call_credentials **creds_addr) {
-  grpc_call_credentials_array result;
-  grpc_call_credentials *creds = *creds_addr;
-  result.creds_array = creds_addr;
-  result.num_creds = 1;
-  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    result = *grpc_composite_call_credentials_get_credentials(creds);
-  }
-  return result;
-}
-
-grpc_call_credentials *grpc_composite_call_credentials_create(
-    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
-    void *reserved) {
-  size_t i;
-  size_t creds_array_byte_size;
-  grpc_call_credentials_array creds1_array;
-  grpc_call_credentials_array creds2_array;
-  grpc_composite_call_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
-      "reserved=%p)",
-      3, (creds1, creds2, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(creds1 != NULL);
-  GPR_ASSERT(creds2 != NULL);
-  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
-  memset(c, 0, sizeof(grpc_composite_call_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
-  c->base.vtable = &composite_call_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  creds1_array = get_creds_array(&creds1);
-  creds2_array = get_creds_array(&creds2);
-  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
-  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
-  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
-  memset(c->inner.creds_array, 0, creds_array_byte_size);
-  for (i = 0; i < creds1_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
-    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
-  }
-  for (i = 0; i < creds2_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
-    c->inner.creds_array[i + creds1_array.num_creds] =
-        grpc_call_credentials_ref(cur_creds);
-  }
-  return &c->base;
-}
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
-  const grpc_composite_call_credentials *c =
-      (const grpc_composite_call_credentials *)creds;
-  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
-  return &c->inner;
-}
-
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds) {
-  size_t i;
-  if (strcmp(creds->type, type) == 0) {
-    if (composite_creds != NULL) *composite_creds = NULL;
-    return creds;
-  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    const grpc_call_credentials_array *inner_creds_array =
-        grpc_composite_call_credentials_get_credentials(creds);
-    for (i = 0; i < inner_creds_array->num_creds; i++) {
-      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
-        if (composite_creds != NULL) *composite_creds = creds;
-        return inner_creds_array->creds_array[i];
-      }
-    }
-  }
-  return NULL;
-}
-
-/* -- IAM credentials. -- */
-
-static void iam_destruct(grpc_call_credentials *creds) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(c->iam_md);
-}
-
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pops *pops,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
-
-grpc_call_credentials *grpc_google_iam_credentials_create(
-    const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
-      "reserved=%p)",
-      3, (token, authority_selector, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(token != NULL);
-  GPR_ASSERT(authority_selector != NULL);
-  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
-  memset(c, 0, sizeof(grpc_google_iam_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
-  c->base.vtable = &iam_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
-  return &c->base;
-}
-
-/* -- Plugin credentials. -- */
-
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
-static void plugin_destruct(grpc_call_credentials *creds) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
-    c->plugin.destroy(c->plugin.state);
-  }
-}
-
-static void plugin_md_request_metadata_ready(void *request,
-                                             const grpc_metadata *md,
-                                             size_t num_md,
-                                             grpc_status_code status,
-                                             const char *error_details) {
-  /* called from application code */
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  } else {
-    size_t i;
-    grpc_credentials_md *md_array = NULL;
-    if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
-        md_array[i].value =
-            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
-      }
-    }
-    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
-    if (md_array != NULL) {
-      for (i = 0; i < num_md; i++) {
-        gpr_slice_unref(md_array[i].key);
-        gpr_slice_unref(md_array[i].value);
-      }
-      gpr_free(md_array);
-    }
-  }
-  gpr_free(r);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_credentials *creds,
-                                        grpc_pops *pops,
-                                        grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
-    memset(request, 0, sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
-    c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
-
-grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
-    grpc_metadata_credentials_plugin plugin, void *reserved) {
-  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
-  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(*c));
-  c->base.type = plugin.type;
-  c->base.vtable = &plugin_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->plugin = plugin;
-  return &c->base;
-}
-
-/* -- Composite channel credentials. -- */
-
-static void composite_channel_destruct(grpc_channel_credentials *creds) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_channel_credentials_unref(c->inner_creds);
-  grpc_call_credentials_unref(c->call_creds);
-}
-
-static grpc_security_status composite_channel_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_ERROR;
-
-  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
-             c->inner_creds->vtable != NULL &&
-             c->inner_creds->vtable->create_security_connector != NULL);
-  /* If we are passed a call_creds, create a call composite to pass it
-     downstream. */
-  if (call_creds != NULL) {
-    grpc_call_credentials *composite_call_creds =
-        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, composite_call_creds, target, args, sc, new_args);
-    grpc_call_credentials_unref(composite_call_creds);
-  } else {
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, c->call_creds, target, args, sc, new_args);
-  }
-  return status;
-}
-
-static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
-    composite_channel_destruct, composite_channel_create_security_connector};
-
-grpc_channel_credentials *grpc_composite_channel_credentials_create(
-    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
-    void *reserved) {
-  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
-  memset(c, 0, sizeof(*c));
-  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
-  GRPC_API_TRACE(
-      "grpc_composite_channel_credentials_create(channel_creds=%p, "
-      "call_creds=%p, reserved=%p)",
-      3, (channel_creds, call_creds, reserved));
-  c->base.type = channel_creds->type;
-  c->base.vtable = &composite_channel_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
-  c->call_creds = grpc_call_credentials_ref(call_creds);
-  return &c->base;
-}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
new file mode 100644
index 0000000000..38abd84b04
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -0,0 +1,263 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/composite/composite_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/iomgr/pops.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Composite call credentials. -- */
+
+typedef struct {
+  grpc_composite_call_credentials *composite_creds;
+  size_t creds_index;
+  grpc_credentials_md_store *md_elems;
+  grpc_auth_metadata_context auth_md_context;
+  void *user_data;
+  grpc_pops *pops;
+  grpc_credentials_metadata_cb cb;
+} grpc_composite_call_credentials_metadata_context;
+
+static void composite_call_destruct(grpc_call_credentials *creds) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->inner.num_creds; i++) {
+    grpc_call_credentials_unref(c->inner.creds_array[i]);
+  }
+  gpr_free(c->inner.creds_array);
+}
+
+static void composite_call_md_context_destroy(
+    grpc_composite_call_credentials_metadata_context *ctx) {
+  grpc_credentials_md_store_unref(ctx->md_elems);
+  gpr_free(ctx);
+}
+
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
+                                       grpc_credentials_md *md_elems,
+                                       size_t num_md,
+                                       grpc_credentials_status status) {
+  grpc_composite_call_credentials_metadata_context *ctx =
+      (grpc_composite_call_credentials_metadata_context *)user_data;
+  if (status != GRPC_CREDENTIALS_OK) {
+    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
+    return;
+  }
+
+  /* Copy the metadata in the context. */
+  if (num_md > 0) {
+    size_t i;
+    for (i = 0; i < num_md; i++) {
+      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
+                                    md_elems[i].value);
+    }
+  }
+
+  /* See if we need to get some more metadata. */
+  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    grpc_call_credentials_get_request_metadata(exec_ctx, inner_creds, ctx->pops,
+                                               ctx->auth_md_context,
+                                               composite_call_metadata_cb, ctx);
+    return;
+  }
+
+  /* We're done!. */
+  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
+          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
+  composite_call_md_context_destroy(ctx);
+}
+
+static void composite_call_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, grpc_pops *pops,
+    grpc_auth_metadata_context auth_md_context, grpc_credentials_metadata_cb cb,
+    void *user_data) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  grpc_composite_call_credentials_metadata_context *ctx;
+
+  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
+  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
+  ctx->auth_md_context = auth_md_context;
+  ctx->user_data = user_data;
+  ctx->cb = cb;
+  ctx->composite_creds = c;
+  ctx->pops = pops;
+  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
+  grpc_call_credentials_get_request_metadata(
+      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pops,
+      auth_md_context, composite_call_metadata_cb, ctx);
+}
+
+static grpc_call_credentials_vtable composite_call_credentials_vtable = {
+    composite_call_destruct, composite_call_get_request_metadata};
+
+static grpc_call_credentials_array get_creds_array(
+    grpc_call_credentials **creds_addr) {
+  grpc_call_credentials_array result;
+  grpc_call_credentials *creds = *creds_addr;
+  result.creds_array = creds_addr;
+  result.num_creds = 1;
+  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    result = *grpc_composite_call_credentials_get_credentials(creds);
+  }
+  return result;
+}
+
+grpc_call_credentials *grpc_composite_call_credentials_create(
+    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
+    void *reserved) {
+  size_t i;
+  size_t creds_array_byte_size;
+  grpc_call_credentials_array creds1_array;
+  grpc_call_credentials_array creds2_array;
+  grpc_composite_call_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
+      "reserved=%p)",
+      3, (creds1, creds2, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(creds1 != NULL);
+  GPR_ASSERT(creds2 != NULL);
+  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
+  memset(c, 0, sizeof(grpc_composite_call_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
+  c->base.vtable = &composite_call_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  creds1_array = get_creds_array(&creds1);
+  creds2_array = get_creds_array(&creds2);
+  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
+  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
+  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
+  memset(c->inner.creds_array, 0, creds_array_byte_size);
+  for (i = 0; i < creds1_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
+    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
+  }
+  for (i = 0; i < creds2_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
+    c->inner.creds_array[i + creds1_array.num_creds] =
+        grpc_call_credentials_ref(cur_creds);
+  }
+  return &c->base;
+}
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
+  const grpc_composite_call_credentials *c =
+      (const grpc_composite_call_credentials *)creds;
+  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
+  return &c->inner;
+}
+
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds) {
+  size_t i;
+  if (strcmp(creds->type, type) == 0) {
+    if (composite_creds != NULL) *composite_creds = NULL;
+    return creds;
+  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    const grpc_call_credentials_array *inner_creds_array =
+        grpc_composite_call_credentials_get_credentials(creds);
+    for (i = 0; i < inner_creds_array->num_creds; i++) {
+      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
+        if (composite_creds != NULL) *composite_creds = creds;
+        return inner_creds_array->creds_array[i];
+      }
+    }
+  }
+  return NULL;
+}
+
+/* -- Composite channel credentials. -- */
+
+static void composite_channel_destruct(grpc_channel_credentials *creds) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_channel_credentials_unref(c->inner_creds);
+  grpc_call_credentials_unref(c->call_creds);
+}
+
+static grpc_security_status composite_channel_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_ERROR;
+
+  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
+             c->inner_creds->vtable != NULL &&
+             c->inner_creds->vtable->create_security_connector != NULL);
+  /* If we are passed a call_creds, create a call composite to pass it
+     downstream. */
+  if (call_creds != NULL) {
+    grpc_call_credentials *composite_call_creds =
+        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, composite_call_creds, target, args, sc, new_args);
+    grpc_call_credentials_unref(composite_call_creds);
+  } else {
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, c->call_creds, target, args, sc, new_args);
+  }
+  return status;
+}
+
+static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
+    composite_channel_destruct, composite_channel_create_security_connector};
+
+grpc_channel_credentials *grpc_composite_channel_credentials_create(
+    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
+    void *reserved) {
+  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
+  memset(c, 0, sizeof(*c));
+  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
+  GRPC_API_TRACE(
+      "grpc_composite_channel_credentials_create(channel_creds=%p, "
+      "call_creds=%p, reserved=%p)",
+      3, (channel_creds, call_creds, reserved));
+  c->base.type = channel_creds->type;
+  c->base.vtable = &composite_channel_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
+  c->call_creds = grpc_call_credentials_ref(call_creds);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h
new file mode 100644
index 0000000000..0d8966f464
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.h
@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials **creds_array;
+  size_t num_creds;
+} grpc_call_credentials_array;
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(
+    grpc_call_credentials *composite_creds);
+
+/* Returns creds if creds is of the specified type or the inner creds of the
+   specified type (if found), if the creds is of type COMPOSITE.
+   If composite_creds is not NULL, *composite_creds will point to creds if of
+   type COMPOSITE in case of success. */
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds);
+
+/* -- Channel composite credentials. -- */
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_channel_credentials *inner_creds;
+  grpc_call_credentials *call_creds;
+} grpc_composite_channel_credentials;
+
+/* -- Composite credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_call_credentials_array inner;
+} grpc_composite_call_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
new file mode 100644
index 0000000000..b33deefb3d
--- /dev/null
+++ b/src/core/lib/security/credentials/credentials.c
@@ -0,0 +1,231 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+
+/* -- Common. -- */
+
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data) {
+  grpc_credentials_metadata_request *r =
+      gpr_malloc(sizeof(grpc_credentials_metadata_request));
+  r->creds = grpc_call_credentials_ref(creds);
+  r->cb = cb;
+  r->user_data = user_data;
+  return r;
+}
+
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r) {
+  grpc_call_credentials_unref(r->creds);
+  gpr_free(r);
+}
+
+grpc_channel_credentials *grpc_channel_credentials_ref(
+    grpc_channel_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
+  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
+  grpc_channel_credentials_unref(creds);
+}
+
+grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_call_credentials_unref(grpc_call_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_call_credentials_release(grpc_call_credentials *creds) {
+  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
+  grpc_call_credentials_unref(creds);
+}
+
+void grpc_call_credentials_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops *pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
+    if (cb != NULL) {
+      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+    }
+    return;
+  }
+  creds->vtable->get_request_metadata(exec_ctx, creds, pops, context, cb,
+                                      user_data);
+}
+
+grpc_security_status grpc_channel_credentials_create_security_connector(
+    grpc_channel_credentials *channel_creds, const char *target,
+    const grpc_channel_args *args, grpc_channel_security_connector **sc,
+    grpc_channel_args **new_args) {
+  *new_args = NULL;
+  if (channel_creds == NULL) {
+    return GRPC_SECURITY_ERROR;
+  }
+  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
+  return channel_creds->vtable->create_security_connector(
+      channel_creds, NULL, target, args, sc, new_args);
+}
+
+grpc_server_credentials *grpc_server_credentials_ref(
+    grpc_server_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_server_credentials_unref(grpc_server_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+      creds->processor.destroy(creds->processor.state);
+    }
+    gpr_free(creds);
+  }
+}
+
+void grpc_server_credentials_release(grpc_server_credentials *creds) {
+  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
+  grpc_server_credentials_unref(creds);
+}
+
+grpc_security_status grpc_server_credentials_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
+    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
+    return GRPC_SECURITY_ERROR;
+  }
+  return creds->vtable->create_security_connector(creds, sc);
+}
+
+void grpc_server_credentials_set_auth_metadata_processor(
+    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
+  GRPC_API_TRACE(
+      "grpc_server_credentials_set_auth_metadata_processor("
+      "creds=%p, "
+      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
+      3, (creds, (void *)(intptr_t)processor.process, processor.state));
+  if (creds == NULL) return;
+  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+    creds->processor.destroy(creds->processor.state);
+  }
+  creds->processor = processor;
+}
+
+static void server_credentials_pointer_arg_destroy(void *p) {
+  grpc_server_credentials_unref(p);
+}
+
+static void *server_credentials_pointer_arg_copy(void *p) {
+  return grpc_server_credentials_ref(p);
+}
+
+static int server_credentials_pointer_cmp(void *a, void *b) {
+  return GPR_ICMP(a, b);
+}
+
+static const grpc_arg_pointer_vtable cred_ptr_vtable = {
+    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
+    server_credentials_pointer_cmp};
+
+grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
+  grpc_arg arg;
+  memset(&arg, 0, sizeof(grpc_arg));
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
+  arg.value.pointer.p = p;
+  arg.value.pointer.vtable = &cred_ptr_vtable;
+  return arg;
+}
+
+grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
+  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
+  if (arg->type != GRPC_ARG_POINTER) {
+    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
+            GRPC_SERVER_CREDENTIALS_ARG);
+    return NULL;
+  }
+  return arg->value.pointer.p;
+}
+
+grpc_server_credentials *grpc_find_server_credentials_in_args(
+    const grpc_channel_args *args) {
+  size_t i;
+  if (args == NULL) return NULL;
+  for (i = 0; i < args->num_args; i++) {
+    grpc_server_credentials *p =
+        grpc_server_credentials_from_arg(&args->args[i]);
+    if (p != NULL) return p;
+  }
+  return NULL;
+}
diff --git a/src/core/lib/security/credentials.h b/src/core/lib/security/credentials/credentials.h
index 3d374949d5..d3531bcfc0 100644
--- a/src/core/lib/security/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
-#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
@@ -42,8 +42,7 @@
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/pops.h"
-#include "src/core/lib/security/json_token.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 struct grpc_http_response;
 
@@ -70,10 +69,6 @@ typedef enum {
   "x-goog-iam-authorization-token"
 #define GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY "x-goog-iam-authority-selector"
 
-#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
-#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
-  "application_default_credentials.json"
-
 #define GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS 60
 
 #define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata"
@@ -189,48 +184,11 @@ void grpc_call_credentials_get_request_metadata(
     grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb,
     void *user_data);
 
-typedef struct {
-  grpc_call_credentials **creds_array;
-  size_t num_creds;
-} grpc_call_credentials_array;
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(
-    grpc_call_credentials *composite_creds);
-
-/* Returns creds if creds is of the specified type or the inner creds of the
-   specified type (if found), if the creds is of type COMPOSITE.
-   If composite_creds is not NULL, *composite_creds will point to creds if of
-   type COMPOSITE in case of success. */
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds);
-
-/* Exposed for testing only. */
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
-
-void grpc_flush_cached_google_default_credentials(void);
-
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
     const char *md_key, const char *md_value, int is_async);
 
-/* Private constructor for jwt credentials from an already parsed json key.
-   Takes ownership of the key. */
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime);
-
-/* Private constructor for refresh token credentials from an already parsed
-   refresh token. Takes ownership of the refresh token. */
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token token);
-
 /* --- grpc_server_credentials. --- */
 
 typedef struct {
@@ -261,118 +219,19 @@ grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg);
 grpc_server_credentials *grpc_find_server_credentials_in_args(
     const grpc_channel_args *args);
 
-/* -- Fake transport security credentials. -- */
-
-/* Creates a fake transport security credentials object for testing. */
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
-/* Creates a fake server transport security credentials object for testing. */
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void);
-
-/* -- Ssl credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_ssl_config config;
-} grpc_ssl_credentials;
-
-typedef struct {
-  grpc_server_credentials base;
-  grpc_ssl_server_config config;
-} grpc_ssl_server_credentials;
-
-/* -- Channel composite credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_channel_credentials *inner_creds;
-  grpc_call_credentials *call_creds;
-} grpc_composite_channel_credentials;
-
-/* -- Jwt credentials -- */
-
-typedef struct {
-  grpc_call_credentials base;
-
-  /* Have a simple cache for now with just 1 entry. We could have a map based on
-     the service_url for a more sophisticated one. */
-  gpr_mu cache_mu;
-  struct {
-    grpc_credentials_md_store *jwt_md;
-    char *service_url;
-    gpr_timespec jwt_expiration;
-  } cached;
-
-  grpc_auth_json_key key;
-  gpr_timespec jwt_lifetime;
-} grpc_service_account_jwt_access_credentials;
-
-/* -- Oauth2TokenFetcher credentials --
-
-   This object is a base for credentials that need to acquire an oauth2 token
-   from an http service. */
-
-typedef struct grpc_credentials_metadata_request
-    grpc_credentials_metadata_request;
-
-typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
-                                       grpc_credentials_metadata_request *req,
-                                       grpc_httpcli_context *http_context,
-                                       grpc_pops *pops,
-                                       grpc_httpcli_response_cb response_cb,
-                                       gpr_timespec deadline);
-
-typedef struct {
-  grpc_call_credentials base;
-  gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
-  gpr_timespec token_expiration;
-  grpc_httpcli_context httpcli_context;
-  grpc_fetch_oauth2_func fetch_func;
-} grpc_oauth2_token_fetcher_credentials;
-
-/* -- GoogleRefreshToken credentials. -- */
-
-typedef struct {
-  grpc_oauth2_token_fetcher_credentials base;
-  grpc_auth_refresh_token refresh_token;
-} grpc_google_refresh_token_credentials;
-
-/* -- Oauth2 Access Token credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
-} grpc_access_token_credentials;
-
-/* --  Metadata-only Test credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
-} grpc_md_only_test_credentials;
-
-/* -- GoogleIAM credentials. -- */
+/* -- Credentials Metadata Request. -- */
 
 typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
-} grpc_google_iam_credentials;
+  grpc_call_credentials *creds;
+  grpc_credentials_metadata_cb cb;
+  void *user_data;
+} grpc_credentials_metadata_request;
 
-/* -- Composite credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_call_credentials_array inner;
-} grpc_composite_call_credentials;
-
-/* -- Plugin credentials. -- */
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data);
 
-typedef struct {
-  grpc_call_credentials base;
-  grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
-} grpc_plugin_credentials;
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r);
 
-#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
index bd00194278..6a352aab3a 100644
--- a/src/core/lib/security/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <grpc/support/alloc.h>
 
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
new file mode 100644
index 0000000000..beb0e5ee1e
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -0,0 +1,138 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/executor.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Fake transport security credentials. -- */
+
+static grpc_security_status fake_transport_security_create_security_connector(
+    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  *sc = grpc_fake_channel_security_connector_create(call_creds);
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_security_status
+fake_transport_security_server_create_security_connector(
+    grpc_server_credentials *c, grpc_server_security_connector **sc) {
+  *sc = grpc_fake_server_security_connector_create();
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_channel_credentials_vtable
+    fake_transport_security_credentials_vtable = {
+        NULL, fake_transport_security_create_security_connector};
+
+static grpc_server_credentials_vtable
+    fake_transport_security_server_credentials_vtable = {
+        NULL, fake_transport_security_server_create_security_connector};
+
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
+    void) {
+  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
+  memset(c, 0, sizeof(grpc_channel_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  c->vtable = &fake_transport_security_credentials_vtable;
+  gpr_ref_init(&c->refcount, 1);
+  return c;
+}
+
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void) {
+  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
+  memset(c, 0, sizeof(grpc_server_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  gpr_ref_init(&c->refcount, 1);
+  c->vtable = &fake_transport_security_server_credentials_vtable;
+  return c;
+}
+
+/* -- Metadata-only test credentials. -- */
+
+static void md_only_test_destruct(grpc_call_credentials *creds) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+  grpc_credentials_md_store_unref(c->md_store);
+}
+
+static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
+                                          void *user_data, bool success) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
+  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
+        GRPC_CREDENTIALS_OK);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void md_only_test_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops *pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+
+  if (c->is_async) {
+    grpc_credentials_metadata_request *cb_arg =
+        grpc_credentials_metadata_request_create(creds, cb, user_data);
+    grpc_executor_enqueue(
+        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
+  } else {
+    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_call_credentials_vtable md_only_test_vtable = {
+    md_only_test_destruct, md_only_test_get_request_metadata};
+
+grpc_call_credentials *grpc_md_only_test_credentials_create(
+    const char *md_key, const char *md_value, int is_async) {
+  grpc_md_only_test_credentials *c =
+      gpr_malloc(sizeof(grpc_md_only_test_credentials));
+  memset(c, 0, sizeof(grpc_md_only_test_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &md_only_test_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->md_store = grpc_credentials_md_store_create(1);
+  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->is_async = is_async;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
new file mode 100644
index 0000000000..9cf38084a3
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+/* -- Fake transport security credentials. -- */
+
+/* Creates a fake transport security credentials object for testing. */
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
+
+/* Creates a fake server transport security credentials object for testing. */
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void);
+
+/* --  Metadata-only Test credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *md_store;
+  int is_async;
+} grpc_md_only_test_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_posix.c b/src/core/lib/security/credentials/google_default/credentials_posix.c
index a07de182a0..42c9d7f997 100644
--- a/src/core/lib/security/credentials_posix.c
+++ b/src/core/lib/security/credentials/google_default/credentials_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/credentials_win32.c b/src/core/lib/security/credentials/google_default/credentials_win32.c
index d29847af38..cd8b48080a 100644
--- a/src/core/lib/security/credentials_win32.c
+++ b/src/core/lib/security/credentials/google_default/credentials_win32.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WIN32
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c
index 395a23b1a7..ddea7a9ed2 100644
--- a/src/core/lib/security/google_default_credentials.c
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <string.h>
 
@@ -42,6 +42,8 @@
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/pops.h"
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/load_file.h"
 #include "src/core/lib/surface/api_trace.h"
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.h b/src/core/lib/security/credentials/google_default/google_default_credentials.h
new file mode 100644
index 0000000000..fac4377e2c
--- /dev/null
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
+#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
+  "application_default_credentials.json"
+
+void grpc_flush_cached_google_default_credentials(void);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
new file mode 100644
index 0000000000..90779cfc71
--- /dev/null
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/iam/iam_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void iam_destruct(grpc_call_credentials *creds) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  grpc_credentials_md_store_unref(c->iam_md);
+}
+
+static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_pops* pops,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
+     GRPC_CREDENTIALS_OK);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
+                                                  iam_get_request_metadata};
+
+grpc_call_credentials *grpc_google_iam_credentials_create(
+    const char *token, const char *authority_selector, void *reserved) {
+  grpc_google_iam_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
+      "reserved=%p)",
+      3, (token, authority_selector, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(token != NULL);
+  GPR_ASSERT(authority_selector != NULL);
+  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
+  memset(c, 0, sizeof(grpc_google_iam_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
+  c->base.vtable = &iam_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->iam_md = grpc_credentials_md_store_create(2);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  return &c->base;
+}
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h b/src/core/lib/security/credentials/iam/iam_credentials.h
index 06d6dbf29d..af54faa586 100644
--- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,11 +31,14 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H
-#define GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
 
-#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
-const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void);
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *iam_md;
+} grpc_google_iam_credentials;
 
-#endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/json_token.c b/src/core/lib/security/credentials/jwt/json_token.c
index d5bc2c8d60..354c13133e 100644
--- a/src/core/lib/security/json_token.c
+++ b/src/core/lib/security/credentials/jwt/json_token.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
 
 #include <string.h>
 
@@ -39,7 +39,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
+#include "src/core/lib/security/util/json_util.h"
 #include "src/core/lib/support/string.h"
 
 #include <openssl/bio.h>
@@ -66,28 +67,6 @@ static grpc_jwt_encode_and_sign_override g_jwt_encode_and_sign_override = NULL;
 
 /* --- grpc_auth_json_key. --- */
 
-static const char *json_get_string_property(const grpc_json *json,
-                                            const char *prop_name) {
-  grpc_json *child;
-  for (child = json->child; child != NULL; child = child->next) {
-    if (strcmp(child->key, prop_name) == 0) break;
-  }
-  if (child == NULL || child->type != GRPC_JSON_STRING) {
-    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
-    return NULL;
-  }
-  return child->value;
-}
-
-static int set_json_key_string_property(const grpc_json *json,
-                                        const char *prop_name,
-                                        char **json_key_field) {
-  const char *prop_value = json_get_string_property(json, prop_name);
-  if (prop_value == NULL) return 0;
-  *json_key_field = gpr_strdup(prop_value);
-  return 1;
-}
-
 int grpc_auth_json_key_is_valid(const grpc_auth_json_key *json_key) {
   return (json_key != NULL) &&
          strcmp(json_key->type, GRPC_AUTH_JSON_TYPE_INVALID);
@@ -106,22 +85,22 @@ grpc_auth_json_key grpc_auth_json_key_create_from_json(const grpc_json *json) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "type");
+  prop_value = grpc_json_get_string_property(json, "type");
   if (prop_value == NULL ||
       strcmp(prop_value, GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT)) {
     goto end;
   }
   result.type = GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT;
 
-  if (!set_json_key_string_property(json, "private_key_id",
-                                    &result.private_key_id) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "client_email",
-                                    &result.client_email)) {
+  if (!grpc_copy_json_string_property(json, "private_key_id",
+                                      &result.private_key_id) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "client_email",
+                                      &result.client_email)) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "private_key");
+  prop_value = grpc_json_get_string_property(json, "private_key");
   if (prop_value == NULL) {
     goto end;
   }
@@ -339,73 +318,3 @@ void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func) {
   g_jwt_encode_and_sign_override = func;
 }
-
-/* --- grpc_auth_refresh_token --- */
-
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token) {
-  return (refresh_token != NULL) &&
-         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json) {
-  grpc_auth_refresh_token result;
-  const char *prop_value;
-  int success = 0;
-
-  memset(&result, 0, sizeof(grpc_auth_refresh_token));
-  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (json == NULL) {
-    gpr_log(GPR_ERROR, "Invalid json.");
-    goto end;
-  }
-
-  prop_value = json_get_string_property(json, "type");
-  if (prop_value == NULL ||
-      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
-    goto end;
-  }
-  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
-
-  if (!set_json_key_string_property(json, "client_secret",
-                                    &result.client_secret) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "refresh_token",
-                                    &result.refresh_token)) {
-    goto end;
-  }
-  success = 1;
-
-end:
-  if (!success) grpc_auth_refresh_token_destruct(&result);
-  return result;
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string) {
-  char *scratchpad = gpr_strdup(json_string);
-  grpc_json *json = grpc_json_parse_string(scratchpad);
-  grpc_auth_refresh_token result =
-      grpc_auth_refresh_token_create_from_json(json);
-  if (json != NULL) grpc_json_destroy(json);
-  gpr_free(scratchpad);
-  return result;
-}
-
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
-  if (refresh_token == NULL) return;
-  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (refresh_token->client_id != NULL) {
-    gpr_free(refresh_token->client_id);
-    refresh_token->client_id = NULL;
-  }
-  if (refresh_token->client_secret != NULL) {
-    gpr_free(refresh_token->client_secret);
-    refresh_token->client_secret = NULL;
-  }
-  if (refresh_token->refresh_token != NULL) {
-    gpr_free(refresh_token->refresh_token);
-    refresh_token->refresh_token = NULL;
-  }
-}
diff --git a/src/core/lib/security/json_token.h b/src/core/lib/security/credentials/jwt/json_token.h
index 123fa652fd..07fc5bf0e0 100644
--- a/src/core/lib/security/json_token.h
+++ b/src/core/lib/security/credentials/jwt/json_token.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
-#define GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
 
 #include <grpc/support/slice.h>
 #include <openssl/rsa.h>
@@ -43,10 +43,6 @@
 
 #define GRPC_JWT_OAUTH2_AUDIENCE "https://www.googleapis.com/oauth2/v3/token"
 
-#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
-#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
-#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
-
 /* --- auth_json_key parsing. --- */
 
 typedef struct {
@@ -89,30 +85,4 @@ typedef char *(*grpc_jwt_encode_and_sign_override)(
 void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func);
 
-/* --- auth_refresh_token parsing. --- */
-
-typedef struct {
-  const char *type;
-  char *client_id;
-  char *client_secret;
-  char *refresh_token;
-} grpc_auth_refresh_token;
-
-/* Returns 1 if the object is valid, 0 otherwise. */
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token);
-
-/* Creates a refresh token object from string. Returns an invalid object if a
-   parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string);
-
-/* Creates a refresh token object from parsed json. Returns an invalid object if
-   a parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json);
-
-/* Destructs the object. */
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
-
-#endif /* GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
new file mode 100644
index 0000000000..590f29b7b7
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -0,0 +1,160 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
+  if (c->cached.jwt_md != NULL) {
+    grpc_credentials_md_store_unref(c->cached.jwt_md);
+    c->cached.jwt_md = NULL;
+  }
+  if (c->cached.service_url != NULL) {
+    gpr_free(c->cached.service_url);
+    c->cached.service_url = NULL;
+  }
+  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+}
+
+static void jwt_destruct(grpc_call_credentials *creds) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  grpc_auth_json_key_destruct(&c->key);
+  jwt_reset_cache(c);
+  gpr_mu_destroy(&c->cache_mu);
+}
+
+static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_pops* pops,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+
+  /* See if we can return a cached jwt. */
+  grpc_credentials_md_store *jwt_md = NULL;
+  {
+    gpr_mu_lock(&c->cache_mu);
+    if (c->cached.service_url != NULL &&
+        strcmp(c->cached.service_url, context.service_url) == 0 &&
+        c->cached.jwt_md != NULL &&
+        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
+                                   gpr_now(GPR_CLOCK_REALTIME)),
+                      refresh_threshold) > 0)) {
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md == NULL) {
+    char *jwt = NULL;
+    /* Generate a new jwt. */
+    gpr_mu_lock(&c->cache_mu);
+    jwt_reset_cache(c);
+    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
+                                   c->jwt_lifetime, NULL);
+    if (jwt != NULL) {
+      char *md_value;
+      gpr_asprintf(&md_value, "Bearer %s", jwt);
+      gpr_free(jwt);
+      c->cached.jwt_expiration =
+          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
+      c->cached.service_url = gpr_strdup(context.service_url);
+      c->cached.jwt_md = grpc_credentials_md_store_create(1);
+      grpc_credentials_md_store_add_cstrings(
+          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      gpr_free(md_value);
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md != NULL) {
+    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
+       GRPC_CREDENTIALS_OK);
+    grpc_credentials_md_store_unref(jwt_md);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  }
+}
+
+static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
+                                                  jwt_get_request_metadata};
+
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime) {
+  grpc_service_account_jwt_access_credentials *c;
+  if (!grpc_auth_json_key_is_valid(&key)) {
+    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
+  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &jwt_vtable;
+  c->key = key;
+  c->jwt_lifetime = token_lifetime;
+  gpr_mu_init(&c->cache_mu);
+  jwt_reset_cache(c);
+  return &c->base;
+}
+
+grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
+    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_service_account_jwt_access_credentials_create("
+      "json_key=%s, "
+      "token_lifetime="
+      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "reserved=%p)",
+      5,
+      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
+       (int)token_lifetime.clock_type, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
+}
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
new file mode 100644
index 0000000000..d572606179
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
+
+typedef struct {
+  grpc_call_credentials base;
+
+  // Have a simple cache for now with just 1 entry. We could have a map based on
+  // the service_url for a more sophisticated one.
+  gpr_mu cache_mu;
+  struct {
+    grpc_credentials_md_store *jwt_md;
+    char *service_url;
+    gpr_timespec jwt_expiration;
+  } cached;
+
+  grpc_auth_json_key key;
+  gpr_timespec jwt_lifetime;
+} grpc_service_account_jwt_access_credentials;
+
+// Private constructor for jwt credentials from an already parsed json key.
+// Takes ownership of the key.
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H */
diff --git a/src/core/lib/security/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
index 1b6b7b175d..a3e0c2d78c 100644
--- a/src/core/lib/security/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -31,14 +31,14 @@
  *
  */
 
-#include "src/core/lib/security/jwt_verifier.h"
+#include "src/core/lib/security/credentials/jwt/jwt_verifier.h"
 
 #include <limits.h>
 #include <string.h>
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/iomgr/pops.h"
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 #include "src/core/lib/tsi/ssl_types.h"
 
 #include <grpc/support/alloc.h>
diff --git a/src/core/lib/security/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h
index 98a4f6b116..b0f6d1c240 100644
--- a/src/core/lib/security/jwt_verifier.h
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
-#define GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
 
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/json/json.h"
@@ -133,4 +133,4 @@ grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer);
 grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
                                                const char *audience);
 
-#endif /* GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H */
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
new file mode 100644
index 0000000000..131aa69fb8
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -0,0 +1,428 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/security/util/json_util.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+//
+// Auth Refresh Token.
+//
+
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token) {
+  return (refresh_token != NULL) &&
+         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json) {
+  grpc_auth_refresh_token result;
+  const char *prop_value;
+  int success = 0;
+
+  memset(&result, 0, sizeof(grpc_auth_refresh_token));
+  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (json == NULL) {
+    gpr_log(GPR_ERROR, "Invalid json.");
+    goto end;
+  }
+
+  prop_value = grpc_json_get_string_property(json, "type");
+  if (prop_value == NULL ||
+      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
+    goto end;
+  }
+  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
+
+  if (!grpc_copy_json_string_property(json, "client_secret",
+                                      &result.client_secret) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "refresh_token",
+                                      &result.refresh_token)) {
+    goto end;
+  }
+  success = 1;
+
+end:
+  if (!success) grpc_auth_refresh_token_destruct(&result);
+  return result;
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string) {
+  char *scratchpad = gpr_strdup(json_string);
+  grpc_json *json = grpc_json_parse_string(scratchpad);
+  grpc_auth_refresh_token result =
+      grpc_auth_refresh_token_create_from_json(json);
+  if (json != NULL) grpc_json_destroy(json);
+  gpr_free(scratchpad);
+  return result;
+}
+
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
+  if (refresh_token == NULL) return;
+  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (refresh_token->client_id != NULL) {
+    gpr_free(refresh_token->client_id);
+    refresh_token->client_id = NULL;
+  }
+  if (refresh_token->client_secret != NULL) {
+    gpr_free(refresh_token->client_secret);
+    refresh_token->client_secret = NULL;
+  }
+  if (refresh_token->refresh_token != NULL) {
+    gpr_free(refresh_token->refresh_token);
+    refresh_token->refresh_token = NULL;
+  }
+}
+
+//
+// Oauth2 Token Fetcher credentials.
+//
+
+static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+  gpr_mu_destroy(&c->mu);
+  grpc_httpcli_context_destroy(&c->httpcli_context);
+}
+
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const grpc_http_response *response, grpc_credentials_md_store **token_md,
+    gpr_timespec *token_lifetime) {
+  char *null_terminated_body = NULL;
+  char *new_access_token = NULL;
+  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
+  grpc_json *json = NULL;
+
+  if (response == NULL) {
+    gpr_log(GPR_ERROR, "Received NULL response.");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  }
+
+  if (response->body_length > 0) {
+    null_terminated_body = gpr_malloc(response->body_length + 1);
+    null_terminated_body[response->body_length] = '\0';
+    memcpy(null_terminated_body, response->body, response->body_length);
+  }
+
+  if (response->status != 200) {
+    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
+            response->status,
+            null_terminated_body != NULL ? null_terminated_body : "");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  } else {
+    grpc_json *access_token = NULL;
+    grpc_json *token_type = NULL;
+    grpc_json *expires_in = NULL;
+    grpc_json *ptr;
+    json = grpc_json_parse_string(null_terminated_body);
+    if (json == NULL) {
+      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (json->type != GRPC_JSON_OBJECT) {
+      gpr_log(GPR_ERROR, "Response should be a JSON object");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    for (ptr = json->child; ptr; ptr = ptr->next) {
+      if (strcmp(ptr->key, "access_token") == 0) {
+        access_token = ptr;
+      } else if (strcmp(ptr->key, "token_type") == 0) {
+        token_type = ptr;
+      } else if (strcmp(ptr->key, "expires_in") == 0) {
+        expires_in = ptr;
+      }
+    }
+    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
+      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
+                 access_token->value);
+    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
+    token_lifetime->tv_nsec = 0;
+    token_lifetime->clock_type = GPR_TIMESPAN;
+    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
+    *token_md = grpc_credentials_md_store_create(1);
+    grpc_credentials_md_store_add_cstrings(
+        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    status = GRPC_CREDENTIALS_OK;
+  }
+
+end:
+  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
+    grpc_credentials_md_store_unref(*token_md);
+    *token_md = NULL;
+  }
+  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
+  if (new_access_token != NULL) gpr_free(new_access_token);
+  if (json != NULL) grpc_json_destroy(json);
+  return status;
+}
+
+static void on_oauth2_token_fetcher_http_response(
+    grpc_exec_ctx *exec_ctx, void *user_data,
+    const grpc_http_response *response) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  gpr_timespec token_lifetime;
+  grpc_credentials_status status;
+
+  gpr_mu_lock(&c->mu);
+  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
+      response, &c->access_token_md, &token_lifetime);
+  if (status == GRPC_CREDENTIALS_OK) {
+    c->token_expiration =
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
+    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
+          c->access_token_md->num_entries, status);
+  } else {
+    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+    r->cb(exec_ctx, r->user_data, NULL, 0, status);
+  }
+  gpr_mu_unlock(&c->mu);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void oauth2_token_fetcher_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops *pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+  grpc_credentials_md_store *cached_access_token_md = NULL;
+  {
+    gpr_mu_lock(&c->mu);
+    if (c->access_token_md != NULL &&
+        (gpr_time_cmp(
+             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+             refresh_threshold) > 0)) {
+      cached_access_token_md =
+          grpc_credentials_md_store_ref(c->access_token_md);
+    }
+    gpr_mu_unlock(&c->mu);
+  }
+  if (cached_access_token_md != NULL) {
+    cb(exec_ctx, user_data, cached_access_token_md->entries,
+       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
+    grpc_credentials_md_store_unref(cached_access_token_md);
+  } else {
+    c->fetch_func(
+        exec_ctx,
+        grpc_credentials_metadata_request_create(creds, cb, user_data),
+        &c->httpcli_context, pops, on_oauth2_token_fetcher_http_response,
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+  }
+}
+
+static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
+                                      grpc_fetch_oauth2_func fetch_func) {
+  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  gpr_ref_init(&c->base.refcount, 1);
+  gpr_mu_init(&c->mu);
+  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+  c->fetch_func = fetch_func;
+  grpc_httpcli_context_init(&c->httpcli_context);
+}
+
+//
+//  Google Compute Engine credentials.
+//
+
+static grpc_call_credentials_vtable compute_engine_vtable = {
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void compute_engine_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_pops* pops,
+    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
+  grpc_http_header header = {"Metadata-Flavor", "Google"};
+  grpc_httpcli_request request;
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
+  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  grpc_httpcli_get(exec_ctx, httpcli_context, pops, &request, deadline,
+                   response_cb, metadata_req);
+}
+
+grpc_call_credentials *grpc_google_compute_engine_credentials_create(
+    void *reserved) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
+  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
+  c->base.vtable = &compute_engine_vtable;
+  return &c->base;
+}
+
+//
+// Google Refresh Token credentials.
+//
+
+static void refresh_token_destruct(grpc_call_credentials *creds) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)creds;
+  grpc_auth_refresh_token_destruct(&c->refresh_token);
+  oauth2_token_fetcher_destruct(&c->base.base);
+}
+
+static grpc_call_credentials_vtable refresh_token_vtable = {
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void refresh_token_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_pops* pops,
+    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)metadata_req->creds;
+  grpc_http_header header = {"Content-Type",
+                             "application/x-www-form-urlencoded"};
+  grpc_httpcli_request request;
+  char *body = NULL;
+  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
+               c->refresh_token.client_id, c->refresh_token.client_secret,
+               c->refresh_token.refresh_token);
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
+  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  request.handshaker = &grpc_httpcli_ssl;
+  grpc_httpcli_post(exec_ctx, httpcli_context, pops, &request, body,
+                    strlen(body), deadline, response_cb, metadata_req);
+  gpr_free(body);
+}
+
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token refresh_token) {
+  grpc_google_refresh_token_credentials *c;
+  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
+    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
+  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
+  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
+  c->base.base.vtable = &refresh_token_vtable;
+  c->refresh_token = refresh_token;
+  return &c->base.base;
+}
+
+grpc_call_credentials *grpc_google_refresh_token_credentials_create(
+    const char *json_refresh_token, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
+      "reserved=%p)",
+      2, (json_refresh_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
+      grpc_auth_refresh_token_create_from_string(json_refresh_token));
+}
+
+//
+// Oauth2 Access Token credentials.
+//
+
+static void access_token_destruct(grpc_call_credentials *creds) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+}
+
+static void access_token_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pops* pops, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
+}
+
+static grpc_call_credentials_vtable access_token_vtable = {
+    access_token_destruct, access_token_get_request_metadata};
+
+grpc_call_credentials *grpc_access_token_credentials_create(
+    const char *access_token, void *reserved) {
+  grpc_access_token_credentials *c =
+      gpr_malloc(sizeof(grpc_access_token_credentials));
+  char *token_md_value;
+  GRPC_API_TRACE(
+      "grpc_access_token_credentials_create(access_token=%s, "
+      "reserved=%p)",
+      2, (access_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_access_token_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &access_token_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->access_token_md = grpc_credentials_md_store_create(1);
+  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
+  grpc_credentials_md_store_add_cstrings(
+      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  gpr_free(token_md_value);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
new file mode 100644
index 0000000000..2ae37f5c24
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/security/credentials/credentials.h"
+
+// auth_refresh_token parsing.
+typedef struct {
+  const char *type;
+  char *client_id;
+  char *client_secret;
+  char *refresh_token;
+} grpc_auth_refresh_token;
+
+/// Returns 1 if the object is valid, 0 otherwise.
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token);
+
+/// Creates a refresh token object from string. Returns an invalid object if a
+/// parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string);
+
+/// Creates a refresh token object from parsed json. Returns an invalid object
+/// if a parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json);
+
+/// Destructs the object.
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
+
+// -- Oauth2 Token Fetcher credentials --
+//
+//  This object is a base for credentials that need to acquire an oauth2 token
+//  from an http service.
+
+typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
+                                       grpc_credentials_metadata_request *req,
+                                       grpc_httpcli_context *http_context,
+                                       grpc_pops *pops,
+                                       grpc_httpcli_response_cb response_cb,
+                                       gpr_timespec deadline);
+typedef struct {
+  grpc_call_credentials base;
+  gpr_mu mu;
+  grpc_credentials_md_store *access_token_md;
+  gpr_timespec token_expiration;
+  grpc_httpcli_context httpcli_context;
+  grpc_fetch_oauth2_func fetch_func;
+} grpc_oauth2_token_fetcher_credentials;
+
+// Google refresh token credentials.
+typedef struct {
+  grpc_oauth2_token_fetcher_credentials base;
+  grpc_auth_refresh_token refresh_token;
+} grpc_google_refresh_token_credentials;
+
+// Access token credentials.
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *access_token_md;
+} grpc_access_token_credentials;
+
+// Private constructor for refresh token credentials from an already parsed
+// refresh token. Takes ownership of the refresh token.
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token token);
+
+// Exposed for testing only.
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const struct grpc_http_response *response,
+    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
new file mode 100644
index 0000000000..348f4c929f
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -0,0 +1,129 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/plugin/plugin_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+typedef struct {
+  void *user_data;
+  grpc_credentials_metadata_cb cb;
+} grpc_metadata_plugin_request;
+
+static void plugin_destruct(grpc_call_credentials *creds) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
+    c->plugin.destroy(c->plugin.state);
+  }
+}
+
+static void plugin_md_request_metadata_ready(void *request,
+                                             const grpc_metadata *md,
+                                             size_t num_md,
+                                             grpc_status_code status,
+                                             const char *error_details) {
+  /* called from application code */
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
+  if (status != GRPC_STATUS_OK) {
+    if (error_details != NULL) {
+      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
+              error_details);
+    }
+    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  } else {
+    size_t i;
+    grpc_credentials_md *md_array = NULL;
+    if (num_md > 0) {
+      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
+      for (i = 0; i < num_md; i++) {
+        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
+        md_array[i].value =
+            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
+      }
+    }
+    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
+    if (md_array != NULL) {
+      for (i = 0; i < num_md; i++) {
+        gpr_slice_unref(md_array[i].key);
+        gpr_slice_unref(md_array[i].value);
+      }
+      gpr_free(md_array);
+    }
+  }
+  gpr_free(r);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_credentials *creds,
+                                        grpc_pops *pops,
+                                        grpc_auth_metadata_context context,
+                                        grpc_credentials_metadata_cb cb,
+                                        void *user_data) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.get_metadata != NULL) {
+    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
+    memset(request, 0, sizeof(*request));
+    request->user_data = user_data;
+    request->cb = cb;
+    c->plugin.get_metadata(c->plugin.state, context,
+                           plugin_md_request_metadata_ready, request);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_call_credentials_vtable plugin_vtable = {
+    plugin_destruct, plugin_get_request_metadata};
+
+grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
+    grpc_metadata_credentials_plugin plugin, void *reserved) {
+  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
+  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(*c));
+  c->base.type = plugin.type;
+  c->base.vtable = &plugin_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->plugin = plugin;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
new file mode 100644
index 0000000000..89073cb3d1
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_credentials_md_store *plugin_md;
+} grpc_plugin_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c
new file mode 100644
index 0000000000..545bca9d98
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c
@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/ssl/ssl_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+//
+// Utils
+//
+
+static void ssl_copy_key_material(const char *input, unsigned char **output,
+                                  size_t *output_size) {
+  *output_size = strlen(input);
+  *output = gpr_malloc(*output_size);
+  memcpy(*output, input, *output_size);
+}
+
+//
+// SSL Channel Credentials.
+//
+
+static void ssl_destruct(grpc_channel_credentials *creds) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
+  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
+}
+
+static grpc_security_status ssl_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_OK;
+  size_t i = 0;
+  const char *overridden_target_name = NULL;
+  grpc_arg new_arg;
+
+  for (i = 0; args && i < args->num_args; i++) {
+    grpc_arg *arg = &args->args[i];
+    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
+        arg->type == GRPC_ARG_STRING) {
+      overridden_target_name = arg->value.string;
+      break;
+    }
+  }
+  status = grpc_ssl_channel_security_connector_create(
+      call_creds, &c->config, target, overridden_target_name, sc);
+  if (status != GRPC_SECURITY_OK) {
+    return status;
+  }
+  new_arg.type = GRPC_ARG_STRING;
+  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
+  new_arg.value.string = "https";
+  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
+  return status;
+}
+
+static grpc_channel_credentials_vtable ssl_vtable = {
+    ssl_destruct, ssl_create_security_connector};
+
+static void ssl_build_config(const char *pem_root_certs,
+                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+                             grpc_ssl_config *config) {
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (pem_key_cert_pair != NULL) {
+    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pair->private_key,
+                          &config->pem_private_key,
+                          &config->pem_private_key_size);
+    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
+                          &config->pem_cert_chain,
+                          &config->pem_cert_chain_size);
+  }
+}
+
+grpc_channel_credentials *grpc_ssl_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+    void *reserved) {
+  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_credentials_create(pem_root_certs=%s, "
+      "pem_key_cert_pair=%p, "
+      "reserved=%p)",
+      3, (pem_root_certs, pem_key_cert_pair, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  c->base.vtable = &ssl_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
+  return &c->base;
+}
+
+//
+// SSL Server Credentials.
+//
+
+static void ssl_server_destruct(grpc_server_credentials *creds) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
+    if (c->config.pem_private_keys[i] != NULL) {
+      gpr_free(c->config.pem_private_keys[i]);
+    }
+    if (c->config.pem_cert_chains[i] != NULL) {
+      gpr_free(c->config.pem_cert_chains[i]);
+    }
+  }
+  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
+  if (c->config.pem_private_keys_sizes != NULL) {
+    gpr_free(c->config.pem_private_keys_sizes);
+  }
+  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
+  if (c->config.pem_cert_chains_sizes != NULL) {
+    gpr_free(c->config.pem_cert_chains_sizes);
+  }
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+}
+
+static grpc_security_status ssl_server_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  return grpc_ssl_server_security_connector_create(&c->config, sc);
+}
+
+static grpc_server_credentials_vtable ssl_server_vtable = {
+    ssl_server_destruct, ssl_server_create_security_connector};
+
+static void ssl_build_server_config(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    grpc_ssl_server_config *config) {
+  size_t i;
+  config->client_certificate_request = client_certificate_request;
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (num_key_cert_pairs > 0) {
+    GPR_ASSERT(pem_key_cert_pairs != NULL);
+    config->pem_private_keys =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_cert_chains =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_private_keys_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+    config->pem_cert_chains_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+  }
+  config->num_key_cert_pairs = num_key_cert_pairs;
+  for (i = 0; i < num_key_cert_pairs; i++) {
+    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
+                          &config->pem_private_keys[i],
+                          &config->pem_private_keys_sizes[i]);
+    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
+                          &config->pem_cert_chains[i],
+                          &config->pem_cert_chains_sizes[i]);
+  }
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
+  return grpc_ssl_server_credentials_create_ex(
+      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
+      force_client_auth
+          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
+          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+      reserved);
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    void *reserved) {
+  grpc_ssl_server_credentials *c =
+      gpr_malloc(sizeof(grpc_ssl_server_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_server_credentials_create_ex("
+      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
+      "client_certificate_request=%d, reserved=%p)",
+      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
+          client_certificate_request, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_server_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &ssl_server_vtable;
+  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
+                          num_key_cert_pairs, client_certificate_request,
+                          &c->config);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.h b/src/core/lib/security/credentials/ssl/ssl_credentials.h
new file mode 100644
index 0000000000..f23dbdbe49
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.h
@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_ssl_config config;
+} grpc_ssl_credentials;
+
+typedef struct {
+  grpc_server_credentials base;
+  grpc_ssl_server_config config;
+} grpc_ssl_server_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H */
diff --git a/src/core/lib/security/auth_filters.h b/src/core/lib/security/transport/auth_filters.h
index 7fb56c3f3a..f688d4ed21 100644
--- a/src/core/lib/security/auth_filters.h
+++ b/src/core/lib/security/transport/auth_filters.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
-#define GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
 
 #include "src/core/lib/channel/channel_stack.h"
 
 extern const grpc_channel_filter grpc_client_auth_filter;
 extern const grpc_channel_filter grpc_server_auth_filter;
 
-#endif /* GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H */
diff --git a/src/core/lib/security/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index ea50c8513e..ac504ec95b 100644
--- a/src/core/lib/security/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <string.h>
 
@@ -40,9 +40,9 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
diff --git a/src/core/lib/security/handshake.c b/src/core/lib/security/transport/handshake.c
index d5fe0c7b7d..6561f4b47d 100644
--- a/src/core/lib/security/handshake.c
+++ b/src/core/lib/security/transport/handshake.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -39,8 +39,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
diff --git a/src/core/lib/security/handshake.h b/src/core/lib/security/transport/handshake.h
index f34476ed49..6ed850b315 100644
--- a/src/core/lib/security/handshake.h
+++ b/src/core/lib/security/transport/handshake.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
-#define GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
 
 #include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 /* Calls the callback upon completion. Takes owership of handshaker. */
 void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
@@ -48,4 +48,4 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
 
-#endif /* GRPC_CORE_LIB_SECURITY_HANDSHAKE_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H */
diff --git a/src/core/lib/security/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
index 27b0e98910..4438c8e559 100644
--- a/src/core/lib/security/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
diff --git a/src/core/lib/security/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h
index ff1c6639de..d00075b769 100644
--- a/src/core/lib/security/secure_endpoint.h
+++ b/src/core/lib/security/transport/secure_endpoint.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
-#define GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
 
 #include <grpc/support/slice.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -46,4 +46,4 @@ grpc_endpoint *grpc_secure_endpoint_create(
     struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
     gpr_slice *leftover_slices, size_t leftover_nslices);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */
diff --git a/src/core/lib/security/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 2d2023bdf5..72173e7c9d 100644
--- a/src/core/lib/security/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -43,10 +43,10 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/alpn/alpn.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/handshake.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/handshake.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/load_file.h"
 #include "src/core/lib/support/string.h"
diff --git a/src/core/lib/security/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 2c893cd5e9..84e586deaa 100644
--- a/src/core/lib/security/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -263,4 +263,4 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     const grpc_auth_context *auth_context);
 void tsi_shallow_peer_destruct(tsi_peer *peer);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H */
diff --git a/src/core/lib/security/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
index 845320975d..249f6af975 100644
--- a/src/core/lib/security/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -33,9 +33,9 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/b64.c b/src/core/lib/security/util/b64.c
index 87f0e05280..9da42e4e73 100644
--- a/src/core/lib/security/b64.c
+++ b/src/core/lib/security/util/b64.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 
 #include <stdint.h>
 #include <string.h>
diff --git a/src/core/lib/security/b64.h b/src/core/lib/security/util/b64.h
index c515e7af2c..6908095287 100644
--- a/src/core/lib/security/b64.h
+++ b/src/core/lib/security/util/b64.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_B64_H
-#define GRPC_CORE_LIB_SECURITY_B64_H
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_B64_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_B64_H
 
 #include <grpc/support/slice.h>
 
@@ -49,4 +49,4 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe);
 gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
                                       int url_safe);
 
-#endif /* GRPC_CORE_LIB_SECURITY_B64_H */
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_B64_H */
diff --git a/src/core/lib/security/util/json_util.c b/src/core/lib/security/util/json_util.c
new file mode 100644
index 0000000000..7eed039baa
--- /dev/null
+++ b/src/core/lib/security/util/json_util.c
@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/util/json_util.h"
+
+#include <string.h>
+
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name) {
+  grpc_json *child;
+  for (child = json->child; child != NULL; child = child->next) {
+    if (strcmp(child->key, prop_name) == 0) break;
+  }
+  if (child == NULL || child->type != GRPC_JSON_STRING) {
+    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
+    return NULL;
+  }
+  return child->value;
+}
+
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name,
+                                    char **copied_value) {
+  const char *prop_value = grpc_json_get_string_property(json, prop_name);
+  if (prop_value == NULL) return false;
+  *copied_value = gpr_strdup(prop_value);
+  return true;
+}
diff --git a/src/core/lib/security/util/json_util.h b/src/core/lib/security/util/json_util.h
new file mode 100644
index 0000000000..137900593f
--- /dev/null
+++ b/src/core/lib/security/util/json_util.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+
+#include <stdbool.h>
+
+#include "src/core/lib/json/json.h"
+
+// Constants.
+#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
+#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
+#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
+
+// Gets a child property from a json node.
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name);
+
+// Copies the value of the json child property specified by prop_name.
+// Returns false if the property was not found.
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name, char **copied_value);
+
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H */
diff --git a/src/core/lib/support/murmur_hash.c b/src/core/lib/support/murmur_hash.c
index 5711fff0c0..7137c1f313 100644
--- a/src/core/lib/support/murmur_hash.c
+++ b/src/core/lib/support/murmur_hash.c
@@ -33,6 +33,8 @@
 
 #include "src/core/lib/support/murmur_hash.h"
 
+#include <string.h>
+
 #define ROTL32(x, r) ((x) << (r)) | ((x) >> (32 - (r)))
 
 #define FMIX32(h)    \
@@ -42,10 +44,6 @@
   (h) *= 0xc2b2ae35; \
   (h) ^= (h) >> 16;
 
-/* Block read - if your platform needs to do endian-swapping or can only
-   handle aligned reads, do the conversion here */
-#define GETBLOCK32(p, i) (p)[(i)]
-
 uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
   const uint8_t *data = (const uint8_t *)key;
   const size_t nblocks = len / 4;
@@ -62,7 +60,7 @@ uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
 
   /* body */
   for (i = -(int)nblocks; i; i++) {
-    k1 = GETBLOCK32(blocks, i);
+    memcpy(&k1, blocks + i, sizeof(uint32_t));
 
     k1 *= c1;
     k1 = ROTL32(k1, 15);
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index 1f82c3bad2..5eb7cf1bf4 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -70,6 +70,8 @@ struct grpc_completion_queue {
   int shutdown;
   int shutdown_called;
   int is_server_cq;
+  /** Can the server cq accept incoming channels */
+  int is_non_listening_server_cq;
   int num_pluckers;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
   grpc_closure pollset_shutdown_done;
@@ -84,6 +86,7 @@ struct grpc_completion_queue {
 };
 
 #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
+#define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
 
 static gpr_mu g_freelist_mu;
 static grpc_completion_queue *g_freelist;
@@ -149,6 +152,7 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
   cc->shutdown = 0;
   cc->shutdown_called = 0;
   cc->is_server_cq = 0;
+  cc->is_non_listening_server_cq = 0;
   cc->num_pluckers = 0;
 #ifndef NDEBUG
   cc->outstanding_tag_count = 0;
@@ -511,6 +515,18 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
   return POLLSET_FROM_CQ(cc);
 }
 
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) {
+  return CQ_FROM_POLLSET(ps);
+}
+
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc) {
+  cc->is_non_listening_server_cq = 1;
+}
+
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) {
+  return (cc->is_non_listening_server_cq == 1);
+}
+
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; }
 
 int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; }
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index eef82cf014..3d0dd13c53 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,7 +81,10 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
                     void *done_arg, grpc_cq_completion *storage);
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps);
 
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc);
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc);
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
 int grpc_cq_is_server_cq(grpc_completion_queue *cc);
 
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 3fda2c9e1e..7ee7b51568 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -37,10 +37,10 @@
 #include <string.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
 
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index aafbc988ce..14d82b515a 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,11 +69,6 @@ typedef struct call_data call_data;
 typedef struct channel_data channel_data;
 typedef struct registered_method registered_method;
 
-typedef struct {
-  call_data *next;
-  call_data *prev;
-} call_link;
-
 typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
 
 typedef struct requested_call {
@@ -81,7 +76,6 @@ typedef struct requested_call {
   void *tag;
   grpc_server *server;
   grpc_completion_queue *cq_bound_to_call;
-  grpc_completion_queue *cq_for_notification;
   grpc_call **call;
   grpc_cq_completion completion;
   grpc_metadata_array *initial_metadata;
@@ -108,6 +102,7 @@ struct channel_data {
   grpc_server *server;
   grpc_connectivity_state connectivity_state;
   grpc_channel *channel;
+  size_t cq_idx;
   /* linked list of all channels on a server */
   channel_data *next;
   channel_data *prev;
@@ -172,7 +167,7 @@ struct request_matcher {
   grpc_server *server;
   call_data *pending_head;
   call_data *pending_tail;
-  gpr_stack_lockfree *requests;
+  gpr_stack_lockfree **requests_per_cq;
 };
 
 struct registered_method {
@@ -180,6 +175,7 @@ struct registered_method {
   char *host;
   grpc_server_register_method_payload_handling payload_handling;
   uint32_t flags;
+  /* one request matcher per method */
   request_matcher request_matcher;
   registered_method *next;
 };
@@ -195,6 +191,7 @@ struct grpc_server {
   grpc_completion_queue **cqs;
   grpc_pollset **pollsets;
   size_t cq_count;
+  bool started;
 
   /* The two following mutexes control access to server-state
      mu_global controls access to non-call-related state (e.g., channel state)
@@ -207,6 +204,7 @@ struct grpc_server {
   gpr_mu mu_call;   /* mutex for call-specific state */
 
   registered_method *registered_methods;
+  /** one request matcher for unregistered methods */
   request_matcher unregistered_request_matcher;
   /** free list of available requested_calls indices */
   gpr_stack_lockfree *request_freelist;
@@ -234,7 +232,7 @@ struct grpc_server {
 
 static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld, bool success);
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc);
+                      size_t cq_idx, requested_call *rc);
 /* Before calling maybe_finish_shutdown, we must hold mu_global and not
    hold mu_call */
 static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_server *server);
@@ -312,12 +310,19 @@ static void request_matcher_init(request_matcher *rm, size_t entries,
                                  grpc_server *server) {
   memset(rm, 0, sizeof(*rm));
   rm->server = server;
-  rm->requests = gpr_stack_lockfree_create(entries);
+  rm->requests_per_cq =
+      gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    rm->requests_per_cq[i] = gpr_stack_lockfree_create(entries);
+  }
 }
 
 static void request_matcher_destroy(request_matcher *rm) {
-  GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests) == -1);
-  gpr_stack_lockfree_destroy(rm->requests);
+  for (size_t i = 0; i < rm->server->cq_count; i++) {
+    GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1);
+    gpr_stack_lockfree_destroy(rm->requests_per_cq[i]);
+  }
+  gpr_free(rm->requests_per_cq);
 }
 
 static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, bool success) {
@@ -343,8 +348,11 @@ static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx,
                                           grpc_server *server,
                                           request_matcher *rm) {
   int request_id;
-  while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
-    fail_call(exec_ctx, server, &server->requested_calls[request_id]);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    while ((request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[i])) !=
+           -1) {
+      fail_call(exec_ctx, server, i, &server->requested_calls[request_id]);
+    }
   }
 }
 
@@ -364,15 +372,19 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
   gpr_mu_destroy(&server->mu_call);
   while ((rm = server->registered_methods) != NULL) {
     server->registered_methods = rm->next;
-    request_matcher_destroy(&rm->request_matcher);
+    if (server->started) {
+      request_matcher_destroy(&rm->request_matcher);
+    }
     gpr_free(rm->method);
     gpr_free(rm->host);
     gpr_free(rm);
   }
+  if (server->started) {
+    request_matcher_destroy(&server->unregistered_request_matcher);
+  }
   for (i = 0; i < server->cq_count; i++) {
     GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server");
   }
-  request_matcher_destroy(&server->unregistered_request_matcher);
   gpr_stack_lockfree_destroy(server->request_freelist);
   gpr_free(server->cqs);
   gpr_free(server->pollsets);
@@ -453,11 +465,11 @@ static void done_request_event(grpc_exec_ctx *exec_ctx, void *req,
 }
 
 static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                         call_data *calld, requested_call *rc) {
+                         call_data *calld, size_t cq_idx, requested_call *rc) {
   grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
   grpc_call *call = calld->call;
   *rc->call = call;
-  calld->cq_new = rc->cq_for_notification;
+  calld->cq_new = server->cqs[cq_idx];
   GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
   switch (rc->type) {
     case BATCH_CALL:
@@ -492,7 +504,9 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
 }
 
 static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
-  call_data *calld = arg;
+  grpc_call_element *call_elem = arg;
+  call_data *calld = call_elem->call_data;
+  channel_data *chand = call_elem->channel_data;
   request_matcher *rm = calld->request_matcher;
   grpc_server *server = rm->server;
 
@@ -507,26 +521,34 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     return;
   }
 
-  int request_id = gpr_stack_lockfree_pop(rm->requests);
-  if (request_id == -1) {
-    gpr_mu_lock(&server->mu_call);
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = PENDING;
-    gpr_mu_unlock(&calld->mu_state);
-    if (rm->pending_head == NULL) {
-      rm->pending_tail = rm->pending_head = calld;
+  for (size_t i = 0; i < server->cq_count; i++) {
+    size_t cq_idx = (chand->cq_idx + i) % server->cq_count;
+    int request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+    if (request_id == -1) {
+      continue;
     } else {
-      rm->pending_tail->pending_next = calld;
-      rm->pending_tail = calld;
+      gpr_mu_lock(&calld->mu_state);
+      calld->state = ACTIVATED;
+      gpr_mu_unlock(&calld->mu_state);
+      publish_call(exec_ctx, server, calld, cq_idx,
+                   &server->requested_calls[request_id]);
+      return; /* early out */
     }
-    calld->pending_next = NULL;
-    gpr_mu_unlock(&server->mu_call);
+  }
+
+  /* no cq to take the request found: queue it on the slow list */
+  gpr_mu_lock(&server->mu_call);
+  gpr_mu_lock(&calld->mu_state);
+  calld->state = PENDING;
+  gpr_mu_unlock(&calld->mu_state);
+  if (rm->pending_head == NULL) {
+    rm->pending_tail = rm->pending_head = calld;
   } else {
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = ACTIVATED;
-    gpr_mu_unlock(&calld->mu_state);
-    publish_call(exec_ctx, server, calld, &server->requested_calls[request_id]);
+    rm->pending_tail->pending_next = calld;
+    rm->pending_tail = calld;
   }
+  calld->pending_next = NULL;
+  gpr_mu_unlock(&server->mu_call);
 }
 
 static void finish_start_new_rpc(
@@ -548,14 +570,14 @@ static void finish_start_new_rpc(
 
   switch (payload_handling) {
     case GRPC_SRM_PAYLOAD_NONE:
-      publish_new_rpc(exec_ctx, calld, true);
+      publish_new_rpc(exec_ctx, elem, true);
       break;
     case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: {
       grpc_op op;
       memset(&op, 0, sizeof(op));
       op.op = GRPC_OP_RECV_MESSAGE;
       op.data.recv_message = &calld->payload;
-      grpc_closure_init(&calld->publish, publish_new_rpc, calld);
+      grpc_closure_init(&calld->publish, publish_new_rpc, elem);
       grpc_call_start_batch_and_execute(exec_ctx, calld->call, &op, 1,
                                         &calld->publish);
       break;
@@ -637,14 +659,16 @@ static int num_channels(grpc_server *server) {
 
 static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx,
                                      grpc_server *server) {
-  registered_method *rm;
-  request_matcher_kill_requests(exec_ctx, server,
-                                &server->unregistered_request_matcher);
-  request_matcher_zombify_all_pending_calls(
-      exec_ctx, &server->unregistered_request_matcher);
-  for (rm = server->registered_methods; rm; rm = rm->next) {
-    request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
-    request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+  if (server->started) {
+    request_matcher_kill_requests(exec_ctx, server,
+                                  &server->unregistered_request_matcher);
+    request_matcher_zombify_all_pending_calls(
+        exec_ctx, &server->unregistered_request_matcher);
+    for (registered_method *rm = server->registered_methods; rm;
+         rm = rm->next) {
+      request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
+      request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+    }
   }
 }
 
@@ -895,25 +919,46 @@ const grpc_channel_filter grpc_server_top_filter = {
     "server",
 };
 
-void grpc_server_register_completion_queue(grpc_server *server,
-                                           grpc_completion_queue *cq,
-                                           void *reserved) {
+static void register_completion_queue(grpc_server *server,
+                                      grpc_completion_queue *cq,
+                                      bool is_non_listening, void *reserved) {
   size_t i, n;
-  GRPC_API_TRACE(
-      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
-      (server, cq, reserved));
   GPR_ASSERT(!reserved);
   for (i = 0; i < server->cq_count; i++) {
     if (server->cqs[i] == cq) return;
   }
-  GRPC_CQ_INTERNAL_REF(cq, "server");
+
   grpc_cq_mark_server_cq(cq);
+
+  if (is_non_listening) {
+    grpc_cq_mark_non_listening_server_cq(cq);
+  }
+
+  GRPC_CQ_INTERNAL_REF(cq, "server");
   n = server->cq_count++;
   server->cqs = gpr_realloc(server->cqs,
                             server->cq_count * sizeof(grpc_completion_queue *));
   server->cqs[n] = cq;
 }
 
+void grpc_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq,
+                                           void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
+      (server, cq, reserved));
+  register_completion_queue(server, cq, false, reserved);
+}
+
+void grpc_server_register_non_listening_completion_queue(
+    grpc_server *server, grpc_completion_queue *cq, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, "
+      "reserved=%p)",
+      3, (server, cq, reserved));
+  register_completion_queue(server, cq, true, reserved);
+}
+
 grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   size_t i;
 
@@ -940,8 +985,6 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   for (i = 0; i < (size_t)server->max_requested_calls; i++) {
     gpr_stack_lockfree_push(server->request_freelist, (int)i);
   }
-  request_matcher_init(&server->unregistered_request_matcher,
-                       server->max_requested_calls, server);
   server->requested_calls = gpr_malloc(server->max_requested_calls *
                                        sizeof(*server->requested_calls));
 
@@ -985,8 +1028,6 @@ void *grpc_server_register_method(
   }
   m = gpr_malloc(sizeof(registered_method));
   memset(m, 0, sizeof(*m));
-  request_matcher_init(&m->request_matcher, server->max_requested_calls,
-                       server);
   m->method = gpr_strdup(method);
   m->host = gpr_strdup(host);
   m->next = server->registered_methods;
@@ -1003,13 +1044,23 @@ void grpc_server_start(grpc_server *server) {
 
   GRPC_API_TRACE("grpc_server_start(server=%p)", 1, (server));
 
+  server->started = true;
+  size_t pollset_count = 0;
   server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
   for (i = 0; i < server->cq_count; i++) {
-    server->pollsets[i] = grpc_cq_pollset(server->cqs[i]);
+    if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) {
+      server->pollsets[pollset_count++] = grpc_cq_pollset(server->cqs[i]);
+    }
+  }
+  request_matcher_init(&server->unregistered_request_matcher,
+                       server->max_requested_calls, server);
+  for (registered_method *rm = server->registered_methods; rm; rm = rm->next) {
+    request_matcher_init(&rm->request_matcher, server->max_requested_calls,
+                         server);
   }
 
   for (l = server->listeners; l; l = l->next) {
-    l->start(&exec_ctx, server, l->arg, server->pollsets, server->cq_count);
+    l->start(&exec_ctx, server, l->arg, server->pollsets, pollset_count);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
@@ -1017,8 +1068,8 @@ void grpc_server_start(grpc_server *server) {
 
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args) {
-  size_t i;
   size_t num_registered_methods;
   size_t alloc;
   registered_method *rm;
@@ -1033,12 +1084,6 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   uint32_t max_probes = 0;
   grpc_transport_op op;
 
-  for (i = 0; i < s->cq_count; i++) {
-    memset(&op, 0, sizeof(op));
-    op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
-    grpc_transport_perform_op(exec_ctx, transport, &op);
-  }
-
   channel =
       grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
   chand = (channel_data *)grpc_channel_stack_element(
@@ -1048,6 +1093,17 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   server_ref(s);
   chand->channel = channel;
 
+  size_t cq_idx;
+  grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
+  for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
+    if (s->cqs[cq_idx] == accepting_cq) break;
+  }
+  if (cq_idx == s->cq_count) {
+    /* completion queue not found: pick a random one to publish new calls to */
+    cq_idx = (size_t)rand() % s->cq_count;
+  }
+  chand->cq_idx = cq_idx;
+
   num_registered_methods = 0;
   for (rm = s->registered_methods; rm; rm = rm->next) {
     num_registered_methods++;
@@ -1218,19 +1274,19 @@ void grpc_server_add_listener(
 }
 
 static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
-                                          grpc_server *server,
+                                          grpc_server *server, size_t cq_idx,
                                           requested_call *rc) {
   call_data *calld = NULL;
   request_matcher *rm = NULL;
   int request_id;
   if (gpr_atm_acq_load(&server->shutdown_flag)) {
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc);
     return GRPC_CALL_OK;
   }
   request_id = gpr_stack_lockfree_pop(server->request_freelist);
   if (request_id == -1) {
     /* out of request ids: just fail this one */
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc);
     return GRPC_CALL_OK;
   }
   switch (rc->type) {
@@ -1243,12 +1299,12 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
   }
   server->requested_calls[request_id] = *rc;
   gpr_free(rc);
-  if (gpr_stack_lockfree_push(rm->requests, request_id)) {
+  if (gpr_stack_lockfree_push(rm->requests_per_cq[cq_idx], request_id)) {
     /* this was the first queued request: we need to lock and start
        matching calls */
     gpr_mu_lock(&server->mu_call);
     while ((calld = rm->pending_head) != NULL) {
-      request_id = gpr_stack_lockfree_pop(rm->requests);
+      request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
       if (request_id == -1) break;
       rm->pending_head = calld->pending_next;
       gpr_mu_unlock(&server->mu_call);
@@ -1264,7 +1320,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
         GPR_ASSERT(calld->state == PENDING);
         calld->state = ACTIVATED;
         gpr_mu_unlock(&calld->mu_state);
-        publish_call(exec_ctx, server, calld,
+        publish_call(exec_ctx, server, calld, cq_idx,
                      &server->requested_calls[request_id]);
       }
       gpr_mu_lock(&server->mu_call);
@@ -1288,7 +1344,13 @@ grpc_call_error grpc_server_request_call(
       "cq_bound_to_call=%p, cq_for_notification=%p, tag=%p)",
       7, (server, call, details, initial_metadata, cq_bound_to_call,
           cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1299,11 +1361,10 @@ grpc_call_error grpc_server_request_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.batch.details = details;
   rc->initial_metadata = initial_metadata;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
@@ -1325,7 +1386,14 @@ grpc_call_error grpc_server_request_registered_call(
       "tag=%p)",
       9, (server, rmp, call, deadline, initial_metadata, optional_payload,
           cq_bound_to_call, cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1341,26 +1409,25 @@ grpc_call_error grpc_server_request_registered_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.registered.registered_method = rm;
   rc->data.registered.deadline = deadline;
   rc->initial_metadata = initial_metadata;
   rc->data.registered.optional_payload = optional_payload;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
 }
 
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc) {
+                      size_t cq_idx, requested_call *rc) {
   *rc->call = NULL;
   rc->initial_metadata->count = 0;
 
   server_ref(server);
-  grpc_cq_end_op(exec_ctx, rc->cq_for_notification, rc->tag, 0,
-                 done_request_event, rc, &rc->completion);
+  grpc_cq_end_op(exec_ctx, server->cqs[cq_idx], rc->tag, 0, done_request_event,
+                 rc, &rc->completion);
 }
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server) {
diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h
index 470ef23c69..fb6e4d60c5 100644
--- a/src/core/lib/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -53,6 +53,7 @@ void grpc_server_add_listener(
    server */
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *server,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args);
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 5847ec9053..82c8e239f6 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -373,7 +373,8 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) {
     ss = g_static_strtab[idx];
     if (ss == NULL) break;
     if (ss->hash == hash && GPR_SLICE_LENGTH(ss->slice) == length &&
-        0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length)) {
+        (length == 0 ||
+         0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length))) {
       GPR_TIMER_END("grpc_mdstr_from_buffer", 0);
       return ss;
     }