Merge github.com:grpc/grpc into wc

22 files changed, 831 insertions, 238 deletions

diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c
index 518024fcfd..0e496829f6 100644
--- a/src/core/lib/iomgr/combiner.c
+++ b/src/core/lib/iomgr/combiner.c
@@ -73,10 +73,8 @@ static const grpc_closure_scheduler_vtable finally_scheduler = {
 static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 
 grpc_combiner *grpc_combiner_create(void) {
-  grpc_combiner *lock = gpr_malloc(sizeof(*lock));
+  grpc_combiner *lock = gpr_zalloc(sizeof(*lock));
   gpr_ref_init(&lock->refs, 1);
-  lock->next_combiner_on_this_exec_ctx = NULL;
-  lock->time_to_execute_final_list = false;
   lock->scheduler.vtable = &scheduler;
   lock->finally_scheduler.vtable = &finally_scheduler;
   gpr_atm_no_barrier_store(&lock->state, STATE_UNORPHANED);
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c
index b89b8af15a..b940d48ba9 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.c
+++ b/src/core/lib/iomgr/ev_epoll1_linux.c
@@ -48,7 +48,60 @@
 #include "src/core/lib/support/string.h"
 
 static grpc_wakeup_fd global_wakeup_fd;
-static int g_epfd;
+
+/*******************************************************************************
+ * Singleton epoll set related fields
+ */
+
+#define MAX_EPOLL_EVENTS 100
+#define MAX_EPOLL_EVENTS_HANDLED_PER_ITERATION 1
+
+/* NOTE ON SYNCHRONIZATION:
+ * - Fields in this struct are only modified by the designated poller. Hence
+ *   there is no need for any locks to protect the struct.
+ * - num_events and cursor fields have to be of atomic type to provide memory
+ *   visibility guarantees only. i.e In case of multiple pollers, the designated
+ *   polling thread keeps changing; the thread that wrote these values may be
+ *   different from the thread reading the values
+ */
+typedef struct epoll_set {
+  int epfd;
+
+  /* The epoll_events after the last call to epoll_wait() */
+  struct epoll_event events[MAX_EPOLL_EVENTS];
+
+  /* The number of epoll_events after the last call to epoll_wait() */
+  gpr_atm num_events;
+
+  /* Index of the first event in epoll_events that has to be processed. This
+   * field is only valid if num_events > 0 */
+  gpr_atm cursor;
+} epoll_set;
+
+/* The global singleton epoll set */
+static epoll_set g_epoll_set;
+
+/* Must be called *only* once */
+static bool epoll_set_init() {
+  g_epoll_set.epfd = epoll_create1(EPOLL_CLOEXEC);
+  if (g_epoll_set.epfd < 0) {
+    gpr_log(GPR_ERROR, "epoll unavailable");
+    return false;
+  }
+
+  gpr_log(GPR_INFO, "grpc epoll fd: %d", g_epoll_set.epfd);
+  gpr_atm_no_barrier_store(&g_epoll_set.num_events, 0);
+  gpr_atm_no_barrier_store(&g_epoll_set.cursor, 0);
+  return true;
+}
+
+/* epoll_set_init() MUST be called before calling this. */
+static void epoll_set_shutdown() {
+  if (g_epoll_set.epfd >= 0) {
+    close(g_epoll_set.epfd);
+    g_epoll_set.epfd = -1;
+  }
+}
 
 /*******************************************************************************
  * Fd Declarations
@@ -122,7 +175,7 @@ struct grpc_pollset {
   bool kicked_without_poller;
 
   /* Set to true if the pollset is observed to have no workers available to
-   * poll */
+     poll */
   bool seen_inactive;
   bool shutting_down;             /* Is the pollset shutting down ? */
   grpc_closure *shutdown_closure; /* Called after after shutdown is complete */
@@ -228,7 +281,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
 
   struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET),
                            .data.ptr = new_fd};
-  if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) {
+  if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, fd, &ev) != 0) {
     gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno));
   }
 
@@ -237,30 +290,43 @@ static grpc_fd *fd_create(int fd, const char *name) {
 
 static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; }
 
-/* Might be called multiple times */
-static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
+/* if 'releasing_fd' is true, it means that we are going to detach the internal
+ * fd from grpc_fd structure (i.e which means we should not be calling
+ * shutdown() syscall on that fd) */
+static void fd_shutdown_internal(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                                 grpc_error *why, bool releasing_fd) {
   if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure,
                              GRPC_ERROR_REF(why))) {
-    shutdown(fd->fd, SHUT_RDWR);
+    if (!releasing_fd) {
+      shutdown(fd->fd, SHUT_RDWR);
+    }
     grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why));
   }
   GRPC_ERROR_UNREF(why);
 }
 
+/* Might be called multiple times */
+static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
+  fd_shutdown_internal(exec_ctx, fd, why, false);
+}
+
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
+  bool is_release_fd = (release_fd != NULL);
 
   if (!grpc_lfev_is_shutdown(&fd->read_closure)) {
-    fd_shutdown(exec_ctx, fd, GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason));
+    fd_shutdown_internal(exec_ctx, fd,
+                         GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason),
+                         is_release_fd);
   }
 
   /* If release_fd is not NULL, we should be relinquishing control of the file
      descriptor fd->fd (but we still own the grpc_fd structure). */
-  if (release_fd != NULL) {
+  if (is_release_fd) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!already_closed) {
     close(fd->fd);
   }
 
@@ -313,7 +379,10 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
 
 GPR_TLS_DECL(g_current_thread_pollset);
 GPR_TLS_DECL(g_current_thread_worker);
+
+/* The designated poller */
 static gpr_atm g_active_poller;
+
 static pollset_neighbourhood *g_neighbourhoods;
 static size_t g_num_neighbourhoods;
 
@@ -367,7 +436,8 @@ static grpc_error *pollset_global_init(void) {
   if (err != GRPC_ERROR_NONE) return err;
   struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET),
                            .data.ptr = &global_wakeup_fd};
-  if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) {
+  if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd,
+                &ev) != 0) {
     return GRPC_OS_ERROR(errno, "epoll_ctl");
   }
   g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS);
@@ -393,7 +463,14 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->mu);
   *mu = &pollset->mu;
   pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()];
+  pollset->reassigning_neighbourhood = false;
+  pollset->root_worker = NULL;
+  pollset->kicked_without_poller = false;
   pollset->seen_inactive = true;
+  pollset->shutting_down = false;
+  pollset->shutdown_closure = NULL;
+  pollset->begin_refs = 0;
+  pollset->next = pollset->prev = NULL;
 }
 
 static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
@@ -425,6 +502,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
 }
 
 static grpc_error *pollset_kick_all(grpc_pollset *pollset) {
+  GPR_TIMER_BEGIN("pollset_kick_all", 0);
   grpc_error *error = GRPC_ERROR_NONE;
   if (pollset->root_worker != NULL) {
     grpc_pollset_worker *worker = pollset->root_worker;
@@ -450,7 +528,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) {
   }
   // TODO: sreek.  Check if we need to set 'kicked_without_poller' to true here
   // in the else case
-
+  GPR_TIMER_END("pollset_kick_all", 0);
   return error;
 }
 
@@ -458,6 +536,7 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
                                           grpc_pollset *pollset) {
   if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL &&
       pollset->begin_refs == 0) {
+    GPR_TIMER_MARK("pollset_finish_shutdown", 0);
     GRPC_CLOSURE_SCHED(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE);
     pollset->shutdown_closure = NULL;
   }
@@ -465,16 +544,16 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
 
 static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                              grpc_closure *closure) {
+  GPR_TIMER_BEGIN("pollset_shutdown", 0);
   GPR_ASSERT(pollset->shutdown_closure == NULL);
   GPR_ASSERT(!pollset->shutting_down);
   pollset->shutdown_closure = closure;
   pollset->shutting_down = true;
   GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset));
   pollset_maybe_finish_shutdown(exec_ctx, pollset);
+  GPR_TIMER_END("pollset_shutdown", 0);
 }
 
-#define MAX_EPOLL_EVENTS 100
-
 static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
                                            gpr_timespec now) {
   gpr_timespec timeout;
@@ -493,52 +572,93 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
   return millis >= 1 ? millis : 1;
 }
 
-static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                 gpr_timespec now, gpr_timespec deadline) {
-  struct epoll_event events[MAX_EPOLL_EVENTS];
-  static const char *err_desc = "pollset_poll";
-
-  int timeout = poll_deadline_to_millis_timeout(deadline, now);
+/* Process the epoll events found by do_epoll_wait() function.
+   - g_epoll_set.cursor points to the index of the first event to be processed
+   - This function then processes up-to MAX_EPOLL_EVENTS_PER_ITERATION and
+     updates the g_epoll_set.cursor
+
+   NOTE ON SYNCRHONIZATION: Similar to do_epoll_wait(), this function is only
+   called by g_active_poller thread. So there is no need for synchronization
+   when accessing fields in g_epoll_set */
+static grpc_error *process_epoll_events(grpc_exec_ctx *exec_ctx,
+                                        grpc_pollset *pollset) {
+  static const char *err_desc = "process_events";
+  grpc_error *error = GRPC_ERROR_NONE;
 
-  if (timeout != 0) {
-    GRPC_SCHEDULING_START_BLOCKING_REGION;
-  }
-  int r;
-  do {
-    r = epoll_wait(g_epfd, events, MAX_EPOLL_EVENTS, timeout);
-  } while (r < 0 && errno == EINTR);
-  if (timeout != 0) {
-    GRPC_SCHEDULING_END_BLOCKING_REGION;
-  }
+  GPR_TIMER_BEGIN("process_epoll_events", 0);
+  long num_events = gpr_atm_acq_load(&g_epoll_set.num_events);
+  long cursor = gpr_atm_acq_load(&g_epoll_set.cursor);
+  for (int idx = 0;
+       (idx < MAX_EPOLL_EVENTS_HANDLED_PER_ITERATION) && cursor != num_events;
+       idx++) {
+    long c = cursor++;
+    struct epoll_event *ev = &g_epoll_set.events[c];
+    void *data_ptr = ev->data.ptr;
 
-  if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait");
-
-  grpc_error *error = GRPC_ERROR_NONE;
-  for (int i = 0; i < r; i++) {
-    void *data_ptr = events[i].data.ptr;
     if (data_ptr == &global_wakeup_fd) {
       append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd),
                    err_desc);
     } else {
       grpc_fd *fd = (grpc_fd *)(data_ptr);
-      bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0;
-      bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0;
-      bool write_ev = (events[i].events & EPOLLOUT) != 0;
+      bool cancel = (ev->events & (EPOLLERR | EPOLLHUP)) != 0;
+      bool read_ev = (ev->events & (EPOLLIN | EPOLLPRI)) != 0;
+      bool write_ev = (ev->events & EPOLLOUT) != 0;
+
       if (read_ev || cancel) {
         fd_become_readable(exec_ctx, fd, pollset);
       }
+
       if (write_ev || cancel) {
         fd_become_writable(exec_ctx, fd);
       }
     }
   }
-
+  gpr_atm_rel_store(&g_epoll_set.cursor, cursor);
+  GPR_TIMER_END("process_epoll_events", 0);
   return error;
 }
 
+/* Do epoll_wait and store the events in g_epoll_set.events field. This does not
+   "process" any of the events yet; that is done in process_epoll_events().
+   *See process_epoll_events() function for more details.
+
+   NOTE ON SYNCHRONIZATION: At any point of time, only the g_active_poller
+   (i.e the designated poller thread) will be calling this function. So there is
+   no need for any synchronization when accesing fields in g_epoll_set */
+static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
+                                 gpr_timespec now, gpr_timespec deadline) {
+  GPR_TIMER_BEGIN("do_epoll_wait", 0);
+
+  int r;
+  int timeout = poll_deadline_to_millis_timeout(deadline, now);
+  if (timeout != 0) {
+    GRPC_SCHEDULING_START_BLOCKING_REGION;
+  }
+  do {
+    r = epoll_wait(g_epoll_set.epfd, g_epoll_set.events, MAX_EPOLL_EVENTS,
+                   timeout);
+  } while (r < 0 && errno == EINTR);
+  if (timeout != 0) {
+    GRPC_SCHEDULING_END_BLOCKING_REGION;
+  }
+
+  if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait");
+
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_DEBUG, "ps: %p poll got %d events", ps, r);
+  }
+
+  gpr_atm_rel_store(&g_epoll_set.num_events, r);
+  gpr_atm_rel_store(&g_epoll_set.cursor, 0);
+
+  GPR_TIMER_END("do_epoll_wait", 0);
+  return GRPC_ERROR_NONE;
+}
+
 static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
                          grpc_pollset_worker **worker_hdl, gpr_timespec *now,
                          gpr_timespec deadline) {
+  GPR_TIMER_BEGIN("begin_worker", 0);
   if (worker_hdl != NULL) *worker_hdl = worker;
   worker->initialized_cv = false;
   SET_KICK_STATE(worker, UNKICKED);
@@ -643,14 +763,17 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
 
   if (pollset->kicked_without_poller) {
     pollset->kicked_without_poller = false;
+    GPR_TIMER_END("begin_worker", 0);
     return false;
   }
 
+  GPR_TIMER_END("begin_worker", 0);
   return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down;
 }
 
 static bool check_neighbourhood_for_available_poller(
     pollset_neighbourhood *neighbourhood) {
+  GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0);
   bool found_worker = false;
   do {
     grpc_pollset *inspect = neighbourhood->active_root;
@@ -672,6 +795,7 @@ static bool check_neighbourhood_for_available_poller(
               }
               SET_KICK_STATE(inspect_worker, DESIGNATED_POLLER);
               if (inspect_worker->initialized_cv) {
+                GPR_TIMER_MARK("signal worker", 0);
                 gpr_cv_signal(&inspect_worker->cv);
               }
             } else {
@@ -707,12 +831,14 @@ static bool check_neighbourhood_for_available_poller(
     }
     gpr_mu_unlock(&inspect->mu);
   } while (!found_worker);
+  GPR_TIMER_END("check_neighbourhood_for_available_poller", 0);
   return found_worker;
 }
 
 static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                        grpc_pollset_worker *worker,
                        grpc_pollset_worker **worker_hdl) {
+  GPR_TIMER_BEGIN("end_worker", 0);
   if (GRPC_TRACER_ON(grpc_polling_trace)) {
     gpr_log(GPR_DEBUG, "PS:%p END_WORKER:%p", pollset, worker);
   }
@@ -782,42 +908,71 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     pollset_maybe_finish_shutdown(exec_ctx, pollset);
   }
   GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker);
+  GPR_TIMER_END("end_worker", 0);
 }
 
 /* pollset->po.mu lock must be held by the caller before calling this.
    The function pollset_work() may temporarily release the lock (pollset->po.mu)
    during the course of its execution but it will always re-acquire the lock and
    ensure that it is held by the time the function returns */
-static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
                                 grpc_pollset_worker **worker_hdl,
                                 gpr_timespec now, gpr_timespec deadline) {
   grpc_pollset_worker worker;
   grpc_error *error = GRPC_ERROR_NONE;
   static const char *err_desc = "pollset_work";
-  if (pollset->kicked_without_poller) {
-    pollset->kicked_without_poller = false;
+  GPR_TIMER_BEGIN("pollset_work", 0);
+  if (ps->kicked_without_poller) {
+    ps->kicked_without_poller = false;
+    GPR_TIMER_END("pollset_work", 0);
     return GRPC_ERROR_NONE;
   }
-  if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) {
-    gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
+
+  if (begin_worker(ps, &worker, worker_hdl, &now, deadline)) {
+    gpr_tls_set(&g_current_thread_pollset, (intptr_t)ps);
     gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
-    GPR_ASSERT(!pollset->shutting_down);
-    GPR_ASSERT(!pollset->seen_inactive);
-    gpr_mu_unlock(&pollset->mu);
-    append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline),
-                 err_desc);
-    gpr_mu_lock(&pollset->mu);
+    GPR_ASSERT(!ps->shutting_down);
+    GPR_ASSERT(!ps->seen_inactive);
+
+    gpr_mu_unlock(&ps->mu); /* unlock */
+    /* This is the designated polling thread at this point and should ideally do
+       polling. However, if there are unprocessed events left from a previous
+       call to do_epoll_wait(), skip calling epoll_wait() in this iteration and
+       process the pending epoll events.
+
+       The reason for decoupling do_epoll_wait and process_epoll_events is to
+       better distrubute the work (i.e handling epoll events) across multiple
+       threads
+
+       process_epoll_events() returns very quickly: It just queues the work on
+       exec_ctx but does not execute it (the actual exectution or more
+       accurately grpc_exec_ctx_flush() happens in end_worker() AFTER selecting
+       a designated poller). So we are not waiting long periods without a
+       designated poller */
+    if (gpr_atm_acq_load(&g_epoll_set.cursor) ==
+        gpr_atm_acq_load(&g_epoll_set.num_events)) {
+      append_error(&error, do_epoll_wait(exec_ctx, ps, now, deadline),
+                   err_desc);
+    }
+    append_error(&error, process_epoll_events(exec_ctx, ps), err_desc);
+
+    gpr_mu_lock(&ps->mu); /* lock */
+
     gpr_tls_set(&g_current_thread_worker, 0);
   } else {
-    gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
+    gpr_tls_set(&g_current_thread_pollset, (intptr_t)ps);
   }
-  end_worker(exec_ctx, pollset, &worker, worker_hdl);
+  end_worker(exec_ctx, ps, &worker, worker_hdl);
+
   gpr_tls_set(&g_current_thread_pollset, 0);
+  GPR_TIMER_END("pollset_work", 0);
   return error;
 }
 
 static grpc_error *pollset_kick(grpc_pollset *pollset,
                                 grpc_pollset_worker *specific_worker) {
+  GPR_TIMER_BEGIN("pollset_kick", 0);
+  grpc_error *ret_err = GRPC_ERROR_NONE;
   if (GRPC_TRACER_ON(grpc_polling_trace)) {
     gpr_strvec log;
     gpr_strvec_init(&log);
@@ -852,7 +1007,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
         if (GRPC_TRACER_ON(grpc_polling_trace)) {
           gpr_log(GPR_ERROR, " .. kicked_without_poller");
         }
-        return GRPC_ERROR_NONE;
+        goto done;
       }
       grpc_pollset_worker *next_worker = root_worker->next;
       if (root_worker->kick_state == KICKED) {
@@ -860,13 +1015,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
           gpr_log(GPR_ERROR, " .. already kicked %p", root_worker);
         }
         SET_KICK_STATE(root_worker, KICKED);
-        return GRPC_ERROR_NONE;
+        goto done;
       } else if (next_worker->kick_state == KICKED) {
         if (GRPC_TRACER_ON(grpc_polling_trace)) {
           gpr_log(GPR_ERROR, " .. already kicked %p", next_worker);
         }
         SET_KICK_STATE(next_worker, KICKED);
-        return GRPC_ERROR_NONE;
+        goto done;
       } else if (root_worker ==
                      next_worker &&  // only try and wake up a poller if
                                      // there is no next worker
@@ -876,7 +1031,8 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
           gpr_log(GPR_ERROR, " .. kicked %p", root_worker);
         }
         SET_KICK_STATE(root_worker, KICKED);
-        return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+        ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+        goto done;
       } else if (next_worker->kick_state == UNKICKED) {
         if (GRPC_TRACER_ON(grpc_polling_trace)) {
           gpr_log(GPR_ERROR, " .. kicked %p", next_worker);
@@ -884,7 +1040,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
         GPR_ASSERT(next_worker->initialized_cv);
         SET_KICK_STATE(next_worker, KICKED);
         gpr_cv_signal(&next_worker->cv);
-        return GRPC_ERROR_NONE;
+        goto done;
       } else if (next_worker->kick_state == DESIGNATED_POLLER) {
         if (root_worker->kick_state != DESIGNATED_POLLER) {
           if (GRPC_TRACER_ON(grpc_polling_trace)) {
@@ -897,59 +1053,64 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
           if (root_worker->initialized_cv) {
             gpr_cv_signal(&root_worker->cv);
           }
-          return GRPC_ERROR_NONE;
+          goto done;
         } else {
           if (GRPC_TRACER_ON(grpc_polling_trace)) {
             gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker,
                     root_worker);
           }
           SET_KICK_STATE(next_worker, KICKED);
-          return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+          ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+          goto done;
         }
       } else {
         GPR_ASSERT(next_worker->kick_state == KICKED);
         SET_KICK_STATE(next_worker, KICKED);
-        return GRPC_ERROR_NONE;
+        goto done;
       }
     } else {
       if (GRPC_TRACER_ON(grpc_polling_trace)) {
         gpr_log(GPR_ERROR, " .. kicked while waking up");
       }
-      return GRPC_ERROR_NONE;
+      goto done;
     }
   } else if (specific_worker->kick_state == KICKED) {
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_ERROR, " .. specific worker already kicked");
     }
-    return GRPC_ERROR_NONE;
+    goto done;
   } else if (gpr_tls_get(&g_current_thread_worker) ==
              (intptr_t)specific_worker) {
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker);
     }
     SET_KICK_STATE(specific_worker, KICKED);
-    return GRPC_ERROR_NONE;
+    goto done;
   } else if (specific_worker ==
              (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) {
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_ERROR, " .. kick active poller");
     }
     SET_KICK_STATE(specific_worker, KICKED);
-    return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+    ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+    goto done;
   } else if (specific_worker->initialized_cv) {
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_ERROR, " .. kick waiting worker");
     }
     SET_KICK_STATE(specific_worker, KICKED);
     gpr_cv_signal(&specific_worker->cv);
-    return GRPC_ERROR_NONE;
+    goto done;
   } else {
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_ERROR, " .. kick non-waiting worker");
     }
     SET_KICK_STATE(specific_worker, KICKED);
-    return GRPC_ERROR_NONE;
+    goto done;
   }
+done:
+  GPR_TIMER_END("pollset_kick", 0);
+  return ret_err;
 }
 
 static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -993,7 +1154,7 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
 static void shutdown_engine(void) {
   fd_global_shutdown();
   pollset_global_shutdown();
-  close(g_epfd);
+  epoll_set_shutdown();
 }
 
 static const grpc_event_engine_vtable vtable = {
@@ -1028,28 +1189,29 @@ static const grpc_event_engine_vtable vtable = {
 };
 
 /* It is possible that GLIBC has epoll but the underlying kernel doesn't.
- * Create a dummy epoll_fd to make sure epoll support is available */
+ * Create epoll_fd (epoll_set_init() takes care of that) to make sure epoll
+ * support is available */
 const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) {
+  if (!explicit_request) {
+    return NULL;
+  }
+
   if (!grpc_has_wakeup_fd()) {
     return NULL;
   }
 
-  g_epfd = epoll_create1(EPOLL_CLOEXEC);
-  if (g_epfd < 0) {
-    gpr_log(GPR_ERROR, "epoll unavailable");
+  if (!epoll_set_init()) {
     return NULL;
   }
 
   fd_global_init();
 
   if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) {
-    close(g_epfd);
     fd_global_shutdown();
+    epoll_set_shutdown();
     return NULL;
   }
 
-  gpr_log(GPR_ERROR, "grpc epoll fd: %d", g_epfd);
-
   return &vtable;
 }
 
diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
index 5166dc2ac2..f2f3e15704 100644
--- a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
@@ -931,7 +931,7 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
@@ -952,8 +952,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
        before doing this.) */
   if (fd->po.pi != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, false /* is_fd_closed */,
-                                    &error);
+    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
index 1058f69a83..07c8eadf4f 100644
--- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
@@ -493,8 +493,8 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
+  bool is_fd_closed = already_closed;
   grpc_error *error = GRPC_ERROR_NONE;
   epoll_set *unref_eps = NULL;
 
@@ -505,7 +505,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!is_fd_closed) {
     close(fd->fd);
     is_fd_closed = true;
   }
diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c
index a2fa4ad9a5..1f4adea5d4 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.c
+++ b/src/core/lib/iomgr/ev_epollex_linux.c
@@ -49,7 +49,7 @@
 #include "src/core/lib/support/spinlock.h"
 
 /*******************************************************************************
- * Pollset-set sibling link
+ * Polling object
  */
 
 typedef enum {
@@ -380,8 +380,8 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
+  bool is_fd_closed = already_closed;
   grpc_error *error = GRPC_ERROR_NONE;
 
   gpr_mu_lock(&fd->pollable.po.mu);
@@ -392,7 +392,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!is_fd_closed) {
     close(fd->fd);
     is_fd_closed = true;
   }
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c
index a84c208e1a..070d75e42a 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.c
+++ b/src/core/lib/iomgr/ev_epollsig_linux.c
@@ -854,7 +854,7 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
@@ -875,8 +875,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
        before doing this.) */
   if (fd->po.pi != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, false /* is_fd_closed */,
-                                    &error);
+    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
@@ -1731,7 +1730,7 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(
   if (!is_grpc_wakeup_signal_initialized) {
     /* TODO(ctiller): when other epoll engines are ready, remove the true || to
      * force this to be explitly chosen if needed */
-    if (explicit_request) {
+    if (true || explicit_request) {
       grpc_use_signal(SIGRTMIN + 6);
     } else {
       return NULL;
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index 593d53e4eb..266c280c48 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -42,6 +42,7 @@
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/murmur_hash.h"
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
 
@@ -239,22 +240,43 @@ struct grpc_pollset_set {
  * condition variable polling definitions
  */
 
+#define POLLCV_THREAD_GRACE_MS 1000
 #define CV_POLL_PERIOD_MS 1000
 #define CV_DEFAULT_TABLE_SIZE 16
 
-typedef enum poll_status_t { INPROGRESS, COMPLETED, CANCELLED } poll_status_t;
-
-typedef struct poll_args {
+typedef struct poll_result {
   gpr_refcount refcount;
-  gpr_cv *cv;
+  cv_node *watchers;
+  int watchcount;
   struct pollfd *fds;
   nfds_t nfds;
-  int timeout;
   int retval;
   int err;
-  gpr_atm status;
+  int completed;
+} poll_result;
+
+typedef struct poll_args {
+  gpr_cv trigger;
+  int trigger_set;
+  struct pollfd *fds;
+  nfds_t nfds;
+  poll_result *result;
+  struct poll_args *next;
+  struct poll_args *prev;
 } poll_args;
 
+// This is a 2-tiered cache, we mantain a hash table
+// of active poll calls, so we can wait on the result
+// of that call.  We also maintain a freelist of inactive
+// poll threads.
+typedef struct poll_hash_table {
+  poll_args *free_pollers;
+  poll_args **active_pollers;
+  unsigned int size;
+  unsigned int count;
+} poll_hash_table;
+
+poll_hash_table poll_cache;
 cv_fd_table g_cvfds;
 
 /*******************************************************************************
@@ -398,11 +420,14 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   fd->on_done_closure = on_done;
   fd->released = release_fd != NULL;
-  if (fd->released) {
+  if (release_fd != NULL) {
     *release_fd = fd->fd;
+    fd->released = true;
+  } else if (already_closed) {
+    fd->released = true;
   }
   gpr_mu_lock(&fd->mu);
   REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
@@ -1284,43 +1309,205 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
  * Condition Variable polling extensions
  */
 
-static void decref_poll_args(poll_args *args) {
-  if (gpr_unref(&args->refcount)) {
-    gpr_free(args->fds);
-    gpr_cv_destroy(args->cv);
-    gpr_free(args->cv);
-    gpr_free(args);
+static void run_poll(void *args);
+static void cache_poller_locked(poll_args *args);
+
+static void cache_insert_locked(poll_args *args) {
+  uint32_t key = gpr_murmur_hash3(args->fds, args->nfds * sizeof(struct pollfd),
+                                  0xDEADBEEF);
+  key = key % poll_cache.size;
+  if (poll_cache.active_pollers[key]) {
+    poll_cache.active_pollers[key]->prev = args;
   }
+  args->next = poll_cache.active_pollers[key];
+  args->prev = NULL;
+  poll_cache.active_pollers[key] = args;
+  poll_cache.count++;
 }
 
-// Poll in a background thread
-static void run_poll(void *arg) {
-  int timeout, retval;
-  poll_args *pargs = (poll_args *)arg;
-  while (gpr_atm_no_barrier_load(&pargs->status) == INPROGRESS) {
-    if (pargs->timeout < 0) {
-      timeout = CV_POLL_PERIOD_MS;
-    } else {
-      timeout = GPR_MIN(CV_POLL_PERIOD_MS, pargs->timeout);
-      pargs->timeout -= timeout;
+static void init_result(poll_args *pargs) {
+  pargs->result = gpr_malloc(sizeof(poll_result));
+  gpr_ref_init(&pargs->result->refcount, 1);
+  pargs->result->watchers = NULL;
+  pargs->result->watchcount = 0;
+  pargs->result->fds = gpr_malloc(sizeof(struct pollfd) * pargs->nfds);
+  memcpy(pargs->result->fds, pargs->fds, sizeof(struct pollfd) * pargs->nfds);
+  pargs->result->nfds = pargs->nfds;
+  pargs->result->retval = 0;
+  pargs->result->err = 0;
+  pargs->result->completed = 0;
+}
+
+// Creates a poll_args object for a given arguments to poll().
+// This object may return a poll_args in the cache.
+static poll_args *get_poller_locked(struct pollfd *fds, nfds_t count) {
+  uint32_t key =
+      gpr_murmur_hash3(fds, count * sizeof(struct pollfd), 0xDEADBEEF);
+  key = key % poll_cache.size;
+  poll_args *curr = poll_cache.active_pollers[key];
+  while (curr) {
+    if (curr->nfds == count &&
+        memcmp(curr->fds, fds, count * sizeof(struct pollfd)) == 0) {
+      gpr_free(fds);
+      return curr;
     }
-    retval = g_cvfds.poll(pargs->fds, pargs->nfds, timeout);
-    if (retval != 0 || pargs->timeout == 0) {
-      pargs->retval = retval;
-      pargs->err = errno;
-      break;
+    curr = curr->next;
+  }
+
+  if (poll_cache.free_pollers) {
+    poll_args *pargs = poll_cache.free_pollers;
+    poll_cache.free_pollers = pargs->next;
+    if (poll_cache.free_pollers) {
+      poll_cache.free_pollers->prev = NULL;
     }
+    pargs->fds = fds;
+    pargs->nfds = count;
+    pargs->next = NULL;
+    pargs->prev = NULL;
+    init_result(pargs);
+    cache_poller_locked(pargs);
+    return pargs;
+  }
+
+  poll_args *pargs = gpr_malloc(sizeof(struct poll_args));
+  gpr_cv_init(&pargs->trigger);
+  pargs->fds = fds;
+  pargs->nfds = count;
+  pargs->next = NULL;
+  pargs->prev = NULL;
+  pargs->trigger_set = 0;
+  init_result(pargs);
+  cache_poller_locked(pargs);
+  gpr_thd_id t_id;
+  gpr_thd_options opt = gpr_thd_options_default();
+  gpr_ref(&g_cvfds.pollcount);
+  gpr_thd_options_set_detached(&opt);
+  GPR_ASSERT(gpr_thd_new(&t_id, &run_poll, pargs, &opt));
+  return pargs;
+}
+
+static void cache_delete_locked(poll_args *args) {
+  if (!args->prev) {
+    uint32_t key = gpr_murmur_hash3(
+        args->fds, args->nfds * sizeof(struct pollfd), 0xDEADBEEF);
+    key = key % poll_cache.size;
+    GPR_ASSERT(poll_cache.active_pollers[key] == args);
+    poll_cache.active_pollers[key] = args->next;
+  } else {
+    args->prev->next = args->next;
   }
-  gpr_mu_lock(&g_cvfds.mu);
-  if (gpr_atm_no_barrier_load(&pargs->status) == INPROGRESS) {
-    // Signal main thread that the poll completed
-    gpr_atm_no_barrier_store(&pargs->status, COMPLETED);
-    gpr_cv_signal(pargs->cv);
+
+  if (args->next) {
+    args->next->prev = args->prev;
   }
-  decref_poll_args(pargs);
-  g_cvfds.pollcount--;
-  if (g_cvfds.shutdown && g_cvfds.pollcount == 0) {
-    gpr_cv_signal(&g_cvfds.shutdown_complete);
+
+  poll_cache.count--;
+  if (poll_cache.free_pollers) {
+    poll_cache.free_pollers->prev = args;
+  }
+  args->prev = NULL;
+  args->next = poll_cache.free_pollers;
+  gpr_free(args->fds);
+  poll_cache.free_pollers = args;
+}
+
+static void cache_poller_locked(poll_args *args) {
+  if (poll_cache.count + 1 > poll_cache.size / 2) {
+    poll_args **old_active_pollers = poll_cache.active_pollers;
+    poll_cache.size = poll_cache.size * 2;
+    poll_cache.count = 0;
+    poll_cache.active_pollers = gpr_malloc(sizeof(void *) * poll_cache.size);
+    for (unsigned int i = 0; i < poll_cache.size; i++) {
+      poll_cache.active_pollers[i] = NULL;
+    }
+    for (unsigned int i = 0; i < poll_cache.size / 2; i++) {
+      poll_args *curr = old_active_pollers[i];
+      poll_args *next = NULL;
+      while (curr) {
+        next = curr->next;
+        cache_insert_locked(curr);
+        curr = next;
+      }
+    }
+    gpr_free(old_active_pollers);
+  }
+
+  cache_insert_locked(args);
+}
+
+static void cache_destroy_locked(poll_args *args) {
+  if (args->next) {
+    args->next->prev = args->prev;
+  }
+
+  if (args->prev) {
+    args->prev->next = args->next;
+  } else {
+    poll_cache.free_pollers = args->next;
+  }
+
+  gpr_free(args);
+}
+
+static void decref_poll_result(poll_result *res) {
+  if (gpr_unref(&res->refcount)) {
+    GPR_ASSERT(!res->watchers);
+    gpr_free(res->fds);
+    gpr_free(res);
+  }
+}
+
+void remove_cvn(cv_node **head, cv_node *target) {
+  if (target->next) {
+    target->next->prev = target->prev;
+  }
+
+  if (target->prev) {
+    target->prev->next = target->next;
+  } else {
+    *head = target->next;
+  }
+}
+
+gpr_timespec thread_grace;
+
+// Poll in a background thread
+static void run_poll(void *args) {
+  poll_args *pargs = (poll_args *)args;
+  while (1) {
+    poll_result *result = pargs->result;
+    int retval = g_cvfds.poll(result->fds, result->nfds, CV_POLL_PERIOD_MS);
+    gpr_mu_lock(&g_cvfds.mu);
+    if (retval != 0) {
+      result->completed = 1;
+      result->retval = retval;
+      result->err = errno;
+      cv_node *watcher = result->watchers;
+      while (watcher) {
+        gpr_cv_signal(watcher->cv);
+        watcher = watcher->next;
+      }
+    }
+    if (result->watchcount == 0 || result->completed) {
+      cache_delete_locked(pargs);
+      decref_poll_result(result);
+      // Leave this polling thread alive for a grace period to do another poll()
+      // op
+      gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME);
+      deadline = gpr_time_add(deadline, thread_grace);
+      pargs->trigger_set = 0;
+      gpr_cv_wait(&pargs->trigger, &g_cvfds.mu, deadline);
+      if (!pargs->trigger_set) {
+        cache_destroy_locked(pargs);
+        break;
+      }
+    }
+    gpr_mu_unlock(&g_cvfds.mu);
+  }
+
+  // We still have the lock here
+  if (gpr_unref(&g_cvfds.pollcount)) {
+    gpr_cv_signal(&g_cvfds.shutdown_cv);
   }
   gpr_mu_unlock(&g_cvfds.mu);
 }
@@ -1329,24 +1516,29 @@ static void run_poll(void *arg) {
 static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
   unsigned int i;
   int res, idx;
-  gpr_cv *pollcv;
-  cv_node *cvn, *prev;
+  cv_node *pollcv;
   int skip_poll = 0;
   nfds_t nsockfds = 0;
-  gpr_thd_id t_id;
-  gpr_thd_options opt;
-  poll_args *pargs = NULL;
+  poll_result *result = NULL;
   gpr_mu_lock(&g_cvfds.mu);
-  pollcv = gpr_malloc(sizeof(gpr_cv));
-  gpr_cv_init(pollcv);
+  pollcv = gpr_malloc(sizeof(cv_node));
+  pollcv->next = NULL;
+  gpr_cv pollcv_cv;
+  gpr_cv_init(&pollcv_cv);
+  pollcv->cv = &pollcv_cv;
+  cv_node *fd_cvs = gpr_malloc(nfds * sizeof(cv_node));
+
   for (i = 0; i < nfds; i++) {
     fds[i].revents = 0;
     if (fds[i].fd < 0 && (fds[i].events & POLLIN)) {
       idx = FD_TO_IDX(fds[i].fd);
-      cvn = gpr_malloc(sizeof(cv_node));
-      cvn->cv = pollcv;
-      cvn->next = g_cvfds.cvfds[idx].cvs;
-      g_cvfds.cvfds[idx].cvs = cvn;
+      fd_cvs[i].cv = &pollcv_cv;
+      fd_cvs[i].prev = NULL;
+      fd_cvs[i].next = g_cvfds.cvfds[idx].cvs;
+      if (g_cvfds.cvfds[idx].cvs) {
+        g_cvfds.cvfds[idx].cvs->prev = &(fd_cvs[i]);
+      }
+      g_cvfds.cvfds[idx].cvs = &(fd_cvs[i]);
       // Don't bother polling if a wakeup fd is ready
       if (g_cvfds.cvfds[idx].is_set) {
         skip_poll = 1;
@@ -1356,81 +1548,68 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
     }
   }
 
+  gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME);
+  if (timeout < 0) {
+    deadline = gpr_inf_future(GPR_CLOCK_REALTIME);
+  } else {
+    deadline =
+        gpr_time_add(deadline, gpr_time_from_millis(timeout, GPR_TIMESPAN));
+  }
+
   res = 0;
   if (!skip_poll && nsockfds > 0) {
-    pargs = gpr_malloc(sizeof(struct poll_args));
-    // Both the main thread and calling thread get a reference
-    gpr_ref_init(&pargs->refcount, 2);
-    pargs->cv = pollcv;
-    pargs->fds = gpr_malloc(sizeof(struct pollfd) * nsockfds);
-    pargs->nfds = nsockfds;
-    pargs->timeout = timeout;
-    pargs->retval = 0;
-    pargs->err = 0;
-    gpr_atm_no_barrier_store(&pargs->status, INPROGRESS);
+    struct pollfd *pollfds = gpr_malloc(sizeof(struct pollfd) * nsockfds);
     idx = 0;
     for (i = 0; i < nfds; i++) {
       if (fds[i].fd >= 0) {
-        pargs->fds[idx].fd = fds[i].fd;
-        pargs->fds[idx].events = fds[i].events;
-        pargs->fds[idx].revents = 0;
+        pollfds[idx].fd = fds[i].fd;
+        pollfds[idx].events = fds[i].events;
+        pollfds[idx].revents = 0;
         idx++;
       }
     }
-    g_cvfds.pollcount++;
-    opt = gpr_thd_options_default();
-    gpr_thd_options_set_detached(&opt);
-    GPR_ASSERT(gpr_thd_new(&t_id, &run_poll, pargs, &opt));
-    // We want the poll() thread to trigger the deadline, so wait forever here
-    gpr_cv_wait(pollcv, &g_cvfds.mu, gpr_inf_future(GPR_CLOCK_MONOTONIC));
-    if (gpr_atm_no_barrier_load(&pargs->status) == COMPLETED) {
-      res = pargs->retval;
-      errno = pargs->err;
-    } else {
-      errno = 0;
-      gpr_atm_no_barrier_store(&pargs->status, CANCELLED);
+    poll_args *pargs = get_poller_locked(pollfds, nsockfds);
+    result = pargs->result;
+    pollcv->next = result->watchers;
+    pollcv->prev = NULL;
+    if (result->watchers) {
+      result->watchers->prev = pollcv;
     }
+    result->watchers = pollcv;
+    result->watchcount++;
+    gpr_ref(&result->refcount);
+
+    pargs->trigger_set = 1;
+    gpr_cv_signal(&pargs->trigger);
+    gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline);
+    res = result->retval;
+    errno = result->err;
+    result->watchcount--;
+    remove_cvn(&result->watchers, pollcv);
   } else if (!skip_poll) {
-    gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME);
-    deadline =
-        gpr_time_add(deadline, gpr_time_from_millis(timeout, GPR_TIMESPAN));
-    gpr_cv_wait(pollcv, &g_cvfds.mu, deadline);
+    gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline);
   }
 
   idx = 0;
   for (i = 0; i < nfds; i++) {
     if (fds[i].fd < 0 && (fds[i].events & POLLIN)) {
-      cvn = g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs;
-      prev = NULL;
-      while (cvn->cv != pollcv) {
-        prev = cvn;
-        cvn = cvn->next;
-        GPR_ASSERT(cvn);
-      }
-      if (!prev) {
-        g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs = cvn->next;
-      } else {
-        prev->next = cvn->next;
-      }
-      gpr_free(cvn);
-
+      remove_cvn(&g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i]));
       if (g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].is_set) {
         fds[i].revents = POLLIN;
         if (res >= 0) res++;
       }
-    } else if (!skip_poll && fds[i].fd >= 0 &&
-               gpr_atm_no_barrier_load(&pargs->status) == COMPLETED) {
-      fds[i].revents = pargs->fds[idx].revents;
+    } else if (!skip_poll && fds[i].fd >= 0 && result->completed) {
+      fds[i].revents = result->fds[idx].revents;
       idx++;
     }
   }
 
-  if (pargs) {
-    decref_poll_args(pargs);
-  } else {
-    gpr_cv_destroy(pollcv);
-    gpr_free(pollcv);
+  gpr_free(fd_cvs);
+  gpr_free(pollcv);
+  if (result) {
+    decref_poll_result(result);
   }
+
   gpr_mu_unlock(&g_cvfds.mu);
 
   return res;
@@ -1439,12 +1618,12 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
 static void global_cv_fd_table_init() {
   gpr_mu_init(&g_cvfds.mu);
   gpr_mu_lock(&g_cvfds.mu);
-  gpr_cv_init(&g_cvfds.shutdown_complete);
-  g_cvfds.shutdown = 0;
-  g_cvfds.pollcount = 0;
+  gpr_cv_init(&g_cvfds.shutdown_cv);
+  gpr_ref_init(&g_cvfds.pollcount, 1);
   g_cvfds.size = CV_DEFAULT_TABLE_SIZE;
   g_cvfds.cvfds = gpr_malloc(sizeof(fd_node) * CV_DEFAULT_TABLE_SIZE);
   g_cvfds.free_fds = NULL;
+  thread_grace = gpr_time_from_millis(POLLCV_THREAD_GRACE_MS, GPR_TIMESPAN);
   for (int i = 0; i < CV_DEFAULT_TABLE_SIZE; i++) {
     g_cvfds.cvfds[i].is_set = 0;
     g_cvfds.cvfds[i].cvs = NULL;
@@ -1454,23 +1633,35 @@ static void global_cv_fd_table_init() {
   // Override the poll function with one that supports cvfds
   g_cvfds.poll = grpc_poll_function;
   grpc_poll_function = &cvfd_poll;
+
+  // Initialize the cache
+  poll_cache.size = 32;
+  poll_cache.count = 0;
+  poll_cache.free_pollers = NULL;
+  poll_cache.active_pollers = gpr_malloc(sizeof(void *) * 32);
+  for (unsigned int i = 0; i < poll_cache.size; i++) {
+    poll_cache.active_pollers[i] = NULL;
+  }
+
   gpr_mu_unlock(&g_cvfds.mu);
 }
 
 static void global_cv_fd_table_shutdown() {
   gpr_mu_lock(&g_cvfds.mu);
-  g_cvfds.shutdown = 1;
   // Attempt to wait for all abandoned poll() threads to terminate
   // Not doing so will result in reported memory leaks
-  if (g_cvfds.pollcount > 0) {
-    int res = gpr_cv_wait(&g_cvfds.shutdown_complete, &g_cvfds.mu,
+  if (!gpr_unref(&g_cvfds.pollcount)) {
+    int res = gpr_cv_wait(&g_cvfds.shutdown_cv, &g_cvfds.mu,
                           gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
                                        gpr_time_from_seconds(3, GPR_TIMESPAN)));
     GPR_ASSERT(res == 0);
   }
-  gpr_cv_destroy(&g_cvfds.shutdown_complete);
+  gpr_cv_destroy(&g_cvfds.shutdown_cv);
   grpc_poll_function = g_cvfds.poll;
   gpr_free(g_cvfds.cvfds);
+
+  gpr_free(poll_cache.active_pollers);
+
   gpr_mu_unlock(&g_cvfds.mu);
   gpr_mu_destroy(&g_cvfds.mu);
 }
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index cff77e641c..91f8cd5482 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -170,8 +170,9 @@ int grpc_fd_wrapped_fd(grpc_fd *fd) {
 }
 
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason) {
-  g_event_engine->fd_orphan(exec_ctx, fd, on_done, release_fd, reason);
+                    int *release_fd, bool already_closed, const char *reason) {
+  g_event_engine->fd_orphan(exec_ctx, fd, on_done, release_fd, already_closed,
+                            reason);
 }
 
 void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 0de7333843..1108e46ef8 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -37,7 +37,7 @@ typedef struct grpc_event_engine_vtable {
   grpc_fd *(*fd_create)(int fd, const char *name);
   int (*fd_wrapped_fd)(grpc_fd *fd);
   void (*fd_orphan)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason);
+                    int *release_fd, bool already_closed, const char *reason);
   void (*fd_shutdown)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why);
   void (*fd_notify_on_read)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                             grpc_closure *closure);
@@ -104,7 +104,7 @@ int grpc_fd_wrapped_fd(grpc_fd *fd);
    notify_on_write.
    MUST NOT be called with a pollset lock taken */
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason);
+                    int *release_fd, bool already_closed, const char *reason);
 
 /* Has grpc_fd_shutdown been called on an fd? */
 bool grpc_fd_is_shutdown(grpc_fd *fd);
diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c
index 833170ceed..41c69add17 100644
--- a/src/core/lib/iomgr/exec_ctx.c
+++ b/src/core/lib/iomgr/exec_ctx.c
@@ -51,33 +51,6 @@ bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) {
          !grpc_closure_list_empty(exec_ctx->closure_list);
 }
 
-bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
-  bool did_something = 0;
-  GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);
-  for (;;) {
-    if (!grpc_closure_list_empty(exec_ctx->closure_list)) {
-      grpc_closure *c = exec_ctx->closure_list.head;
-      exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL;
-      while (c != NULL) {
-        grpc_closure *next = c->next_data.next;
-        grpc_error *error = c->error_data.error;
-        did_something = true;
-#ifndef NDEBUG
-        c->scheduled = false;
-#endif
-        c->cb(exec_ctx, c->cb_arg, error);
-        GRPC_ERROR_UNREF(error);
-        c = next;
-      }
-    } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) {
-      break;
-    }
-  }
-  GPR_ASSERT(exec_ctx->active_combiner == NULL);
-  GPR_TIMER_END("grpc_exec_ctx_flush", 0);
-  return did_something;
-}
-
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {
   exec_ctx->flags |= GRPC_EXEC_CTX_FLAG_IS_FINISHED;
   grpc_exec_ctx_flush(exec_ctx);
@@ -103,6 +76,29 @@ static void exec_ctx_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
   GRPC_ERROR_UNREF(error);
 }
 
+bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
+  bool did_something = 0;
+  GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);
+  for (;;) {
+    if (!grpc_closure_list_empty(exec_ctx->closure_list)) {
+      grpc_closure *c = exec_ctx->closure_list.head;
+      exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL;
+      while (c != NULL) {
+        grpc_closure *next = c->next_data.next;
+        grpc_error *error = c->error_data.error;
+        did_something = true;
+        exec_ctx_run(exec_ctx, c, error);
+        c = next;
+      }
+    } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) {
+      break;
+    }
+  }
+  GPR_ASSERT(exec_ctx->active_combiner == NULL);
+  GPR_TIMER_END("grpc_exec_ctx_flush", 0);
+  return did_something;
+}
+
 static void exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                            grpc_error *error) {
   grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
diff --git a/src/core/lib/iomgr/gethostname.h b/src/core/lib/iomgr/gethostname.h
new file mode 100644
index 0000000000..9c6b9d8d42
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname.h
@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H
+#define GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H
+
+// Returns the hostname of the local machine.
+// Caller takes ownership of result.
+char *grpc_gethostname();
+
+#endif /* GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H */
diff --git a/src/core/lib/iomgr/gethostname_fallback.c b/src/core/lib/iomgr/gethostname_fallback.c
new file mode 100644
index 0000000000..6229461568
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname_fallback.c
@@ -0,0 +1,27 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_GETHOSTNAME_FALLBACK
+
+#include <stddef.h>
+
+char *grpc_gethostname() { return NULL; }
+
+#endif  // GRPC_GETHOSTNAME_FALLBACK
diff --git a/src/core/lib/iomgr/gethostname_host_name_max.c b/src/core/lib/iomgr/gethostname_host_name_max.c
new file mode 100644
index 0000000000..4d0511412e
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname_host_name_max.c
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_POSIX_HOST_NAME_MAX
+
+#include <limits.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+
+char *grpc_gethostname() {
+  char *hostname = (char *)gpr_malloc(HOST_NAME_MAX);
+  if (gethostname(hostname, HOST_NAME_MAX) != 0) {
+    gpr_free(hostname);
+    return NULL;
+  }
+  return hostname;
+}
+
+#endif  // GRPC_POSIX_HOST_NAME_MAX
diff --git a/src/core/lib/iomgr/gethostname_sysconf.c b/src/core/lib/iomgr/gethostname_sysconf.c
new file mode 100644
index 0000000000..51bac5d69d
--- /dev/null
+++ b/src/core/lib/iomgr/gethostname_sysconf.c
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_POSIX_SYSCONF
+
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+
+char *grpc_gethostname() {
+  size_t host_name_max = (size_t)sysconf(_SC_HOST_NAME_MAX);
+  char *hostname = (char *)gpr_malloc(host_name_max);
+  if (gethostname(hostname, host_name_max) != 0) {
+    gpr_free(hostname);
+    return NULL;
+  }
+  return hostname;
+}
+
+#endif  // GRPC_POSIX_SYSCONF
diff --git a/src/core/lib/iomgr/nameser.h b/src/core/lib/iomgr/nameser.h
new file mode 100644
index 0000000000..daed6de518
--- /dev/null
+++ b/src/core/lib/iomgr/nameser.h
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_NAMESER_H
+#define GRPC_CORE_LIB_IOMGR_NAMESER_H
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_HAVE_ARPA_NAMESER
+
+#include <arpa/nameser.h>
+
+#else /* GRPC_HAVE_ARPA_NAMESER */
+
+typedef enum __ns_class {
+  ns_c_invalid = 0, /* Cookie. */
+  ns_c_in = 1,      /* Internet. */
+  ns_c_2 = 2,       /* unallocated/unsupported. */
+  ns_c_chaos = 3,   /* MIT Chaos-net. */
+  ns_c_hs = 4,      /* MIT Hesiod. */
+  /* Query class values which do not appear in resource records */
+  ns_c_none = 254, /* for prereq. sections in update requests */
+  ns_c_any = 255,  /* Wildcard match. */
+  ns_c_max = 65536
+} ns_class;
+
+typedef enum __ns_type {
+  ns_t_invalid = 0,   /* Cookie. */
+  ns_t_a = 1,         /* Host address. */
+  ns_t_ns = 2,        /* Authoritative server. */
+  ns_t_md = 3,        /* Mail destination. */
+  ns_t_mf = 4,        /* Mail forwarder. */
+  ns_t_cname = 5,     /* Canonical name. */
+  ns_t_soa = 6,       /* Start of authority zone. */
+  ns_t_mb = 7,        /* Mailbox domain name. */
+  ns_t_mg = 8,        /* Mail group member. */
+  ns_t_mr = 9,        /* Mail rename name. */
+  ns_t_null = 10,     /* Null resource record. */
+  ns_t_wks = 11,      /* Well known service. */
+  ns_t_ptr = 12,      /* Domain name pointer. */
+  ns_t_hinfo = 13,    /* Host information. */
+  ns_t_minfo = 14,    /* Mailbox information. */
+  ns_t_mx = 15,       /* Mail routing information. */
+  ns_t_txt = 16,      /* Text strings. */
+  ns_t_rp = 17,       /* Responsible person. */
+  ns_t_afsdb = 18,    /* AFS cell database. */
+  ns_t_x25 = 19,      /* X_25 calling address. */
+  ns_t_isdn = 20,     /* ISDN calling address. */
+  ns_t_rt = 21,       /* Router. */
+  ns_t_nsap = 22,     /* NSAP address. */
+  ns_t_nsap_ptr = 23, /* Reverse NSAP lookup (deprecated). */
+  ns_t_sig = 24,      /* Security signature. */
+  ns_t_key = 25,      /* Security key. */
+  ns_t_px = 26,       /* X.400 mail mapping. */
+  ns_t_gpos = 27,     /* Geographical position (withdrawn). */
+  ns_t_aaaa = 28,     /* Ip6 Address. */
+  ns_t_loc = 29,      /* Location Information. */
+  ns_t_nxt = 30,      /* Next domain (security). */
+  ns_t_eid = 31,      /* Endpoint identifier. */
+  ns_t_nimloc = 32,   /* Nimrod Locator. */
+  ns_t_srv = 33,      /* Server Selection. */
+  ns_t_atma = 34,     /* ATM Address */
+  ns_t_naptr = 35,    /* Naming Authority PoinTeR */
+  ns_t_kx = 36,       /* Key Exchange */
+  ns_t_cert = 37,     /* Certification record */
+  ns_t_a6 = 38,       /* IPv6 address (deprecates AAAA) */
+  ns_t_dname = 39,    /* Non-terminal DNAME (for IPv6) */
+  ns_t_sink = 40,     /* Kitchen sink (experimentatl) */
+  ns_t_opt = 41,      /* EDNS0 option (meta-RR) */
+  ns_t_apl = 42,      /* Address prefix list (RFC3123) */
+  ns_t_ds = 43,       /* Delegation Signer (RFC4034) */
+  ns_t_sshfp = 44,    /* SSH Key Fingerprint (RFC4255) */
+  ns_t_rrsig = 46,    /* Resource Record Signature (RFC4034) */
+  ns_t_nsec = 47,     /* Next Secure (RFC4034) */
+  ns_t_dnskey = 48,   /* DNS Public Key (RFC4034) */
+  ns_t_tkey = 249,    /* Transaction key */
+  ns_t_tsig = 250,    /* Transaction signature. */
+  ns_t_ixfr = 251,    /* Incremental zone transfer. */
+  ns_t_axfr = 252,    /* Transfer zone of authority. */
+  ns_t_mailb = 253,   /* Transfer mailbox records. */
+  ns_t_maila = 254,   /* Transfer mail agent records. */
+  ns_t_any = 255,     /* Wildcard match. */
+  ns_t_zxfr = 256,    /* BIND-specific, nonstandard. */
+  ns_t_max = 65536
+} ns_type;
+
+#endif /* GRPC_HAVE_ARPA_NAMESER */
+
+#endif /* GRPC_CORE_LIB_IOMGR_NAMESER_H */
diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h
index f5d15b4850..42033d0ba4 100644
--- a/src/core/lib/iomgr/port.h
+++ b/src/core/lib/iomgr/port.h
@@ -24,6 +24,7 @@
 #if defined(GRPC_UV)
 // Do nothing
 #elif defined(GPR_MANYLINUX1)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
 #define GRPC_HAVE_IP_PKTINFO 1
@@ -51,12 +52,14 @@
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_LINUX)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
 #define GRPC_HAVE_IP_PKTINFO 1
 #define GRPC_HAVE_MSG_NOSIGNAL 1
 #define GRPC_HAVE_UNIX_SOCKET 1
 #define GRPC_LINUX_MULTIPOLL_WITH_EPOLL 1
+#define GRPC_POSIX_HOST_NAME_MAX 1
 #define GRPC_POSIX_SOCKET 1
 #define GRPC_POSIX_SOCKETADDR 1
 #define GRPC_POSIX_WAKEUP_FD 1
@@ -82,6 +85,7 @@
 #define GRPC_POSIX_SOCKETUTILS
 #endif
 #elif defined(GPR_APPLE)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_SO_NOSIGPIPE 1
 #define GRPC_HAVE_UNIX_SOCKET 1
@@ -90,9 +94,11 @@
 #define GRPC_POSIX_SOCKET 1
 #define GRPC_POSIX_SOCKETADDR 1
 #define GRPC_POSIX_SOCKETUTILS 1
+#define GRPC_POSIX_SYSCONF 1
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_FREEBSD)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_HAVE_IFADDRS 1
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
 #define GRPC_HAVE_SO_NOSIGPIPE 1
@@ -104,6 +110,7 @@
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_NACL)
+#define GRPC_HAVE_ARPA_NAMESER 1
 #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1
 #define GRPC_POSIX_SOCKET 1
 #define GRPC_POSIX_SOCKETADDR 1
@@ -120,4 +127,11 @@
 #error Must define exactly one of GRPC_POSIX_SOCKET, GRPC_WINSOCK_SOCKET, GPR_CUSTOM_SOCKET
 #endif
 
+#if defined(GRPC_POSIX_HOST_NAME_MAX) && defined(GRPC_POSIX_SYSCONF)
+#error "Cannot define both GRPC_POSIX_HOST_NAME_MAX and GRPC_POSIX_SYSCONF"
+#endif
+#if !defined(GRPC_POSIX_HOST_NAME_MAX) && !defined(GRPC_POSIX_SYSCONF)
+#define GRPC_GETHOSTNAME_FALLBACK 1
+#endif
+
 #endif /* GRPC_CORE_LIB_IOMGR_PORT_H */
diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index 21e320a6e7..a25fba4527 100644
--- a/src/core/lib/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -209,7 +209,8 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
 finish:
   if (fd != NULL) {
     grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd);
-    grpc_fd_orphan(exec_ctx, fd, NULL, NULL, "tcp_client_orphan");
+    grpc_fd_orphan(exec_ctx, fd, NULL, NULL, false /* already_closed */,
+                   "tcp_client_orphan");
     fd = NULL;
   }
   done = (--ac->refs == 0);
@@ -295,7 +296,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   }
 
   if (errno != EWOULDBLOCK && errno != EINPROGRESS) {
-    grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error");
+    grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, false /* already_closed */,
+                   "tcp_client_connect_error");
     GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"));
     goto done;
   }
diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index 9d27de0b1c..1bace788c5 100644
--- a/src/core/lib/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -296,7 +296,7 @@ static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
 static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   grpc_fd_orphan(exec_ctx, tcp->em_fd, tcp->release_fd_cb, tcp->release_fd,
-                 "tcp_unref_orphan");
+                 false /* already_closed */, "tcp_unref_orphan");
   grpc_slice_buffer_destroy_internal(exec_ctx, &tcp->last_read_buffer);
   grpc_resource_user_unref(exec_ctx, tcp->resource_user);
   gpr_free(tcp->peer_string);
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index f304642951..0fc5c0fd86 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -166,7 +166,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
       GRPC_CLOSURE_INIT(&sp->destroyed_closure, destroyed_port, s,
                         grpc_schedule_on_exec_ctx);
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
-                     "tcp_listener_shutdown");
+                     false /* already_closed */, "tcp_listener_shutdown");
     }
     gpr_mu_unlock(&s->mu);
   } else {
diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_common.c b/src/core/lib/iomgr/tcp_server_utils_posix_common.c
index dbb43186bd..ad535bc43e 100644
--- a/src/core/lib/iomgr/tcp_server_utils_posix_common.c
+++ b/src/core/lib/iomgr/tcp_server_utils_posix_common.c
@@ -39,7 +39,7 @@
 
 #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100
 
-static gpr_once s_init_max_accept_queue_size;
+static gpr_once s_init_max_accept_queue_size = GPR_ONCE_INIT;
 static int s_max_accept_queue_size;
 
 /* get max listen queue size on linux */
diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c
index 54e7f417a7..88fa34cb7a 100644
--- a/src/core/lib/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@@ -214,7 +214,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
                       sp->server->user_data);
       }
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
-                     "udp_listener_shutdown");
+                     false /* already_closed */, "udp_listener_shutdown");
     }
     gpr_mu_unlock(&s->mu);
   } else {
diff --git a/src/core/lib/iomgr/wakeup_fd_cv.h b/src/core/lib/iomgr/wakeup_fd_cv.h
index c5dcdc9746..46e84f5843 100644
--- a/src/core/lib/iomgr/wakeup_fd_cv.h
+++ b/src/core/lib/iomgr/wakeup_fd_cv.h
@@ -43,6 +43,7 @@
 typedef struct cv_node {
   gpr_cv* cv;
   struct cv_node* next;
+  struct cv_node* prev;
 } cv_node;
 
 typedef struct fd_node {
@@ -53,9 +54,8 @@ typedef struct fd_node {
 
 typedef struct cv_fd_table {
   gpr_mu mu;
-  int pollcount;
-  int shutdown;
-  gpr_cv shutdown_complete;
+  gpr_refcount pollcount;
+  gpr_cv shutdown_cv;
   fd_node* cvfds;
   fd_node* free_fds;
   unsigned int size;