Merge branch 'timer_pool' into uberpoll

44 files changed, 459 insertions, 220 deletions

diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c
index 94382980eb..7db54d1107 100644
--- a/src/core/lib/channel/channel_stack.c
+++ b/src/core/lib/channel/channel_stack.c
@@ -38,7 +38,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-int grpc_trace_channel = 0;
+grpc_tracer_flag grpc_trace_channel = GRPC_TRACER_INITIALIZER(false);
 
 /* Memory layouts.
 
diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index fdbcbdb018..c26d61b2ef 100644
--- a/src/core/lib/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -307,10 +307,10 @@ void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx,
                                     grpc_call_element *cur_elem,
                                     grpc_error *error);
 
-extern int grpc_trace_channel;
+extern grpc_tracer_flag grpc_trace_channel;
 
 #define GRPC_CALL_LOG_OP(sev, elem, op) \
-  if (grpc_trace_channel) grpc_call_log_op(sev, elem, op)
+  if (GRPC_TRACER_ON(grpc_trace_channel)) grpc_call_log_op(sev, elem, op)
 
 #ifdef __cplusplus
 }
diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c
index 88c02edb70..44b030e4d1 100644
--- a/src/core/lib/channel/channel_stack_builder.c
+++ b/src/core/lib/channel/channel_stack_builder.c
@@ -38,7 +38,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 
-int grpc_trace_channel_stack_builder = 0;
+grpc_tracer_flag grpc_trace_channel_stack_builder =
+    GRPC_TRACER_INITIALIZER(false);
 
 typedef struct filter_node {
   struct filter_node *next;
diff --git a/src/core/lib/channel/channel_stack_builder.h b/src/core/lib/channel/channel_stack_builder.h
index c78111b00d..8cb36eb117 100644
--- a/src/core/lib/channel/channel_stack_builder.h
+++ b/src/core/lib/channel/channel_stack_builder.h
@@ -165,7 +165,7 @@ grpc_error *grpc_channel_stack_builder_finish(
 void grpc_channel_stack_builder_destroy(grpc_exec_ctx *exec_ctx,
                                         grpc_channel_stack_builder *builder);
 
-extern int grpc_trace_channel_stack_builder;
+extern grpc_tracer_flag grpc_trace_channel_stack_builder;
 
 #ifdef __cplusplus
 }
diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c
index c56046785b..54653868b8 100644
--- a/src/core/lib/debug/trace.c
+++ b/src/core/lib/debug/trace.c
@@ -42,17 +42,23 @@
 
 typedef struct tracer {
   const char *name;
-  int *flag;
+  grpc_tracer_flag *flag;
   struct tracer *next;
 } tracer;
 static tracer *tracers;
 
-void grpc_register_tracer(const char *name, int *flag) {
+#ifdef GRPC_THREADSAFE_TRACER
+#define TRACER_SET(flag, on) gpr_atm_no_barrier_store(&(flag).value, (on))
+#else
+#define TRACER_SET(flag, on) (flag).value = (on)
+#endif
+
+void grpc_register_tracer(const char *name, grpc_tracer_flag *flag) {
   tracer *t = gpr_malloc(sizeof(*t));
   t->name = name;
   t->flag = flag;
   t->next = tracers;
-  *flag = 0;
+  TRACER_SET(*flag, false);
   tracers = t;
 }
 
@@ -121,13 +127,13 @@ int grpc_tracer_set_enabled(const char *name, int enabled) {
   tracer *t;
   if (0 == strcmp(name, "all")) {
     for (t = tracers; t; t = t->next) {
-      *t->flag = enabled;
+      TRACER_SET(*t->flag, enabled);
     }
   } else {
     int found = 0;
     for (t = tracers; t; t = t->next) {
       if (0 == strcmp(name, t->name)) {
-        *t->flag = enabled;
+        TRACER_SET(*t->flag, enabled);
         found = 1;
       }
     }
diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h
index 7afc38db7e..ba432574d0 100644
--- a/src/core/lib/debug/trace.h
+++ b/src/core/lib/debug/trace.h
@@ -34,9 +34,35 @@
 #ifndef GRPC_CORE_LIB_DEBUG_TRACE_H
 #define GRPC_CORE_LIB_DEBUG_TRACE_H
 
+#include <grpc/support/atm.h>
 #include <grpc/support/port_platform.h>
+#include <stdbool.h>
 
-void grpc_register_tracer(const char *name, int *flag);
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+#define GRPC_THREADSAFE_TRACER
+#endif
+#endif
+
+typedef struct {
+#ifdef GRPC_THREADSAFE_TRACER
+  gpr_atm value;
+#else
+  bool value;
+#endif
+} grpc_tracer_flag;
+
+#ifdef GRPC_THREADSAFE_TRACER
+#define GRPC_TRACER_ON(flag) (gpr_atm_no_barrier_load(&(flag).value) != 0)
+#define GRPC_TRACER_INITIALIZER(on) \
+  { (gpr_atm)(on) }
+#else
+#define GRPC_TRACER_ON(flag) ((flag).value)
+#define GRPC_TRACER_INITIALIZER(on) \
+  { (on) }
+#endif
+
+void grpc_register_tracer(const char *name, grpc_tracer_flag *flag);
 void grpc_tracer_init(const char *env_var_name);
 void grpc_tracer_shutdown(void);
 
diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c
index aac506b800..a4357978e4 100644
--- a/src/core/lib/http/parser.c
+++ b/src/core/lib/http/parser.c
@@ -40,7 +40,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-int grpc_http1_trace = 0;
+grpc_tracer_flag grpc_http1_trace = GRPC_TRACER_INITIALIZER(false);
 
 static char *buf2str(void *buffer, size_t length) {
   char *out = gpr_malloc(length + 1);
@@ -308,7 +308,7 @@ static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte,
     case GRPC_HTTP_FIRST_LINE:
     case GRPC_HTTP_HEADERS:
       if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) {
-        if (grpc_http1_trace)
+        if (GRPC_TRACER_ON(grpc_http1_trace))
           gpr_log(GPR_ERROR, "HTTP header max line length (%d) exceeded",
                   GRPC_HTTP_PARSER_MAX_HEADER_LENGTH);
         return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h
index a68011dd43..a155fecf11 100644
--- a/src/core/lib/http/parser.h
+++ b/src/core/lib/http/parser.h
@@ -36,6 +36,7 @@
 
 #include <grpc/slice.h>
 #include <grpc/support/port_platform.h>
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/error.h"
 
 /* Maximum length of a header string of the form 'Key: Value\r\n' */
@@ -121,6 +122,6 @@ grpc_error *grpc_http_parser_eof(grpc_http_parser *parser);
 void grpc_http_request_destroy(grpc_http_request *request);
 void grpc_http_response_destroy(grpc_http_response *response);
 
-extern int grpc_http1_trace;
+extern grpc_tracer_flag grpc_http1_trace;
 
 #endif /* GRPC_CORE_LIB_HTTP_PARSER_H */
diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c
index 05cdbdad2b..863f22c614 100644
--- a/src/core/lib/iomgr/combiner.c
+++ b/src/core/lib/iomgr/combiner.c
@@ -42,13 +42,13 @@
 #include "src/core/lib/iomgr/workqueue.h"
 #include "src/core/lib/profiling/timers.h"
 
-int grpc_combiner_trace = 0;
+grpc_tracer_flag grpc_combiner_trace = GRPC_TRACER_INITIALIZER(false);
 
-#define GRPC_COMBINER_TRACE(fn) \
-  do {                          \
-    if (grpc_combiner_trace) {  \
-      fn;                       \
-    }                           \
+#define GRPC_COMBINER_TRACE(fn)                \
+  do {                                         \
+    if (GRPC_TRACER_ON(grpc_combiner_trace)) { \
+      fn;                                      \
+    }                                          \
   } while (0)
 
 #define STATE_UNORPHANED 1
diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h
index 75dcb0b70a..6ab7a2b26b 100644
--- a/src/core/lib/iomgr/combiner.h
+++ b/src/core/lib/iomgr/combiner.h
@@ -37,6 +37,7 @@
 #include <stddef.h>
 
 #include <grpc/support/atm.h>
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/support/mpscq.h"
 
@@ -78,6 +79,6 @@ grpc_closure_scheduler *grpc_combiner_finally_scheduler(grpc_combiner *lock,
 
 bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx);
 
-extern int grpc_combiner_trace;
+extern grpc_tracer_flag grpc_combiner_trace;
 
 #endif /* GRPC_CORE_LIB_IOMGR_COMBINER_H */
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c
index 65259912a9..8f453eb2d0 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.c
+++ b/src/core/lib/iomgr/ev_epollsig_linux.c
@@ -76,11 +76,6 @@ static int grpc_polling_trace = 0; /* Disabled by default */
 static int grpc_wakeup_signal = -1;
 static bool is_grpc_wakeup_signal_initialized = false;
 
-/* TODO: sreek: Right now, this wakes up all pollers. In future we should make
- * sure to wake up one polling thread (which can wake up other threads if
- * needed) */
-static grpc_wakeup_fd global_wakeup_fd;
-
 /* Implements the function defined in grpc_posix.h. This function might be
  * called before even calling grpc_init() to set either a different signal to
  * use. If signum == -1, then the use of signals is disabled */
@@ -454,8 +449,8 @@ static void polling_island_add_wakeup_fd_locked(polling_island *pi,
     gpr_asprintf(&err_msg,
                  "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with "
                  "error: %d (%s)",
-                 pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd),
-                 errno, strerror(errno));
+                 pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno,
+                 strerror(errno));
     append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
     gpr_free(err_msg);
   }
@@ -558,7 +553,6 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx,
     goto done;
   }
 
-  polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error);
   polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error);
 
   if (initial_fd != NULL) {
@@ -1116,11 +1110,10 @@ static grpc_error *pollset_global_init(void) {
   gpr_tls_init(&g_current_thread_pollset);
   gpr_tls_init(&g_current_thread_worker);
   poller_kick_init();
-  return grpc_wakeup_fd_init(&global_wakeup_fd);
+  return GRPC_ERROR_NONE;
 }
 
 static void pollset_global_shutdown(void) {
-  grpc_wakeup_fd_destroy(&global_wakeup_fd);
   gpr_tls_destroy(&g_current_thread_pollset);
   gpr_tls_destroy(&g_current_thread_worker);
 }
@@ -1226,10 +1219,6 @@ static grpc_error *pollset_kick(grpc_pollset *p,
   return error;
 }
 
-static grpc_error *kick_poller(void) {
-  return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
-}
-
 static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->po.mu);
   *mu = &pollset->po.mu;
@@ -1453,11 +1442,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
 
     for (int i = 0; i < ep_rv; ++i) {
       void *data_ptr = ep_ev[i].data.ptr;
-      if (data_ptr == &global_wakeup_fd) {
-        grpc_timer_consume_kick();
-        append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd),
-                     err_desc);
-      } else if (data_ptr == &pi->workqueue_wakeup_fd) {
+      if (data_ptr == &pi->workqueue_wakeup_fd) {
         append_error(error,
                      grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd),
                      err_desc);
@@ -1897,8 +1882,6 @@ static const grpc_event_engine_vtable vtable = {
     .pollset_set_add_fd = pollset_set_add_fd,
     .pollset_set_del_fd = pollset_set_del_fd,
 
-    .kick_poller = kick_poller,
-
     .workqueue_ref = workqueue_ref,
     .workqueue_unref = workqueue_unref,
     .workqueue_scheduler = workqueue_scheduler,
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index 69f322f7df..f070902ea4 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -122,8 +122,6 @@ struct grpc_fd {
   grpc_pollset *read_notifier_pollset;
 };
 
-static grpc_wakeup_fd global_wakeup_fd;
-
 /* Begin polling on an fd.
    Registers that the given pollset is interested in this fd - so that if read
    or writability interest changes, the pollset can be kicked to pick up that
@@ -784,19 +782,14 @@ static grpc_error *pollset_kick(grpc_pollset *p,
 static grpc_error *pollset_global_init(void) {
   gpr_tls_init(&g_current_thread_poller);
   gpr_tls_init(&g_current_thread_worker);
-  return grpc_wakeup_fd_init(&global_wakeup_fd);
+  return GRPC_ERROR_NONE;
 }
 
 static void pollset_global_shutdown(void) {
-  grpc_wakeup_fd_destroy(&global_wakeup_fd);
   gpr_tls_destroy(&g_current_thread_poller);
   gpr_tls_destroy(&g_current_thread_worker);
 }
 
-static grpc_error *kick_poller(void) {
-  return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
-}
-
 /* main interface */
 
 static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
@@ -952,13 +945,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
 
       fd_count = 0;
-      pfd_count = 2;
-      pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd);
+      pfd_count = 1;
+      pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd);
       pfds[0].events = POLLIN;
       pfds[0].revents = 0;
-      pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd);
-      pfds[1].events = POLLIN;
-      pfds[1].revents = 0;
       for (i = 0; i < pollset->fd_count; i++) {
         if (fd_is_orphaned(pollset->fds[i])) {
           GRPC_FD_UNREF(pollset->fds[i], "multipoller");
@@ -974,7 +964,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       pollset->fd_count = fd_count;
       gpr_mu_unlock(&pollset->mu);
 
-      for (i = 2; i < pfd_count; i++) {
+      for (i = 1; i < pfd_count; i++) {
         grpc_fd *fd = watchers[i].fd;
         pfds[i].events = (short)fd_begin_poll(fd, pollset, &worker, POLLIN,
                                               POLLOUT, &watchers[i]);
@@ -992,7 +982,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           work_combine_error(&error, GRPC_OS_ERROR(errno, "poll"));
         }
 
-        for (i = 2; i < pfd_count; i++) {
+        for (i = 1; i < pfd_count; i++) {
           if (watchers[i].fd == NULL) {
             fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
           } else {
@@ -1002,20 +992,15 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           }
         }
       } else if (r == 0) {
-        for (i = 2; i < pfd_count; i++) {
+        for (i = 1; i < pfd_count; i++) {
           fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else {
         if (pfds[0].revents & POLLIN_CHECK) {
-          grpc_timer_consume_kick();
-          work_combine_error(&error,
-                             grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd));
-        }
-        if (pfds[1].revents & POLLIN_CHECK) {
           work_combine_error(
               &error, grpc_wakeup_fd_consume_wakeup(&worker.wakeup_fd->fd));
         }
-        for (i = 2; i < pfd_count; i++) {
+        for (i = 1; i < pfd_count; i++) {
           if (watchers[i].fd == NULL) {
             fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
           } else {
@@ -1560,8 +1545,6 @@ static const grpc_event_engine_vtable vtable = {
     .pollset_set_add_fd = pollset_set_add_fd,
     .pollset_set_del_fd = pollset_set_del_fd,
 
-    .kick_poller = kick_poller,
-
     .workqueue_ref = workqueue_ref,
     .workqueue_unref = workqueue_unref,
     .workqueue_scheduler = workqueue_scheduler,
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index a436d649dd..4017a9e29b 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -264,8 +264,6 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
   g_event_engine->pollset_set_del_fd(exec_ctx, pollset_set, fd);
 }
 
-grpc_error *grpc_kick_poller(void) { return g_event_engine->kick_poller(); }
-
 #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG
 grpc_workqueue *grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file,
                                    int line, const char *reason) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index becc4d359e..a77720e61f 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -93,8 +93,6 @@ typedef struct grpc_event_engine_vtable {
   void (*pollset_set_del_fd)(grpc_exec_ctx *exec_ctx,
                              grpc_pollset_set *pollset_set, grpc_fd *fd);
 
-  grpc_error *(*kick_poller)(void);
-
   void (*shutdown_engine)(void);
 
 #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG
diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c
index 001e528409..37804d8600 100644
--- a/src/core/lib/iomgr/iomgr.c
+++ b/src/core/lib/iomgr/iomgr.c
@@ -47,6 +47,7 @@
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/iomgr/timer_manager.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 
@@ -67,6 +68,10 @@ void grpc_iomgr_init(void) {
   grpc_iomgr_platform_init();
 }
 
+void grpc_iomgr_start(void) {
+  grpc_timer_manager_init();
+}
+
 static size_t count_objects(void) {
   grpc_iomgr_object *obj;
   size_t n = 0;
@@ -88,6 +93,7 @@ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx) {
       gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN));
   gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME);
 
+  grpc_timer_manager_shutdown();
   grpc_iomgr_platform_flush();
 
   gpr_mu_lock(&g_mu);
diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h
index 245a1e08aa..6e2e023615 100644
--- a/src/core/lib/iomgr/iomgr.h
+++ b/src/core/lib/iomgr/iomgr.h
@@ -40,6 +40,9 @@
 /** Initializes the iomgr. */
 void grpc_iomgr_init(void);
 
+/** Starts any background threads for iomgr. */
+void grpc_iomgr_start(void);
+
 /** Signals the intention to shutdown the iomgr. Expects to be able to flush
  * exec_ctx. */
 void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx);
diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c
index 04c6b71747..bea4232273 100644
--- a/src/core/lib/iomgr/pollset_windows.c
+++ b/src/core/lib/iomgr/pollset_windows.c
@@ -227,6 +227,4 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p,
   return GRPC_ERROR_NONE;
 }
 
-void grpc_kick_poller(void) { grpc_iocp_kick(); }
-
 #endif /* GRPC_WINSOCK_SOCKET */
diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c
index c3ee878651..6b2b85cce0 100644
--- a/src/core/lib/iomgr/resource_quota.c
+++ b/src/core/lib/iomgr/resource_quota.c
@@ -44,7 +44,7 @@
 
 #include "src/core/lib/iomgr/combiner.h"
 
-int grpc_resource_quota_trace = 0;
+grpc_tracer_flag grpc_resource_quota_trace = GRPC_TRACER_INITIALIZER(false);
 
 #define MEMORY_USAGE_ESTIMATION_MAX 65536
 
@@ -307,13 +307,14 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx,
       resource_user->free_pool = 0;
       resource_quota->free_pool -= amt;
       rq_update_estimate(resource_quota);
-      if (grpc_resource_quota_trace) {
+      if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
         gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64
                            " bytes; rq_free_pool -> %" PRId64,
                 resource_quota->name, resource_user->name, amt,
                 resource_quota->free_pool);
       }
-    } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) {
+    } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) &&
+               resource_user->free_pool >= 0) {
       gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request",
               resource_quota->name, resource_user->name);
     }
@@ -342,7 +343,7 @@ static bool rq_reclaim_from_per_user_free_pool(
       resource_user->free_pool = 0;
       resource_quota->free_pool += amt;
       rq_update_estimate(resource_quota);
-      if (grpc_resource_quota_trace) {
+      if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
         gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64
                            " bytes; rq_free_pool -> %" PRId64,
                 resource_quota->name, resource_user->name, amt,
@@ -365,7 +366,7 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx,
                                  : GRPC_RULIST_RECLAIMER_BENIGN;
   grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list);
   if (resource_user == NULL) return false;
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation",
             resource_quota->name, resource_user->name,
             destructive ? "destructive" : "benign");
@@ -786,7 +787,7 @@ void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&resource_user->mu);
   ru_ref_by(resource_user, (gpr_atm)size);
   resource_user->free_pool -= (int64_t)size;
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; free_pool -> %" PRId64,
             resource_user->resource_quota->name, resource_user->name, size,
             resource_user->free_pool);
@@ -810,7 +811,7 @@ void grpc_resource_user_free(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&resource_user->mu);
   bool was_zero_or_negative = resource_user->free_pool <= 0;
   resource_user->free_pool += (int64_t)size;
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; free_pool -> %" PRId64,
             resource_user->resource_quota->name, resource_user->name, size,
             resource_user->free_pool);
@@ -839,7 +840,7 @@ void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx,
 
 void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx,
                                            grpc_resource_user *resource_user) {
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete",
             resource_user->resource_quota->name, resource_user->name);
   }
diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h
index 6f99be0d51..51122dad01 100644
--- a/src/core/lib/iomgr/resource_quota.h
+++ b/src/core/lib/iomgr/resource_quota.h
@@ -36,6 +36,7 @@
 
 #include <grpc/grpc.h>
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
 /** \file Tracks resource usage against a pool.
@@ -75,7 +76,7 @@
     maintain lists of users (which users arrange to leave before they are
     destroyed) */
 
-extern int grpc_resource_quota_trace;
+extern grpc_tracer_flag grpc_resource_quota_trace;
 
 grpc_resource_quota *grpc_resource_quota_ref_internal(
     grpc_resource_quota *resource_quota);
diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index a2692707d9..ed3fd94a98 100644
--- a/src/core/lib/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -58,7 +58,7 @@
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
 #include "src/core/lib/support/string.h"
 
-extern int grpc_tcp_trace;
+extern grpc_tracer_flag grpc_tcp_trace;
 
 typedef struct {
   gpr_mu mu;
@@ -114,7 +114,7 @@ done:
 static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   int done;
   async_connect *ac = acp;
-  if (grpc_tcp_trace) {
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     const char *str = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", ac->addr_str,
             str);
@@ -152,7 +152,7 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
 
   GRPC_ERROR_REF(error);
 
-  if (grpc_tcp_trace) {
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     const char *str = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: error=%s",
             ac->addr_str, str);
@@ -330,7 +330,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
                     grpc_schedule_on_exec_ctx);
   ac->channel_args = grpc_channel_args_copy(channel_args);
 
-  if (grpc_tcp_trace) {
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting",
             ac->addr_str);
   }
diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index 5f4b38de2b..5d360b0b80 100644
--- a/src/core/lib/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -74,7 +74,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type;
 typedef size_t msg_iovlen_type;
 #endif
 
-int grpc_tcp_trace = 0;
+grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false);
 
 typedef struct {
   grpc_endpoint base;
@@ -221,7 +221,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
                          grpc_error *error) {
   grpc_closure *cb = tcp->read_cb;
 
-  if (grpc_tcp_trace) {
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     size_t i;
     const char *str = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "read: error=%s", str);
@@ -468,14 +468,14 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
   }
 
   if (!tcp_flush(tcp, &error)) {
-    if (grpc_tcp_trace) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       gpr_log(GPR_DEBUG, "write: delayed");
     }
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
     cb = tcp->write_cb;
     tcp->write_cb = NULL;
-    if (grpc_tcp_trace) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       const char *str = grpc_error_string(error);
       gpr_log(GPR_DEBUG, "write: %s", str);
     }
@@ -490,7 +490,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   grpc_tcp *tcp = (grpc_tcp *)ep;
   grpc_error *error = GRPC_ERROR_NONE;
 
-  if (grpc_tcp_trace) {
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     size_t i;
 
     for (i = 0; i < buf->count; i++) {
@@ -521,12 +521,12 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   if (!tcp_flush(tcp, &error)) {
     TCP_REF(tcp, "write");
     tcp->write_cb = cb;
-    if (grpc_tcp_trace) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       gpr_log(GPR_DEBUG, "write: delayed");
     }
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
-    if (grpc_tcp_trace) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       const char *str = grpc_error_string(error);
       gpr_log(GPR_DEBUG, "write: %s", str);
     }
diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h
index 1ad5788331..4ad60c116e 100644
--- a/src/core/lib/iomgr/tcp_posix.h
+++ b/src/core/lib/iomgr/tcp_posix.h
@@ -44,10 +44,11 @@
    otherwise specified.
 */
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 
-extern int grpc_tcp_trace;
+extern grpc_tracer_flag grpc_tcp_trace;
 
 /* Create a tcp endpoint given a file desciptor and a read slice size.
    Takes ownership of fd. */
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index e66ffc9b1c..08997b5e2b 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -257,7 +257,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
     addr_str = grpc_sockaddr_to_uri(&addr);
     gpr_asprintf(&name, "tcp-server-connection:%s", addr_str);
 
-    if (grpc_tcp_trace) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       gpr_log(GPR_DEBUG, "SERVER_CONNECT: incoming connection: %s", addr_str);
     }
 
diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c
index d8e6068431..b28340b71c 100644
--- a/src/core/lib/iomgr/timer_generic.c
+++ b/src/core/lib/iomgr/timer_generic.c
@@ -56,8 +56,8 @@
 #define MIN_QUEUE_WINDOW_DURATION 0.01
 #define MAX_QUEUE_WINDOW_DURATION 1
 
-int grpc_timer_trace = 0;
-int grpc_timer_check_trace = 0;
+grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false);
 
 typedef struct {
   gpr_mu mu;
@@ -232,14 +232,13 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
   GPR_ASSERT(deadline.clock_type == g_clock_type);
   GPR_ASSERT(now.clock_type == g_clock_type);
   timer->closure = closure;
-  timer->deadline = timespec_to_atm_round_up(deadline);
+  gpr_atm deadline_atm = timer->deadline = timespec_to_atm_round_up(deadline);
 
-  if (grpc_timer_trace) {
+  if (GRPC_TRACER_ON(grpc_timer_trace)) {
     gpr_log(GPR_DEBUG, "TIMER %p: SET %" PRId64 ".%09d [%" PRIdPTR
                        "] now %" PRId64 ".%09d [%" PRIdPTR "] call %p[%p]",
-            timer, deadline.tv_sec, deadline.tv_nsec, timer->deadline,
-            now.tv_sec, now.tv_nsec, timespec_to_atm_round_down(now), closure,
-            closure->cb);
+            timer, deadline.tv_sec, deadline.tv_nsec, deadline_atm, now.tv_sec,
+            now.tv_nsec, timespec_to_atm_round_down(now), closure, closure->cb);
   }
 
   if (!g_shared_mutables.initialized) {
@@ -262,13 +261,13 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 
   grpc_time_averaged_stats_add_sample(&shard->stats,
                                       ts_to_dbl(gpr_time_sub(deadline, now)));
-  if (timer->deadline < shard->queue_deadline_cap) {
+  if (deadline_atm < shard->queue_deadline_cap) {
     is_first_timer = grpc_timer_heap_add(&shard->heap, timer);
   } else {
     timer->heap_index = INVALID_HEAP_INDEX;
     list_join(&shard->list, timer);
   }
-  if (grpc_timer_trace) {
+  if (GRPC_TRACER_ON(grpc_timer_trace)) {
     gpr_log(GPR_DEBUG, "  .. add to shard %d with queue_deadline_cap=%" PRIdPTR
                        " => is_first_timer=%s",
             (int)(shard - g_shards), shard->queue_deadline_cap,
@@ -289,16 +288,16 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
      grpc_timer_check. */
   if (is_first_timer) {
     gpr_mu_lock(&g_shared_mutables.mu);
-    if (grpc_timer_trace) {
+    if (GRPC_TRACER_ON(grpc_timer_trace)) {
       gpr_log(GPR_DEBUG, "  .. old shard min_deadline=%" PRIdPTR,
               shard->min_deadline);
     }
-    if (timer->deadline < shard->min_deadline) {
+    if (deadline_atm < shard->min_deadline) {
       gpr_atm old_min_deadline = g_shard_queue[0]->min_deadline;
-      shard->min_deadline = timer->deadline;
+      shard->min_deadline = deadline_atm;
       note_deadline_change(shard);
-      if (shard->shard_queue_index == 0 && timer->deadline < old_min_deadline) {
-        gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, timer->deadline);
+      if (shard->shard_queue_index == 0 && deadline_atm < old_min_deadline) {
+        gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, deadline_atm);
         grpc_kick_poller();
       }
     }
@@ -319,7 +318,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
 
   shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)];
   gpr_mu_lock(&shard->mu);
-  if (grpc_timer_trace) {
+  if (GRPC_TRACER_ON(grpc_timer_trace)) {
     gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer,
             timer->pending ? "true" : "false");
   }
@@ -355,7 +354,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) {
       saturating_add(GPR_MAX(now, shard->queue_deadline_cap),
                      (gpr_atm)(deadline_delta * 1000.0));
 
-  if (grpc_timer_check_trace) {
+  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
     gpr_log(GPR_DEBUG, "  .. shard[%d]->queue_deadline_cap --> %" PRIdPTR,
             (int)(shard - g_shards), shard->queue_deadline_cap);
   }
@@ -363,7 +362,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) {
     next = timer->next;
 
     if (timer->deadline < shard->queue_deadline_cap) {
-      if (grpc_timer_check_trace) {
+      if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
         gpr_log(GPR_DEBUG, "  .. add timer with deadline %" PRIdPTR " to heap",
                 timer->deadline);
       }
@@ -380,7 +379,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) {
 static grpc_timer *pop_one(shard_type *shard, gpr_atm now) {
   grpc_timer *timer;
   for (;;) {
-    if (grpc_timer_check_trace) {
+    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
       gpr_log(GPR_DEBUG, "  .. shard[%d]: heap_empty=%s",
               (int)(shard - g_shards),
               grpc_timer_heap_is_empty(&shard->heap) ? "true" : "false");
@@ -390,13 +389,13 @@ static grpc_timer *pop_one(shard_type *shard, gpr_atm now) {
       if (!refill_queue(shard, now)) return NULL;
     }
     timer = grpc_timer_heap_top(&shard->heap);
-    if (grpc_timer_check_trace) {
+    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
       gpr_log(GPR_DEBUG,
               "  .. check top timer deadline=%" PRIdPTR " now=%" PRIdPTR,
               timer->deadline, now);
     }
     if (timer->deadline > now) return NULL;
-    if (grpc_timer_trace) {
+    if (GRPC_TRACER_ON(grpc_timer_trace)) {
       gpr_log(GPR_DEBUG, "TIMER %p: FIRE %" PRIdPTR "ms late", timer,
               now - timer->deadline);
     }
@@ -436,7 +435,7 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now,
   if (gpr_spinlock_trylock(&g_shared_mutables.checker_mu)) {
     gpr_mu_lock(&g_shared_mutables.mu);
 
-    if (grpc_timer_check_trace) {
+    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
       gpr_log(GPR_DEBUG, "  .. shard[%d]->min_deadline = %" PRIdPTR,
               (int)(g_shard_queue[0] - g_shards),
               g_shard_queue[0]->min_deadline);
@@ -452,7 +451,7 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now,
       n +=
           pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline, error);
 
-      if (grpc_timer_check_trace) {
+      if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
         gpr_log(GPR_DEBUG, "  .. popped --> %" PRIdPTR
                            ", shard[%d]->min_deadline %" PRIdPTR
                            " --> %" PRIdPTR ", now=%" PRIdPTR,
@@ -509,7 +508,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
       *next =
           atm_to_timespec(GPR_MIN(timespec_to_atm_round_up(*next), min_timer));
     }
-    if (grpc_timer_check_trace) {
+    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
       gpr_log(GPR_DEBUG,
               "TIMER CHECK SKIP: now_atm=%" PRIdPTR " min_timer=%" PRIdPTR,
               now_atm, min_timer);
@@ -523,7 +522,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
           : GRPC_ERROR_CREATE_FROM_STATIC_STRING("Shutting down timer system");
 
   // tracing
-  if (grpc_timer_check_trace) {
+  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
     char *next_str;
     if (next == NULL) {
       next_str = gpr_strdup("NULL");
@@ -549,7 +548,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
     *next = atm_to_timespec(next_atm);
   }
   // tracing
-  if (grpc_timer_check_trace) {
+  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
     char *next_str;
     if (next == NULL) {
       next_str = gpr_strdup("NULL");
diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c
new file mode 100644
index 0000000000..1d83341299
--- /dev/null
+++ b/src/core/lib/iomgr/timer_manager.c
@@ -0,0 +1,198 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/timer_manager.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/thd.h>
+
+#include "src/core/lib/iomgr/timer.h"
+
+typedef struct completed_thread {
+  gpr_thd_id t;
+  struct completed_thread *next;
+} completed_thread;
+
+static gpr_mu g_mu;
+static bool g_threaded;
+static gpr_cv g_cv_wait;
+static gpr_cv g_cv_shutdown;
+static int g_thread_count;
+static int g_waiter_count;
+static completed_thread *g_completed_threads;
+static bool g_kicked;
+
+#define MAX_WAITERS 3
+
+static void timer_thread(void *unused);
+
+static void gc_completed_threads(void) {
+  if (g_completed_threads != NULL) {
+    completed_thread *to_gc = g_completed_threads;
+    g_completed_threads = NULL;
+    gpr_mu_unlock(&g_mu);
+    while (to_gc != NULL) {
+      gpr_thd_join(to_gc->t);
+      completed_thread *next = to_gc->next;
+      gpr_free(to_gc);
+      to_gc = next;
+    }
+    gpr_mu_lock(&g_mu);
+  }
+}
+
+static void start_timer_thread_and_unlock(void) {
+  ++g_waiter_count;
+  ++g_thread_count;
+  gpr_mu_unlock(&g_mu);
+  gpr_log(GPR_DEBUG, "Spawn timer thread");
+  gpr_thd_id thd;
+  gpr_thd_options opt = gpr_thd_options_default();
+  gpr_thd_options_set_joinable(&opt);
+  gpr_thd_new(&thd, timer_thread, NULL, &opt);
+}
+
+void grpc_timer_manager_tick() {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+  grpc_timer_check(&exec_ctx, now, &next);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void timer_thread(void *unused) {
+  grpc_exec_ctx exec_ctx =
+      GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
+  for (;;) {
+    gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+    gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+    if (grpc_timer_check(&exec_ctx, now, &next)) {
+      gpr_mu_lock(&g_mu);
+      --g_waiter_count;
+      bool start_thread = g_waiter_count == 0;
+      if (start_thread && g_threaded) {
+        start_timer_thread_and_unlock();
+      } else {
+        gpr_mu_unlock(&g_mu);
+      }
+      grpc_exec_ctx_flush(&exec_ctx);
+      gpr_mu_lock(&g_mu);
+      gc_completed_threads();
+      ++g_waiter_count;
+      gpr_mu_unlock(&g_mu);
+    } else {
+      gpr_mu_lock(&g_mu);
+      if (!g_threaded) break;
+      if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) {
+        if (g_kicked) {
+          grpc_timer_consume_kick();
+          g_kicked = false;
+        } else if (g_waiter_count > MAX_WAITERS) {
+          break;
+        }
+      }
+      gpr_mu_unlock(&g_mu);
+    }
+  }
+  --g_waiter_count;
+  --g_thread_count;
+  if (0 == g_thread_count) {
+    gpr_cv_signal(&g_cv_shutdown);
+  }
+  completed_thread *ct = gpr_malloc(sizeof(*ct));
+  ct->t = gpr_thd_currentid();
+  ct->next = g_completed_threads;
+  g_completed_threads = ct;
+  gpr_mu_unlock(&g_mu);
+  gpr_log(GPR_DEBUG, "End timer thread");
+}
+
+static void start_threads(void) {
+  gpr_mu_lock(&g_mu);
+  if (!g_threaded) {
+    g_threaded = true;
+    start_timer_thread_and_unlock();
+  } else {
+    g_threaded = false;
+    gpr_mu_unlock(&g_mu);
+  }
+}
+
+void grpc_timer_manager_init(void) {
+  gpr_mu_init(&g_mu);
+  gpr_cv_init(&g_cv_wait);
+  gpr_cv_init(&g_cv_shutdown);
+  g_threaded = false;
+  g_thread_count = 0;
+  g_waiter_count = 0;
+  g_completed_threads = NULL;
+
+  start_threads();
+}
+
+static void stop_threads(void) {
+  gpr_mu_lock(&g_mu);
+  if (g_threaded) {
+    g_threaded = false;
+    gpr_cv_broadcast(&g_cv_wait);
+    while (g_thread_count > 0) {
+      gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME));
+      gc_completed_threads();
+    }
+  }
+  gpr_mu_unlock(&g_mu);
+}
+
+void grpc_timer_manager_shutdown(void) {
+  stop_threads();
+
+  gpr_mu_destroy(&g_mu);
+  gpr_cv_destroy(&g_cv_wait);
+  gpr_cv_destroy(&g_cv_shutdown);
+}
+
+void grpc_timer_manager_set_threading(bool threaded) {
+  if (threaded) {
+    start_threads();
+  } else {
+    stop_threads();
+  }
+}
+
+void grpc_kick_poller(void) {
+  gpr_mu_lock(&g_mu);
+  g_kicked = true;
+  gpr_cv_signal(&g_cv_wait);
+  gpr_mu_unlock(&g_mu);
+}
diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h
new file mode 100644
index 0000000000..0b21262b1a
--- /dev/null
+++ b/src/core/lib/iomgr/timer_manager.h
@@ -0,0 +1,52 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H
+#define GRPC_CORE_IOMGR_TIMER_MANAGER_H
+
+#include <stdbool.h>
+
+/* Timer Manager tries to keep one thread waiting for the next timeout at all
+   times */
+
+void grpc_timer_manager_init(void);
+void grpc_timer_manager_shutdown(void);
+
+/* enable/disable threading - must be called after grpc_timer_manager_init and
+ * before grpc_timer_manager_shutdown */
+void grpc_timer_manager_set_threading(bool enabled);
+/* explicitly perform one tick of the timer system - for when threading is
+ * disabled */
+void grpc_timer_manager_tick(void);
+
+#endif
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
index 178ce89aa6..0e7c1afb02 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.c
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -171,7 +171,7 @@ static char *redact_private_key(const char *json_key) {
 
 grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
     const char *json_key, gpr_timespec token_lifetime, void *reserved) {
-  if (grpc_api_trace) {
+  if (GRPC_TRACER_ON(grpc_api_trace)) {
     char *clean_json = redact_private_key(json_key);
     gpr_log(GPR_INFO,
             "grpc_service_account_jwt_access_credentials_create("
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
index ccfb3566c1..29235b6eb3 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -412,7 +412,7 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create(
     const char *json_refresh_token, void *reserved) {
   grpc_auth_refresh_token token =
       grpc_auth_refresh_token_create_from_string(json_refresh_token);
-  if (grpc_api_trace) {
+  if (GRPC_TRACER_ON(grpc_api_trace)) {
     char *loggable_token = create_loggable_refresh_token(&token);
     gpr_log(GPR_INFO,
             "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index 1f0daf7325..dff05633ec 100644
--- a/src/core/lib/security/transport/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -253,7 +253,7 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
   grpc_linked_mdelem *l;
   grpc_client_security_context *sec_ctx = NULL;
 
-  if (calld->security_context_set == 0 && !op->cancel_stream) {
+  if (!op->cancel_stream && calld->security_context_set == 0) {
     calld->security_context_set = 1;
     GPR_ASSERT(op->payload->context != NULL);
     if (op->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) {
diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
index 0d5c7432c6..48d368a2a7 100644
--- a/src/core/lib/security/transport/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -75,7 +75,7 @@ typedef struct {
   gpr_refcount ref;
 } secure_endpoint;
 
-int grpc_trace_secure_endpoint = 0;
+grpc_tracer_flag grpc_trace_secure_endpoint = GRPC_TRACER_INITIALIZER(false);
 
 static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) {
   secure_endpoint *ep = secure_ep;
@@ -137,7 +137,7 @@ static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur,
 
 static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep,
                          grpc_error *error) {
-  if (grpc_trace_secure_endpoint) {
+  if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) {
     size_t i;
     for (i = 0; i < ep->read_buffer->count; i++) {
       char *data = grpc_dump_slice(ep->read_buffer->slices[i],
@@ -269,7 +269,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
 
   grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &ep->output_buffer);
 
-  if (grpc_trace_secure_endpoint) {
+  if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) {
     for (i = 0; i < slices->count; i++) {
       char *data =
           grpc_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
diff --git a/src/core/lib/security/transport/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h
index a61f40a4fa..f1a5c8cb6d 100644
--- a/src/core/lib/security/transport/secure_endpoint.h
+++ b/src/core/lib/security/transport/secure_endpoint.h
@@ -39,7 +39,7 @@
 
 struct tsi_frame_protector;
 
-extern int grpc_trace_secure_endpoint;
+extern grpc_tracer_flag grpc_trace_secure_endpoint;
 
 /* Takes ownership of protector and to_wrap, and refs leftover_slices. */
 grpc_endpoint *grpc_secure_endpoint_create(
diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c
index 79e3e5ca9b..55c69cd5b0 100644
--- a/src/core/lib/surface/api_trace.c
+++ b/src/core/lib/surface/api_trace.c
@@ -31,6 +31,7 @@
  *
  */
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/surface/api_trace.h"
 
-int grpc_api_trace = 0;
+grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false);
diff --git a/src/core/lib/surface/api_trace.h b/src/core/lib/surface/api_trace.h
index c60aaba5e9..d4fbc8d90d 100644
--- a/src/core/lib/surface/api_trace.h
+++ b/src/core/lib/surface/api_trace.h
@@ -37,7 +37,7 @@
 #include <grpc/support/log.h>
 #include "src/core/lib/debug/trace.h"
 
-extern int grpc_api_trace;
+extern grpc_tracer_flag grpc_api_trace;
 
 /* Provide unwrapping macros because we're in C89 and variadic macros weren't
    introduced until C99... */
@@ -58,7 +58,7 @@ extern int grpc_api_trace;
 /* Due to the limitations of C89's preprocessor, the arity of the var-arg list
    'nargs' must be specified. */
 #define GRPC_API_TRACE(fmt, nargs, args)                      \
-  if (grpc_api_trace) {                                       \
+  if (GRPC_TRACER_ON(grpc_api_trace)) {                       \
     gpr_log(GPR_INFO, fmt GRPC_API_TRACE_UNWRAP##nargs args); \
   }
 
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index 7525806583..1bd158f614 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -244,8 +244,8 @@ struct grpc_call {
   void *saved_receiving_stream_ready_bctlp;
 };
 
-int grpc_call_error_trace = 0;
-int grpc_compression_trace = 0;
+grpc_tracer_flag grpc_call_error_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_compression_trace = GRPC_TRACER_INITIALIZER(false);
 
 #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1))
 #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1)
@@ -702,7 +702,7 @@ static void get_final_status(grpc_call *call,
   for (i = 0; i < STATUS_SOURCE_COUNT; i++) {
     status[i] = unpack_received_status(gpr_atm_acq_load(&call->status[i]));
   }
-  if (grpc_call_error_trace) {
+  if (GRPC_TRACER_ON(grpc_call_error_trace)) {
     gpr_log(GPR_DEBUG, "get_final_status %s", call->is_client ? "CLI" : "SVR");
     for (i = 0; i < STATUS_SOURCE_COUNT; i++) {
       if (status[i].is_set) {
@@ -1259,7 +1259,7 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
   }
 
   if (error != GRPC_ERROR_NONE) {
-    if (grpc_trace_operation_failures) {
+    if (GRPC_TRACER_ON(grpc_trace_operation_failures)) {
       GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
     }
     grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
@@ -1355,8 +1355,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(call->encodings_accepted_by_peer != 0);
   if (!GPR_BITGET(call->encodings_accepted_by_peer,
                   call->incoming_compression_algorithm)) {
-    extern int grpc_compression_trace;
-    if (grpc_compression_trace) {
+    if (GRPC_TRACER_ON(grpc_compression_trace)) {
       char *algo_name = NULL;
       grpc_compression_algorithm_name(call->incoming_compression_algorithm,
                                       &algo_name);
diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h
index 7d4d0db28d..256a5fa2fe 100644
--- a/src/core/lib/surface/call.h
+++ b/src/core/lib/surface/call.h
@@ -117,7 +117,8 @@ void grpc_call_context_set(grpc_call *call, grpc_context_index elem,
 void *grpc_call_context_get(grpc_call *call, grpc_context_index elem);
 
 #define GRPC_CALL_LOG_BATCH(sev, call, ops, nops, tag) \
-  if (grpc_api_trace) grpc_call_log_batch(sev, call, ops, nops, tag)
+  if (GRPC_TRACER_ON(grpc_api_trace))                  \
+  grpc_call_log_batch(sev, call, ops, nops, tag)
 
 uint8_t grpc_call_is_client(grpc_call *call);
 
@@ -126,7 +127,8 @@ uint8_t grpc_call_is_client(grpc_call *call);
 grpc_compression_algorithm grpc_call_compression_for_level(
     grpc_call *call, grpc_compression_level level);
 
-extern int grpc_call_error_trace;
+extern grpc_tracer_flag grpc_call_error_trace;
+extern grpc_tracer_flag grpc_compression_trace;
 
 #ifdef __cplusplus
 }
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index eae3f103b1..f5ce96f62d 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -50,9 +50,9 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/event_string.h"
 
-int grpc_trace_operation_failures;
+grpc_tracer_flag grpc_trace_operation_failures = GRPC_TRACER_INITIALIZER(false);
 #ifndef NDEBUG
-int grpc_trace_pending_tags;
+grpc_tracer_flag grpc_trace_pending_tags = GRPC_TRACER_INITIALIZER(false);
 #endif
 
 typedef struct {
@@ -242,15 +242,16 @@ struct grpc_completion_queue {
 #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
 #define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
 
-int grpc_cq_pluck_trace;
-int grpc_cq_event_timeout_trace;
+grpc_tracer_flag grpc_cq_pluck_trace = GRPC_TRACER_INITIALIZER(true);
+grpc_tracer_flag grpc_cq_event_timeout_trace = GRPC_TRACER_INITIALIZER(true);
 
-#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)                  \
-  if (grpc_api_trace &&                                               \
-      (grpc_cq_pluck_trace || (event)->type != GRPC_QUEUE_TIMEOUT)) { \
-    char *_ev = grpc_event_string(event);                             \
-    gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev);               \
-    gpr_free(_ev);                                                    \
+#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)    \
+  if (GRPC_TRACER_ON(grpc_api_trace) &&                 \
+      (GRPC_TRACER_ON(grpc_cq_pluck_trace) ||           \
+       (event)->type != GRPC_QUEUE_TIMEOUT)) {          \
+    char *_ev = grpc_event_string(event);               \
+    gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \
+    gpr_free(_ev);                                      \
   }
 
 static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc,
@@ -375,14 +376,16 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
 #endif
 
   GPR_TIMER_BEGIN("grpc_cq_end_op", 0);
-  if (grpc_api_trace ||
-      (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) {
+  if (GRPC_TRACER_ON(grpc_api_trace) ||
+      (GRPC_TRACER_ON(grpc_trace_operation_failures) &&
+       error != GRPC_ERROR_NONE)) {
     const char *errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
         "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, "
         "done_arg=%p, storage=%p)",
         7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage));
-    if (grpc_trace_operation_failures && error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_trace_operation_failures) &&
+        error != GRPC_ERROR_NONE) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
     }
   }
@@ -481,7 +484,7 @@ static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) {
 
 #ifndef NDEBUG
 static void dump_pending_tags(grpc_completion_queue *cc) {
-  if (!grpc_trace_pending_tags) return;
+  if (!GRPC_TRACER_ON(grpc_trace_pending_tags)) return;
 
   gpr_strvec v;
   gpr_strvec_init(&v);
@@ -580,31 +583,18 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
       dump_pending_tags(cc);
       break;
     }
-    /* Check alarms - these are a global resource so we just ping
-       each time through on every pollset.
-       May update deadline to ensure timely wakeups.
-       TODO(ctiller): can this work be localized? */
-    gpr_timespec iteration_deadline = deadline;
-    if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) {
-      GPR_TIMER_MARK("alarm_triggered", 0);
+    grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc),
+                                              NULL, now, deadline);
+    if (err != GRPC_ERROR_NONE) {
       gpr_mu_unlock(cc->mu);
-      grpc_exec_ctx_flush(&exec_ctx);
-      gpr_mu_lock(cc->mu);
-      continue;
-    } else {
-      grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc),
-                                                NULL, now, iteration_deadline);
-      if (err != GRPC_ERROR_NONE) {
-        gpr_mu_unlock(cc->mu);
-        const char *msg = grpc_error_string(err);
-        gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
+      const char *msg = grpc_error_string(err);
+      gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
 
-        GRPC_ERROR_UNREF(err);
-        memset(&ret, 0, sizeof(ret));
-        ret.type = GRPC_QUEUE_TIMEOUT;
-        dump_pending_tags(cc);
-        break;
-      }
+      GRPC_ERROR_UNREF(err);
+      memset(&ret, 0, sizeof(ret));
+      ret.type = GRPC_QUEUE_TIMEOUT;
+      dump_pending_tags(cc);
+      break;
     }
     is_finished_arg.first_loop = false;
   }
@@ -690,7 +680,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
     abort();
   }
 
-  if (grpc_cq_pluck_trace) {
+  if (GRPC_TRACER_ON(grpc_cq_pluck_trace)) {
     GRPC_API_TRACE(
         "grpc_completion_queue_pluck("
         "cc=%p, tag=%p, "
@@ -773,31 +763,19 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
       dump_pending_tags(cc);
       break;
     }
-    /* Check alarms - these are a global resource so we just ping
-       each time through on every pollset.
-       May update deadline to ensure timely wakeups.
-       TODO(ctiller): can this work be localized? */
-    gpr_timespec iteration_deadline = deadline;
-    if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) {
-      GPR_TIMER_MARK("alarm_triggered", 0);
+    grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc),
+                                              &worker, now, deadline);
+    if (err != GRPC_ERROR_NONE) {
+      del_plucker(cc, tag, &worker);
       gpr_mu_unlock(cc->mu);
-      grpc_exec_ctx_flush(&exec_ctx);
-      gpr_mu_lock(cc->mu);
-    } else {
-      grpc_error *err = cc->poller_vtable->work(
-          &exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline);
-      if (err != GRPC_ERROR_NONE) {
-        del_plucker(cc, tag, &worker);
-        gpr_mu_unlock(cc->mu);
-        const char *msg = grpc_error_string(err);
-        gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
+      const char *msg = grpc_error_string(err);
+      gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
 
-        GRPC_ERROR_UNREF(err);
-        memset(&ret, 0, sizeof(ret));
-        ret.type = GRPC_QUEUE_TIMEOUT;
-        dump_pending_tags(cc);
-        break;
-      }
+      GRPC_ERROR_UNREF(err);
+      memset(&ret, 0, sizeof(ret));
+      ret.type = GRPC_QUEUE_TIMEOUT;
+      dump_pending_tags(cc);
+      break;
     }
     is_finished_arg.first_loop = false;
     del_plucker(cc, tag, &worker);
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index a932087939..a750687b27 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -37,15 +37,16 @@
 /* Internal API for completion queues */
 
 #include <grpc/grpc.h>
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/pollset.h"
 
 /* These trace flags default to 1. The corresponding lines are only traced
    if grpc_api_trace is also truthy */
-extern int grpc_cq_pluck_trace;
-extern int grpc_cq_event_timeout_trace;
-extern int grpc_trace_operation_failures;
+extern grpc_tracer_flag grpc_cq_pluck_trace;
+extern grpc_tracer_flag grpc_cq_event_timeout_trace;
+extern grpc_tracer_flag grpc_trace_operation_failures;
 #ifndef NDEBUG
-extern int grpc_trace_pending_tags;
+extern grpc_tracer_flag grpc_trace_pending_tags;
 #endif
 
 typedef struct grpc_cq_completion {
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
index 4b381b1954..6163776152 100644
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@@ -145,10 +145,8 @@ void grpc_init(void) {
     grpc_register_tracer("server_channel", &grpc_server_channel_trace);
     grpc_register_tracer("bdp_estimator", &grpc_bdp_estimator_trace);
     // Default pluck trace to 1
-    grpc_cq_pluck_trace = 1;
     grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace);
     // Default timeout trace to 1
-    grpc_cq_event_timeout_trace = 1;
     grpc_register_tracer("op_failure", &grpc_trace_operation_failures);
     grpc_register_tracer("resource_quota", &grpc_resource_quota_trace);
     grpc_register_tracer("call_error", &grpc_call_error_trace);
@@ -173,6 +171,7 @@ void grpc_init(void) {
     grpc_tracer_init("GRPC_TRACE");
     /* no more changes to channel init pipelines */
     grpc_channel_init_finalize();
+    grpc_iomgr_start();
   }
   gpr_mu_unlock(&g_init_mu);
   GRPC_API_TRACE("grpc_init(void)", 0, ());
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index 934ca0431a..795429ed1e 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -73,7 +73,7 @@ typedef struct registered_method registered_method;
 
 typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
 
-int grpc_server_channel_trace = 0;
+grpc_tracer_flag grpc_server_channel_trace = GRPC_TRACER_INITIALIZER(false);
 
 typedef struct requested_call {
   requested_call_type type;
@@ -456,7 +456,7 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand,
   grpc_closure_init(&chand->finish_destroy_channel_closure,
                     finish_destroy_channel, chand, grpc_schedule_on_exec_ctx);
 
-  if (grpc_server_channel_trace && error != GRPC_ERROR_NONE) {
+  if (GRPC_TRACER_ON(grpc_server_channel_trace) && error != GRPC_ERROR_NONE) {
     const char *msg = grpc_error_string(error);
     gpr_log(GPR_INFO, "Disconnected client: %s", msg);
   }
diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h
index a85d9f4964..cd2fca0fe0 100644
--- a/src/core/lib/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -36,12 +36,13 @@
 
 #include <grpc/grpc.h>
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/transport/transport.h"
 
 extern const grpc_channel_filter grpc_server_top_filter;
 
 /** Lightweight tracing of server channel state */
-extern int grpc_server_channel_trace;
+extern grpc_tracer_flag grpc_server_channel_trace;
 
 /* Add a listener to the server: when the server starts, it will call start,
    and when it shuts down, it will call destroy */
diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c
index e1483677fd..eaf9caaa64 100644
--- a/src/core/lib/transport/bdp_estimator.c
+++ b/src/core/lib/transport/bdp_estimator.c
@@ -38,7 +38,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-int grpc_bdp_estimator_trace = 0;
+grpc_tracer_flag grpc_bdp_estimator_trace = GRPC_TRACER_INITIALIZER(false);
 
 void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) {
   estimator->estimate = 65536;
@@ -67,7 +67,7 @@ bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator,
 }
 
 void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) {
-  if (grpc_bdp_estimator_trace) {
+  if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) {
     gpr_log(GPR_DEBUG, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64,
             estimator->name, estimator->accumulator, estimator->estimate);
   }
@@ -77,7 +77,7 @@ void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) {
 }
 
 void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) {
-  if (grpc_bdp_estimator_trace) {
+  if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) {
     gpr_log(GPR_DEBUG, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64,
             estimator->name, estimator->accumulator, estimator->estimate);
   }
@@ -87,14 +87,14 @@ void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) {
 }
 
 void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator) {
-  if (grpc_bdp_estimator_trace) {
+  if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) {
     gpr_log(GPR_DEBUG, "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64,
             estimator->name, estimator->accumulator, estimator->estimate);
   }
   GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_STARTED);
   if (estimator->accumulator > 2 * estimator->estimate / 3) {
     estimator->estimate *= 2;
-    if (grpc_bdp_estimator_trace) {
+    if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_DEBUG, "bdp[%s]: estimate increased to %" PRId64,
               estimator->name, estimator->estimate);
     }
diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h
index df8d1f6fc0..1b125d8d61 100644
--- a/src/core/lib/transport/bdp_estimator.h
+++ b/src/core/lib/transport/bdp_estimator.h
@@ -36,11 +36,12 @@
 
 #include <stdbool.h>
 #include <stdint.h>
+#include "src/core/lib/debug/trace.h"
 
 #define GRPC_BDP_SAMPLES 16
 #define GRPC_BDP_MIN_SAMPLES_FOR_ESTIMATE 3
 
-extern int grpc_bdp_estimator_trace;
+extern grpc_tracer_flag grpc_bdp_estimator_trace;
 
 typedef enum {
   GRPC_BDP_PING_UNSCHEDULED,
diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c
index 3757b25267..e30cd523fa 100644
--- a/src/core/lib/transport/connectivity_state.c
+++ b/src/core/lib/transport/connectivity_state.c
@@ -39,7 +39,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-int grpc_connectivity_state_trace = 0;
+grpc_tracer_flag grpc_connectivity_state_trace = GRPC_TRACER_INITIALIZER(false);
 
 const char *grpc_connectivity_state_name(grpc_connectivity_state state) {
   switch (state) {
@@ -94,7 +94,7 @@ grpc_connectivity_state grpc_connectivity_state_check(
   grpc_connectivity_state cur =
       (grpc_connectivity_state)gpr_atm_no_barrier_load(
           &tracker->current_state_atm);
-  if (grpc_connectivity_state_trace) {
+  if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
     gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
             grpc_connectivity_state_name(cur));
   }
@@ -106,7 +106,7 @@ grpc_connectivity_state grpc_connectivity_state_get(
   grpc_connectivity_state cur =
       (grpc_connectivity_state)gpr_atm_no_barrier_load(
           &tracker->current_state_atm);
-  if (grpc_connectivity_state_trace) {
+  if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
     gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
             grpc_connectivity_state_name(cur));
   }
@@ -127,7 +127,7 @@ bool grpc_connectivity_state_notify_on_state_change(
   grpc_connectivity_state cur =
       (grpc_connectivity_state)gpr_atm_no_barrier_load(
           &tracker->current_state_atm);
-  if (grpc_connectivity_state_trace) {
+  if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
     if (current == NULL) {
       gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker,
               tracker->name, notify);
@@ -180,7 +180,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
       (grpc_connectivity_state)gpr_atm_no_barrier_load(
           &tracker->current_state_atm);
   grpc_connectivity_state_watcher *w;
-  if (grpc_connectivity_state_trace) {
+  if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
     const char *error_string = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker,
             tracker->name, grpc_connectivity_state_name(cur),
@@ -208,7 +208,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
   while ((w = tracker->watchers) != NULL) {
     *w->current = state;
     tracker->watchers = w->next;
-    if (grpc_connectivity_state_trace) {
+    if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
       gpr_log(GPR_DEBUG, "NOTIFY: %p %s: %p", tracker, tracker->name,
               w->notify);
     }
diff --git a/src/core/lib/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h
index c9604c34dd..cdc2930c11 100644
--- a/src/core/lib/transport/connectivity_state.h
+++ b/src/core/lib/transport/connectivity_state.h
@@ -35,6 +35,7 @@
 #define GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H
 
 #include <grpc/grpc.h>
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef struct grpc_connectivity_state_watcher {
@@ -57,7 +58,7 @@ typedef struct {
   char *name;
 } grpc_connectivity_state_tracker;
 
-extern int grpc_connectivity_state_trace;
+extern grpc_tracer_flag grpc_connectivity_state_trace;
 
 /** enum --> string conversion */
 const char *grpc_connectivity_state_name(grpc_connectivity_state state);