From dd86b69b8b8d7092198249acdef50ede1e7bfac5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 10:43:11 -0700 Subject: Stubbed out new polling engine --- src/core/lib/iomgr/ev_epollex_linux.c | 635 ++++++++++++++++++++++++++++++++++ src/core/lib/iomgr/ev_epollex_linux.h | 42 +++ src/core/lib/iomgr/ev_posix.c | 2 + 3 files changed, 679 insertions(+) create mode 100644 src/core/lib/iomgr/ev_epollex_linux.c create mode 100644 src/core/lib/iomgr/ev_epollex_linux.h (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c new file mode 100644 index 0000000000..71e8b7e4fb --- /dev/null +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -0,0 +1,635 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* Uncomment the following to enable extra checks on poll_object operations */ +/* #define PO_DEBUG */ + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +/******************************************************************************* + * Fd Declarations + */ + +#define FD_FROM_PO(po) ((grpc_fd *)(po)) + +struct grpc_fd { + gpr_mu mu; + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +/******************************************************************************* + * Pollset Declarations + */ +struct grpc_pollset_worker { + /* Thread id of this worker */ + pthread_t pt_id; + + /* Used to prevent a worker from getting kicked multiple times */ + gpr_atm is_kicked; + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset {}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set {}; + +/******************************************************************************* + * Common helpers + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + abort(); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + abort(); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + abort(); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + abort(); + } +} +#endif + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + // grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + abort(); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + abort(); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->mu); + } + + /* Note: It is not really needed to get the new_fd->mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->mu); + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + + gpr_mu_lock(&fd->mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +static void poller_kick_init() {} + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + poller_kick_init(); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + abort(); +} + +static grpc_error *kick_poller(void) { abort(); } + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + abort(); +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) {} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) {} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + abort(); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + abort(); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + abort(); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + abort(); +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + abort(); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + abort(); +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + abort(); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + abort(); +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epollex_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epollex_available()) { + return NULL; + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ + +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.h b/src/core/lib/iomgr/ev_epollex_linux.h new file mode 100644 index 0000000000..4d7cd713df --- /dev/null +++ b/src/core/lib/iomgr/ev_epollex_linux.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epollex_linux(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index b5be5504b9..2663152aec 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,6 +45,7 @@ #include #include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -65,6 +66,7 @@ typedef struct { } event_engine_factory; static const event_engine_factory g_factories[] = { + {"epollex", grpc_init_epollex_linux}, {"epoll", grpc_init_epoll_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, -- cgit v1.2.3 From e24b24d3c646c40248ea4581f3c5b03597797544 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:05:45 -0700 Subject: Implement pollset for epollex --- src/core/lib/iomgr/ev_epoll_linux.c | 2 + src/core/lib/iomgr/ev_epollex_linux.c | 279 ++++++++++++++++++++++++++++++---- src/core/lib/iomgr/ev_poll_posix.c | 2 + src/core/lib/iomgr/pollset.h | 5 +- src/core/lib/iomgr/pollset_windows.c | 2 + 5 files changed, 254 insertions(+), 36 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 6db8e1a77c..d41c164d71 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -63,6 +63,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ static int grpc_polling_trace = 0; /* Disabled by default */ #define GRPC_POLLING_TRACE(fmt, ...) \ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 71e8b7e4fb..0985755a43 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -62,8 +62,9 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -/* Uncomment the following to enable extra checks on poll_object operations */ -/* #define PO_DEBUG */ +#ifndef EPOLLEXCLUSIVE +#define EPOLLEXCLUSIVE (1u << 28) +#endif /* TODO: sreek: Right now, this wakes up all pollers. In future we should make * sure to wake up one polling thread (which can wake up other threads if @@ -85,6 +86,8 @@ struct grpc_fd { Ref/Unref by two to avoid altering the orphaned bit */ gpr_atm refst; + grpc_wakeup_fd workqueue_wakeup_fd; + /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer valid */ @@ -131,16 +134,22 @@ static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { * Pollset Declarations */ struct grpc_pollset_worker { - /* Thread id of this worker */ - pthread_t pt_id; - - /* Used to prevent a worker from getting kicked multiple times */ - gpr_atm is_kicked; - struct grpc_pollset_worker *next; - struct grpc_pollset_worker *prev; + bool kicked; + bool initialized_cv; + gpr_cv cv; + grpc_pollset_worker *next; + grpc_pollset_worker *prev; }; -struct grpc_pollset {}; +struct grpc_pollset { + gpr_mu mu; + int epfd; + int num_pollers; + gpr_atm shutdown_atm; + grpc_closure *shutdown_closure; + grpc_wakeup_fd pollset_wakeup; + grpc_pollset_worker *root_worker; +}; /******************************************************************************* * Pollset-set Declarations @@ -151,6 +160,16 @@ struct grpc_pollset_set {}; * Common helpers */ +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, @@ -400,13 +419,10 @@ static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); -static void poller_kick_init() {} - /* Global state management */ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - poller_kick_init(); return grpc_wakeup_fd_init(&global_wakeup_fd); } @@ -419,12 +435,41 @@ static void pollset_global_shutdown(void) { /* p->mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { - abort(); + if (specific_worker == NULL) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + } else { + return GRPC_ERROR_NONE; + } + } else if (gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + return GRPC_ERROR_NONE; + } else if (specific_worker == p->root_worker) { + return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + } else { + gpr_cv_signal(&specific_worker->cv); + return GRPC_ERROR_NONE; + } } -static grpc_error *kick_poller(void) { abort(); } +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} -static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {} +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + pollset->epfd = epoll_create1(EPOLL_CLOEXEC); + if (pollset->epfd < 0) { + GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); + } + pollset->num_pollers = 0; + gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); + pollset->shutdown_closure = NULL; + GRPC_LOG_IF_ERROR("pollset_init", + grpc_wakeup_fd_init(&pollset->pollset_wakeup)); + pollset->root_worker = NULL; + *mu = &pollset->mu; +} /* Convert a timespec to milliseconds: - Very small or negative poll times are clamped to zero to do a non-blocking @@ -469,33 +514,186 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset) { - abort(); -} - /* pollset->po.mu lock must be held by the caller before calling this */ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_closure *closure) {} + grpc_closure *closure) { + GPR_ASSERT(pollset->shutdown_closure == NULL); + pollset->shutdown_closure = closure; + if (pollset->num_pollers > 0) { + struct epoll_event ev = {.events = EPOLLIN, + .data.ptr = &pollset->pollset_wakeup}; + epoll_ctl(pollset->epfd, EPOLL_CTL_MOD, pollset->pollset_wakeup.read_fd, + &ev); + GRPC_LOG_IF_ERROR("pollset_shutdown", + grpc_wakeup_fd_wakeup(&pollset->pollset_wakeup)); + } else { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); + } +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. */ +static void pollset_destroy(grpc_pollset *pollset) { + gpr_mu_destroy(&pollset->mu); + if (pollset->epfd >= 0) close(pollset->epfd); + grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); +} -/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other - * than destroying the mutexes, there is nothing special that needs to be done - * here */ -static void pollset_destroy(grpc_pollset *pollset) {} +#define MAX_EPOLL_EVENTS 100 -/* pollset->po.mu lock must be held by the caller before calling this. - The function pollset_work() may temporarily release the lock (pollset->po.mu) +static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { + struct epoll_event events[MAX_EPOLL_EVENTS]; + static const char *err_desc = "pollset_poll"; + + if (pollset->epfd < 0) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "epoll fd failed to initialize"); + } + + int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, + poll_deadline_to_millis_timeout(deadline, now)); + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + + grpc_error *error = GRPC_ERROR_NONE; + for (int i = 0; i < r; i++) { + void *data_ptr = events[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pollset->pollset_wakeup) { + /* once we start shutting down we stop consuming the wakeup: + the fd is level triggered and non-exclusive, which should result in all + pollers waking */ + if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } + } else { + grpc_fd *fd = (grpc_fd *)(((intptr_t)data_ptr) & ~(intptr_t)1); + bool is_workqueue = (((intptr_t)data_ptr) & 1) != 0; + bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; + bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; + bool write_ev = (events[i].events & EPOLLOUT) != 0; + if (is_workqueue) { + append_error(&error, + grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), + err_desc); + fd_invoke_workqueue(exec_ctx, fd); + } else { + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + } + + return error; +} + +/* Return true if this thread should poll */ +static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl, + gpr_timespec deadline) { + if (worker_hdl != NULL) { + *worker_hdl = worker; + worker->kicked = false; + if (pollset->root_worker == NULL) { + pollset->root_worker = worker; + worker->next = worker->prev = worker; + worker->initialized_cv = false; + } else { + worker->next = pollset->root_worker; + worker->prev = worker->next->prev; + worker->next->prev = worker->prev->next = worker; + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (pollset->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline)) return false; + if (worker->kicked) return false; + } + } + } + return pollset->shutdown_closure == NULL; +} + +static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl) { + if (worker_hdl != NULL) { + if (worker == pollset->root_worker) { + if (worker == worker->next) { + pollset->root_worker = NULL; + } else { + pollset->root_worker = worker->next; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; + } + } else { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + } +} + +/* pollset->mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->mu) during the course of its execution but it will always re-acquire the lock and ensure that it is held by the time the function returns */ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { - abort(); + grpc_pollset_worker worker; + grpc_error *error = GRPC_ERROR_NONE; + if (begin_worker(pollset, &worker, worker_hdl, deadline)) { + GPR_ASSERT(!pollset->shutdown_closure); + pollset->num_pollers++; + gpr_mu_unlock(&pollset->mu); + error = pollset_poll(exec_ctx, pollset, now, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + pollset->num_pollers--; + if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + } + } + end_worker(pollset, &worker, worker_hdl); + return error; } static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - abort(); + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_add_fd"; + struct epoll_event ev_fd = { + .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { + switch (errno) { + case EEXIST: /* if this fd is already in the epoll set, the workqueue fd + must also be - just return */ + return; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, + .data.ptr = fd}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, + &ev_wq) != 0) { + switch (errno) { + case EEXIST: /* if the workqueue fd is already in the epoll set we're ok - + no need to do anything special */ + break; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + GRPC_LOG_IF_ERROR("pollset_add_fd", error); } /******************************************************************************* @@ -593,15 +791,32 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epollex_available() { +static bool is_epollex_available(void) { int fd = epoll_create1(EPOLL_CLOEXEC); if (fd < 0) { gpr_log( GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epoll polling engine", + "epoll_create1 failed with error: %d. Not using epollex polling engine", fd); return false; } + grpc_wakeup_fd wakeup; + if (!GRPC_LOG_IF_ERROR("check_wakeupfd_for_epollex", + grpc_wakeup_fd_init(&wakeup))) { + return false; + } + struct epoll_event ev = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, + .data.ptr = NULL}; + if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE failed with error: %d. Not using " + "epollex polling engine", + fd); + close(fd); + grpc_wakeup_fd_destroy(&wakeup); + return false; + } + grpc_wakeup_fd_destroy(&wakeup); close(fd); return true; } diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 9834cdd197..b13ec2cbc1 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -58,6 +58,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + /******************************************************************************* * FD declarations */ diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index 9bf3cdac89..6f3a51e717 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -40,8 +40,6 @@ #include "src/core/lib/iomgr/exec_ctx.h" -#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) - /* A grpc_pollset is a set of file descriptors that a higher level item is interested in. For example: - a server will typically keep a pollset containing all connected channels, @@ -88,8 +86,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec deadline) GRPC_MUST_USE_RESULT; /* Break one polling thread out of polling work for this pollset. - If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers. - Otherwise, if specific_worker is non-NULL, then kick that worker. */ + If specific_worker is non-NULL, then kick that worker. */ grpc_error *grpc_pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 04c6b71747..6dca37b481 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -43,6 +43,8 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_windows.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + gpr_mu grpc_polling_mu; static grpc_pollset_worker *g_active_poller; static grpc_pollset_worker g_global_root_worker; -- cgit v1.2.3 From a74808604a03682108fc4a6d72384f1f5216849c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:13:41 -0700 Subject: Fix pollset_init to also add the wakeup fd --- src/core/lib/iomgr/ev_epollex_linux.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0985755a43..839807c246 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -465,8 +465,16 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->num_pollers = 0; gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); pollset->shutdown_closure = NULL; - GRPC_LOG_IF_ERROR("pollset_init", - grpc_wakeup_fd_init(&pollset->pollset_wakeup)); + if (GRPC_LOG_IF_ERROR("pollset_init", + grpc_wakeup_fd_init(&pollset->pollset_wakeup)) && + pollset->epfd >= 0) { + struct epoll_event ev = {.events = EPOLLIN | EPOLLET, + .data.ptr = &pollset->pollset_wakeup}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, pollset->pollset_wakeup.read_fd, + &ev) != 0) { + GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); + } + } pollset->root_worker = NULL; *mu = &pollset->mu; } -- cgit v1.2.3 From 12f0a582fb28f5a7838aff00328d6729ea458670 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:14:54 -0700 Subject: Add global wakeup to pollset --- src/core/lib/iomgr/ev_epollex_linux.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 839807c246..8d3e3bd562 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -461,6 +461,13 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); + } else { + struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, + &ev) != 0) { + GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); + } } pollset->num_pollers = 0; gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); -- cgit v1.2.3 From 9fae6f9cc2939522912eeed3057af461b487ca90 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:25:28 -0700 Subject: Starting to flesh out fd --- src/core/lib/iomgr/ev_epollex_linux.c | 107 ++++++++++++++++------------------ 1 file changed, 49 insertions(+), 58 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 8d3e3bd562..b3390c602e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -87,6 +87,7 @@ struct grpc_fd { gpr_atm refst; grpc_wakeup_fd workqueue_wakeup_fd; + grpc_closure_scheduler workqueue_scheduler; /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer @@ -106,21 +107,6 @@ struct grpc_fd { grpc_iomgr_object iomgr_object; }; -/* Reference counting for fds */ -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd *fd); -static void fd_unref(grpc_fd *fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - static void fd_global_init(void); static void fd_global_shutdown(void); @@ -170,49 +156,6 @@ static bool append_error(grpc_error **composite, grpc_error *error, return false; } -#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, - const char *file, int line, - const char *reason) { - if (workqueue != NULL) { - abort(); - } - return workqueue; -} - -static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, - const char *file, int line, const char *reason) { - if (workqueue != NULL) { - abort(); - } -} -#else -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { - if (workqueue != NULL) { - abort(); - } - return workqueue; -} - -static void workqueue_unref(grpc_exec_ctx *exec_ctx, - grpc_workqueue *workqueue) { - if (workqueue != NULL) { - abort(); - } -} -#endif - -static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, - grpc_error *error) { - GPR_TIMER_BEGIN("workqueue.enqueue", 0); - // grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; - abort(); -} - -static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - abort(); -} - /******************************************************************************* * Fd Definitions */ @@ -323,6 +266,10 @@ static grpc_fd *fd_create(int fd, const char *name) { grpc_lfev_init(&new_fd->write_closure); gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + GRPC_LOG_IF_ERROR("fd_create", + grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); + new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -413,6 +360,50 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + ref_by((grpc_fd *)workqueue, 2, file, line, reason); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + unref_by((grpc_fd *)workqueue, 2, file, line, reason); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + ref_by((grpc_fd *)workqueue, 2); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + unref_by((grpc_fd *)workqueue, 2); + } +} +#endif + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - + offsetof(grpc_fd, workqueue_scheduler)); + abort(); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + return &((grpc_fd *)workqueue)->workqueue_scheduler; +} + /******************************************************************************* * Pollset Definitions */ -- cgit v1.2.3 From 0b4c9011846c9f9640eabefcbecc9d8736b3ccc0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 17:19:37 -0700 Subject: fd-workqueue-melding --- src/core/lib/iomgr/ev_epollex_linux.c | 50 ++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b3390c602e..608cda4a3b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -61,6 +61,7 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/spinlock.h" #ifndef EPOLLEXCLUSIVE #define EPOLLEXCLUSIVE (1u << 28) @@ -86,8 +87,16 @@ struct grpc_fd { Ref/Unref by two to avoid altering the orphaned bit */ gpr_atm refst; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; grpc_closure_scheduler workqueue_scheduler; + /* Spinlock guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_spinlock workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer @@ -269,6 +278,9 @@ static grpc_fd *fd_create(int fd, const char *name) { GRPC_LOG_IF_ERROR("fd_create", grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + new_fd->workqueue_read_mu = GPR_SPINLOCK_INIT; + gpr_mpscq_init(&new_fd->workqueue_items); + gpr_atm_no_barrier_store(&new_fd->workqueue_item_count); new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -392,12 +404,46 @@ static void workqueue_unref(grpc_exec_ctx *exec_ctx, } #endif +static void workqueue_wakeup(grpc_fd *fd) { + GRPC_LOG_IF_ERROR("workqueue_enqueue", + grpc_wakeup_fd_wakeup(&fd->workqueue_wakeup_fd)); +} + static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error) { GPR_TIMER_BEGIN("workqueue.enqueue", 0); grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - offsetof(grpc_fd, workqueue_scheduler)); - abort(); + REF_BY(fd, 2); + gpr_atm last = gpr_atm_no_barrier_fetch_add(&fd->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&fd->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_wakeup(fd); + } + UNREF_BY(fd, 2); +} + +static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + /* handle spurious wakeups */ + if (!gpr_spinlock_trylock(&fd->workqueue_read_mu)) return; + gpr_mpscq_node *n = gpr_mpscq_pop(&fd->workqueue_items); + gpr_spinlock_unlock(&fd->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_wakeup(fd); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_wakeup(fd); + } } static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { @@ -556,8 +602,10 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, "epoll fd failed to initialize"); } + GRPC_SCHEDULING_START_BLOCKING_REGION; int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, poll_deadline_to_millis_timeout(deadline, now)); + GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); grpc_error *error = GRPC_ERROR_NONE; -- cgit v1.2.3 From 40664c7189f2599168edb28aa49f6cd7a6e9bb40 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 08:19:17 -0700 Subject: fix workqueue --- src/core/lib/iomgr/ev_epollex_linux.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 608cda4a3b..dd3c52ae32 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -278,9 +278,9 @@ static grpc_fd *fd_create(int fd, const char *name) { GRPC_LOG_IF_ERROR("fd_create", grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - new_fd->workqueue_read_mu = GPR_SPINLOCK_INIT; + new_fd->workqueue_read_mu = GPR_SPINLOCK_INITIALIZER; gpr_mpscq_init(&new_fd->workqueue_items); - gpr_atm_no_barrier_store(&new_fd->workqueue_item_count); + gpr_atm_no_barrier_store(&new_fd->workqueue_item_count, 0); new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -414,14 +414,14 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, GPR_TIMER_BEGIN("workqueue.enqueue", 0); grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - offsetof(grpc_fd, workqueue_scheduler)); - REF_BY(fd, 2); + REF_BY(fd, 2, "workqueue_enqueue"); gpr_atm last = gpr_atm_no_barrier_fetch_add(&fd->workqueue_item_count, 1); closure->error_data.error = error; gpr_mpscq_push(&fd->workqueue_items, &closure->next_data.atm_next); if (last == 0) { workqueue_wakeup(fd); } - UNREF_BY(fd, 2); + UNREF_BY(fd, 2, "workqueue_enqueue"); } static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { @@ -430,15 +430,14 @@ static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { gpr_mpscq_node *n = gpr_mpscq_pop(&fd->workqueue_items); gpr_spinlock_unlock(&fd->workqueue_read_mu); if (n != NULL) { - if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + if (gpr_atm_full_fetch_add(&fd->workqueue_item_count, -1) > 1) { workqueue_wakeup(fd); } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); - return true; - } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + } else if (gpr_atm_no_barrier_load(&fd->workqueue_item_count) > 0) { /* n == NULL might mean there's work but it's not available to be popped * yet - try to ensure another workqueue wakes up to check shortly if so */ -- cgit v1.2.3 From f4360d77d45b589f3f3820c1d3e23d9f1161846e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 08:51:00 -0700 Subject: Fix build, fix epollex detection --- CMakeLists.txt | 7 +++++ Makefile | 7 +++++ binding.gyp | 1 + build.yaml | 2 ++ config.m4 | 1 + gRPC-Core.podspec | 3 +++ grpc.gemspec | 2 ++ package.xml | 2 ++ src/core/lib/iomgr/ev_epollex_linux.c | 30 +++++++++++++++++----- src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.c++.internal | 2 ++ tools/doxygen/Doxyfile.core.internal | 2 ++ tools/run_tests/generated/sources_and_headers.json | 3 +++ vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 +++ vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 6 +++++ .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 +++ .../grpc++_unsecure.vcxproj.filters | 6 +++++ vsprojects/vcxproj/grpc/grpc.vcxproj | 3 +++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 +++++ .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 3 +++ .../grpc_test_util/grpc_test_util.vcxproj.filters | 6 +++++ .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 3 +++ .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 6 +++++ 23 files changed, 102 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/CMakeLists.txt b/CMakeLists.txt index ca2b259842..aa21ae6b2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -933,6 +933,7 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1256,6 +1257,7 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1565,6 +1567,7 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1826,6 +1829,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2243,6 +2247,7 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2574,6 +2579,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -3273,6 +3279,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c diff --git a/Makefile b/Makefile index 1eeae6071e..525a78937f 100644 --- a/Makefile +++ b/Makefile @@ -2835,6 +2835,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3156,6 +3157,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3464,6 +3466,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3697,6 +3700,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4091,6 +4095,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4430,6 +4435,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -5121,6 +5127,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/binding.gyp b/binding.gyp index 3df6d0aa2f..c1a9f733ea 100644 --- a/binding.gyp +++ b/binding.gyp @@ -675,6 +675,7 @@ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/build.yaml b/build.yaml index c8098c51e5..d258f91ac9 100644 --- a/build.yaml +++ b/build.yaml @@ -199,6 +199,7 @@ filegroups: - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h - src/core/lib/iomgr/ev_epoll_linux.h + - src/core/lib/iomgr/ev_epollex_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -311,6 +312,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + - src/core/lib/iomgr/ev_epollex_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c - src/core/lib/iomgr/exec_ctx.c diff --git a/config.m4 b/config.m4 index 9470e43aaa..e43c0296f8 100644 --- a/config.m4 +++ b/config.m4 @@ -109,6 +109,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2e260189fb..a84c12a999 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -281,6 +281,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epollex_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -485,6 +486,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', @@ -733,6 +735,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epollex_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/grpc.gemspec b/grpc.gemspec index 1165a91b6f..4054a49cc3 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -197,6 +197,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epollex_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -401,6 +402,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epollex_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) diff --git a/package.xml b/package.xml index c6b86a66a2..3910ade1bd 100644 --- a/package.xml +++ b/package.xml @@ -206,6 +206,7 @@ + @@ -410,6 +411,7 @@ + diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index dd3c52ae32..5045697f60 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -328,6 +328,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; + if (!is_fd_closed) { + gpr_log(GPR_DEBUG, "TODO: handle fd removal?"); + } + /* Remove the active status but keep referenced. We want this grpc_fd struct to be alive (and not added to freelist) until the end of this function */ REF_BY(fd, 1, reason); @@ -858,13 +862,27 @@ static bool is_epollex_available(void) { grpc_wakeup_fd_init(&wakeup))) { return false; } - struct epoll_event ev = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = NULL}; + struct epoll_event ev = { + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + .events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT, + .data.ptr = NULL}; if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { + if (errno != EINVAL) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + close(fd); + grpc_wakeup_fd_destroy(&wakeup); + return false; + } + } else { gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE failed with error: %d. Not using " - "epollex polling engine", - fd); + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); close(fd); grpc_wakeup_fd_destroy(&wakeup); return false; @@ -897,7 +915,7 @@ const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 0d0a5fb088..b74204337b 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -103,6 +103,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index afab2296a0..b600de2fcb 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -952,6 +952,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epollex_linux.c \ +src/core/lib/iomgr/ev_epollex_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ee1fd1b242..97b0d8114e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1073,6 +1073,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epollex_linux.c \ +src/core/lib/iomgr/ev_epollex_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fa62d0aaac..6f5f119131 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7540,6 +7540,7 @@ "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epollex_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -7692,6 +7693,8 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epollex_linux.c", + "src/core/lib/iomgr/ev_epollex_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index cae44c5e23..bf798a02a8 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -403,6 +403,7 @@ + @@ -628,6 +629,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 36da089470..ee7447aa26 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -202,6 +202,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -944,6 +947,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 4d284006e6..2991de4f6f 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -397,6 +397,7 @@ + @@ -612,6 +613,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 9d641553a2..e3fb5cfe98 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -187,6 +187,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -911,6 +914,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index dd6fdd861e..13f745c6da 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -326,6 +326,7 @@ + @@ -559,6 +560,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 51c9b7201d..7b02f2ce30 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -82,6 +82,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -878,6 +881,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index b1db7e5d15..3a87cdf311 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -221,6 +221,7 @@ + @@ -400,6 +401,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index a7ecbea87c..bead542aaa 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -139,6 +139,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -656,6 +659,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 8614b11589..47aec42cce 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -316,6 +316,7 @@ + @@ -527,6 +528,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index e7aa9fb8d6..6ecbf24ed2 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -85,6 +85,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -791,6 +794,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr -- cgit v1.2.3 From ef814ca29ae27a1c9b8193b185c82bd958b68778 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 08:55:03 -0700 Subject: Sentences finish with a full stop --- src/core/lib/iomgr/ev_epollex_linux.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 5045697f60..e846263a4b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -851,10 +851,10 @@ static const grpc_event_engine_vtable vtable = { static bool is_epollex_available(void) { int fd = epoll_create1(EPOLL_CLOEXEC); if (fd < 0) { - gpr_log( - GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epollex polling engine", - fd); + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); return false; } grpc_wakeup_fd wakeup; -- cgit v1.2.3 From 8fc1ca1e3f3fe3cfb6040405f764a150a95110ba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 13:01:48 -0700 Subject: Initial pollset_set implementation --- src/core/lib/iomgr/ev_epollex_linux.c | 298 ++++++++++++++++++++++++++++++---- 1 file changed, 267 insertions(+), 31 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index e846263a4b..6d79b82982 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -73,13 +73,30 @@ static grpc_wakeup_fd global_wakeup_fd; /******************************************************************************* - * Fd Declarations + * Pollset-set sibling link */ -#define FD_FROM_PO(po) ((grpc_fd *)(po)) +typedef enum { + PSS_FD, + PSS_POLLSET, + PSS_POLLSET_SET, + PSS_OBJ_TYPE_COUNT +} pss_obj_type; -struct grpc_fd { +typedef struct pss_obj { gpr_mu mu; + struct pss_obj *pss_next; + struct pss_obj *pss_prev; + int pss_refs; + grpc_pollset_set *pss_master; +} pss_obj; + +/******************************************************************************* + * Fd Declarations + */ + +struct grpc_fd { + pss_obj po; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -137,19 +154,31 @@ struct grpc_pollset_worker { }; struct grpc_pollset { - gpr_mu mu; + pss_obj po; int epfd; int num_pollers; gpr_atm shutdown_atm; grpc_closure *shutdown_closure; grpc_wakeup_fd pollset_wakeup; grpc_pollset_worker *root_worker; + + grpc_pollset *pss_next; + grpc_pollset *pss_prev; + int pss_refs; + grpc_pollset_set *pss_master; }; /******************************************************************************* * Pollset-set Declarations */ -struct grpc_pollset_set {}; +struct grpc_pollset_set { + pss_obj po; + gpr_refcount refs; + grpc_pollset_set *master; + + /* roots are only used if master == self */ + pss_obj *roots[PSS_OBJ_TYPE_COUNT]; +}; /******************************************************************************* * Common helpers @@ -242,7 +271,7 @@ static void fd_global_shutdown(void) { while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->mu); + gpr_mu_destroy(&fd->po.mu); gpr_free(fd); } gpr_mu_destroy(&fd_freelist_mu); @@ -260,13 +289,13 @@ static grpc_fd *fd_create(int fd, const char *name) { if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->mu); + gpr_mu_init(&new_fd->po.mu); } - /* Note: It is not really needed to get the new_fd->mu lock here. If this + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this * is a newly created fd (or an fd we got from the freelist), no one else * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->mu); + gpr_mu_lock(&new_fd->po.mu); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -285,7 +314,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; - gpr_mu_unlock(&new_fd->mu); + gpr_mu_unlock(&new_fd->po.mu); char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); @@ -299,11 +328,11 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { int ret_fd = -1; - gpr_mu_lock(&fd->mu); + gpr_mu_lock(&fd->po.mu); if (!fd->orphaned) { ret_fd = fd->fd; } - gpr_mu_unlock(&fd->mu); + gpr_mu_unlock(&fd->po.mu); return ret_fd; } @@ -314,7 +343,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->mu); + gpr_mu_lock(&fd->po.mu); fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file @@ -338,7 +367,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); - gpr_mu_unlock(&fd->mu); + gpr_mu_unlock(&fd->po.mu); UNREF_BY(fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -472,7 +501,7 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } -/* p->mu must be held before calling this function */ +/* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker == NULL) { @@ -497,7 +526,7 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - gpr_mu_init(&pollset->mu); + gpr_mu_init(&pollset->po.mu); pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); @@ -523,7 +552,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { } } pollset->root_worker = NULL; - *mu = &pollset->mu; + *mu = &pollset->po.mu; } /* Convert a timespec to milliseconds: @@ -588,7 +617,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->mu); + gpr_mu_destroy(&pollset->po.mu); if (pollset->epfd >= 0) close(pollset->epfd); grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); } @@ -669,7 +698,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = true; gpr_cv_init(&worker->cv); while (pollset->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline)) return false; + if (gpr_cv_wait(&worker->cv, &pollset->po.mu, deadline)) return false; if (worker->kicked) return false; } } @@ -698,8 +727,8 @@ static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } } -/* pollset->mu lock must be held by the caller before calling this. - The function pollset_work() may temporarily release the lock (pollset->mu) +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) during the course of its execution but it will always re-acquire the lock and ensure that it is held by the time the function returns */ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -710,10 +739,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (begin_worker(pollset, &worker, worker_hdl, deadline)) { GPR_ASSERT(!pollset->shutdown_closure); pollset->num_pollers++; - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(&pollset->po.mu); error = pollset_poll(exec_ctx, pollset, now, deadline); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(&pollset->po.mu); pollset->num_pollers--; if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); @@ -758,45 +787,252 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, */ static grpc_pollset_set *pollset_set_create(void) { - grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); + gpr_mu_init(&pss->po.mu); + pss->roots[PSS_POLLSET_SET] = &pss->po; + pss->po.pss_next = pss->po.pss_prev = &pss->po; + return pss; +} + +static void pss_destroy(grpc_pollset_set *pss) { + gpr_mu_destroy(&pss->po.mu); + gpr_free(pss); +} + +static grpc_pollset_set *pss_ref(grpc_pollset_set *pss) { + gpr_ref(&pss->refs); return pss; } +static void pss_unref(grpc_pollset_set *pss) { + if (gpr_unref(&pss->refs)) pss_destroy(pss); +} + static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss) { - gpr_free(pss); + pss_unref(pss); +} + +static grpc_pollset_set *pss_ref_and_lock_master( + grpc_pollset_set *master_or_slave) { + pss_ref(master_or_slave); + gpr_mu_lock(&master_or_slave->po.mu); + while (master_or_slave != master_or_slave->master) { + grpc_pollset_set *master = pss_ref(master_or_slave->master); + gpr_mu_unlock(&master_or_slave->po.mu); + pss_unref(master_or_slave); + master_or_slave = master; + gpr_mu_lock(&master_or_slave->po.mu); + } + return master_or_slave; +} + +static void pss_broadcast_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *dst, + pss_obj *obj) { + grpc_fd *fd = (grpc_fd *)obj; + if (dst->roots[PSS_POLLSET] == NULL) return; + pss_obj *tgt = dst->roots[PSS_POLLSET]; + do { + pollset_add_fd(exec_ctx, (grpc_pollset *)tgt, fd); + tgt = tgt->pss_next; + } while (tgt != dst->roots[PSS_POLLSET]); +} + +static void pss_broadcast_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *dst, pss_obj *obj) { + grpc_pollset *pollset = (grpc_pollset *)obj; + if (dst->roots[PSS_FD] == NULL) return; + pss_obj *tgt = dst->roots[PSS_FD]; + do { + pollset_add_fd(exec_ctx, pollset, (grpc_fd *)tgt); + tgt = tgt->pss_next; + } while (tgt != dst->roots[PSS_FD]); +} + +static pss_obj *pss_splice(pss_obj *p, pss_obj *q) { + if (p == NULL) return q; + if (q == NULL) return p; + p->pss_next->pss_prev = q->pss_prev; + q->pss_prev->pss_next = p->pss_next; + p->pss_next = q; + q->pss_prev = p; + return p; +} + +static void (*const broadcast[PSS_OBJ_TYPE_COUNT])(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *dst, + pss_obj *obj) = { + pss_broadcast_fd, pss_broadcast_pollset, NULL}; + +static void pss_merge_broadcast_and_patch(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *a, + grpc_pollset_set *b, + pss_obj_type type) { + pss_obj *obj; + if (a->roots[type] != NULL) { + obj = a->roots[PSS_FD]; + do { + broadcast[type](exec_ctx, b, obj); + obj = obj->pss_next; + } while (obj != a->roots[PSS_FD]); + } + if (b->roots[type] != NULL) { + obj = b->roots[PSS_FD]; + do { + broadcast[type](exec_ctx, a, obj); + gpr_mu_lock(&obj->mu); + obj->pss_master = a; + gpr_mu_unlock(&obj->mu); + obj = obj->pss_next; + } while (obj != b->roots[PSS_FD]); + } + a->roots[type] = pss_splice(a->roots[type], b->roots[type]); +} + +static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, + grpc_pollset_set *b) { + pss_ref(a); + pss_ref(b); + bool changed; + for (;;) { + if (a == b) { + pss_unref(a); + pss_unref(b); + return; + } else if (a < b) { + gpr_mu_lock(&a->po.mu); + gpr_mu_lock(&b->po.mu); + } else { + gpr_mu_lock(&b->po.mu); + gpr_mu_lock(&a->po.mu); + } + changed = false; + if (a != a->master) { + grpc_pollset_set *master = pss_ref(a->master); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pss_unref(a); + a = master; + changed = true; + } else if (b != b->master) { + grpc_pollset_set *master = pss_ref(b->master); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pss_unref(b); + b = master; + changed = true; + } else { + /* a, b locked and are at their respective masters */ + pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); + pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); + b->po.pss_master = a; + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + } + } +} + +static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + pss_obj *obj, pss_obj_type type) { + pss = pss_ref_and_lock_master(pss); + gpr_mu_lock(&obj->mu); + if (obj->pss_master == pss) { + /* obj is already a member -- just bump refcount */ + obj->pss_refs++; + gpr_mu_unlock(&obj->mu); + gpr_mu_unlock(&pss->po.mu); + pss_unref(pss); + return; + } else if (obj->pss_master != NULL) { + grpc_pollset_set *other_pss = pss_ref(obj->pss_master); + obj->pss_refs++; + gpr_mu_unlock(&obj->mu); + gpr_mu_unlock(&pss->po.mu); + pss_merge(exec_ctx, pss, other_pss); + pss_unref(other_pss); + pss_unref(pss); + } else { + GPR_ASSERT(obj->pss_refs == 0); + obj->pss_refs = 1; + obj->pss_master = pss; + if (pss->roots[type] == NULL) { + pss->roots[type] = obj; + obj->pss_next = obj->pss_prev = obj; + } else { + obj->pss_next = pss->roots[type]; + obj->pss_prev = obj->pss_next->pss_prev; + obj->pss_prev->pss_next = obj; + obj->pss_next->pss_prev = obj; + } + gpr_mu_unlock(&obj->mu); + switch (type) { + case PSS_FD: + pss_broadcast_fd(exec_ctx, pss, obj); + break; + case PSS_POLLSET: + pss_broadcast_pollset(exec_ctx, pss, obj); + break; + case PSS_POLLSET_SET: + case PSS_OBJ_TYPE_COUNT: + GPR_UNREACHABLE_CODE(break); + } + gpr_mu_unlock(&pss->po.mu); + pss_unref(pss); + } +} + +static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + pss_obj *obj, pss_obj_type type) { + pss = pss_ref_and_lock_master(pss); + gpr_mu_lock(&obj->mu); + obj->pss_refs--; + if (obj->pss_refs == 0) { + obj->pss_master = NULL; + if (obj == pss->roots[type]) { + pss->roots[type] = obj->pss_next; + } + if (obj->pss_next == obj) { + pss->roots[type] = NULL; + } else { + obj->pss_next->pss_prev = obj->pss_prev; + obj->pss_prev->pss_next = obj->pss_next; + } + } + gpr_mu_unlock(&obj->mu); + gpr_mu_unlock(&pss->po.mu); + pss_unref(pss); } static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - abort(); + pss_add_obj(exec_ctx, pss, &fd->po, PSS_FD); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - abort(); + pss_del_obj(exec_ctx, pss, &fd->po, PSS_FD); } static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - abort(); + pss_add_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - abort(); + pss_del_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); } static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, grpc_pollset_set *item) { - abort(); + pss_add_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); } static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, grpc_pollset_set *item) { - abort(); + pss_del_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); } /******************************************************************************* -- cgit v1.2.3 From bcf8db50b90574dc83836990106b484e0f48a195 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 20:41:40 +0000 Subject: Fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 20 +++++++++----------- test/core/slice/slice_buffer_test.c | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 6d79b82982..23ca26933a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -174,7 +174,6 @@ struct grpc_pollset { struct grpc_pollset_set { pss_obj po; gpr_refcount refs; - grpc_pollset_set *master; /* roots are only used if master == self */ pss_obj *roots[PSS_OBJ_TYPE_COUNT]; @@ -789,7 +788,9 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static grpc_pollset_set *pollset_set_create(void) { grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); gpr_mu_init(&pss->po.mu); + gpr_ref_init(&pss->refs, 1); pss->roots[PSS_POLLSET_SET] = &pss->po; + pss->po.pss_master = pss; pss->po.pss_next = pss->po.pss_prev = &pss->po; return pss; } @@ -817,8 +818,8 @@ static grpc_pollset_set *pss_ref_and_lock_master( grpc_pollset_set *master_or_slave) { pss_ref(master_or_slave); gpr_mu_lock(&master_or_slave->po.mu); - while (master_or_slave != master_or_slave->master) { - grpc_pollset_set *master = pss_ref(master_or_slave->master); + while (master_or_slave != master_or_slave->po.pss_master) { + grpc_pollset_set *master = pss_ref(master_or_slave->po.pss_master); gpr_mu_unlock(&master_or_slave->po.mu); pss_unref(master_or_slave); master_or_slave = master; @@ -893,7 +894,6 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, grpc_pollset_set *b) { pss_ref(a); pss_ref(b); - bool changed; for (;;) { if (a == b) { pss_unref(a); @@ -906,21 +906,18 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, gpr_mu_lock(&b->po.mu); gpr_mu_lock(&a->po.mu); } - changed = false; - if (a != a->master) { - grpc_pollset_set *master = pss_ref(a->master); + if (a != a->po.pss_master) { + grpc_pollset_set *master = pss_ref(a->po.pss_master); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); pss_unref(a); a = master; - changed = true; - } else if (b != b->master) { - grpc_pollset_set *master = pss_ref(b->master); + } else if (b != b->po.pss_master) { + grpc_pollset_set *master = pss_ref(b->po.pss_master); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); pss_unref(b); b = master; - changed = true; } else { /* a, b locked and are at their respective masters */ pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); @@ -928,6 +925,7 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, b->po.pss_master = a; gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); + return; } } } diff --git a/test/core/slice/slice_buffer_test.c b/test/core/slice/slice_buffer_test.c index bf9ae197d2..51cab2a87f 100644 --- a/test/core/slice/slice_buffer_test.c +++ b/test/core/slice/slice_buffer_test.c @@ -115,8 +115,8 @@ void test_slice_buffer_move_first() { grpc_slice_buffer_move_first(&src, 2, &dst); src_len -= 2; dst_len += 2; - GPR_ASSERT(src.length == src.length); - GPR_ASSERT(dst.length == dst.length); + GPR_ASSERT(src_len == src.length); + GPR_ASSERT(dst_len == dst.length); } int main(int argc, char **argv) { -- cgit v1.2.3 From bd412f9d3d3a567568f7ba4f081447315c957495 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 13:45:19 -0700 Subject: always use pss_obj_init --- src/core/lib/iomgr/ev_epollex_linux.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 23ca26933a..734f0c9a57 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -91,6 +91,14 @@ typedef struct pss_obj { grpc_pollset_set *pss_master; } pss_obj; +static void pss_obj_init(pss_obj *obj) { + gpr_mu_init(&obj->mu); + obj->pss_refs = 0; + obj->pss_next = NULL; + obj->pss_prev = NULL; + obj->pss_master = NULL; +} + /******************************************************************************* * Fd Declarations */ @@ -288,13 +296,9 @@ static grpc_fd *fd_create(int fd, const char *name) { if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->po.mu); } - /* Note: It is not really needed to get the new_fd->po.mu lock here. If this - * is a newly created fd (or an fd we got from the freelist), no one else - * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->po.mu); + pss_obj_init(&new_fd->po); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -313,8 +317,6 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; - gpr_mu_unlock(&new_fd->po.mu); - char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); @@ -525,7 +527,7 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - gpr_mu_init(&pollset->po.mu); + pss_obj_init(&pollset->po); pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); @@ -787,7 +789,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static grpc_pollset_set *pollset_set_create(void) { grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); - gpr_mu_init(&pss->po.mu); + pss_obj_init(&pss->po); gpr_ref_init(&pss->refs, 1); pss->roots[PSS_POLLSET_SET] = &pss->po; pss->po.pss_master = pss; -- cgit v1.2.3 From f081d8985a1a44da1fe35ba345e71bffd8d77945 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 14:04:42 -0700 Subject: different merge --- src/core/lib/iomgr/ev_epollex_linux.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 734f0c9a57..9bba20e9a6 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1026,14 +1026,12 @@ static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, grpc_pollset_set *item) { - pss_add_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); + pss_merge(exec_ctx, bag, item); } static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, - grpc_pollset_set *item) { - pss_del_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); -} + grpc_pollset_set *item) {} /******************************************************************************* * Event engine binding -- cgit v1.2.3 From 05da6e98c15f263d18c6d28e84e85b085784c490 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 21:21:37 +0000 Subject: Stash timeout in a stack var to ease debug --- src/core/lib/iomgr/ev_epollex_linux.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 9bba20e9a6..ac63bee95e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -404,7 +404,7 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { REF_BY(fd, 2, "return_workqueue"); return (grpc_workqueue*)fd; } #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, @@ -636,8 +636,9 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } GRPC_SCHEDULING_START_BLOCKING_REGION; +int timeout=poll_deadline_to_millis_timeout(deadline, now); int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, - poll_deadline_to_millis_timeout(deadline, now)); + timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); @@ -873,21 +874,21 @@ static void pss_merge_broadcast_and_patch(grpc_exec_ctx *exec_ctx, pss_obj_type type) { pss_obj *obj; if (a->roots[type] != NULL) { - obj = a->roots[PSS_FD]; + obj = a->roots[type]; do { broadcast[type](exec_ctx, b, obj); obj = obj->pss_next; - } while (obj != a->roots[PSS_FD]); + } while (obj != a->roots[type]); } if (b->roots[type] != NULL) { - obj = b->roots[PSS_FD]; + obj = b->roots[type]; do { broadcast[type](exec_ctx, a, obj); gpr_mu_lock(&obj->mu); obj->pss_master = a; gpr_mu_unlock(&obj->mu); obj = obj->pss_next; - } while (obj != b->roots[PSS_FD]); + } while (obj != b->roots[type]); } a->roots[type] = pss_splice(a->roots[type], b->roots[type]); } -- cgit v1.2.3 From 7f5fac90547d54f0f98d510995e697e6ea3ec9ed Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 21:47:41 +0000 Subject: minor optimizations --- src/core/lib/iomgr/ev_epollex_linux.c | 10 ++++------ test/cpp/microbenchmarks/bm_pollset.cc | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index ac63bee95e..cb996d8884 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -566,19 +566,17 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { gpr_timespec timeout; - static const int64_t max_spin_polling_us = 10; if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { return -1; } - if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( - max_spin_polling_us, - GPR_TIMESPAN))) <= 0) { + if (gpr_time_cmp(deadline, now) <= 0) { return 0; } + + static const gpr_timespec round_up = {.clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS-1}; timeout = gpr_time_sub(deadline, now); - int millis = gpr_time_to_millis(gpr_time_add( - timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; } diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 0f3d3cef66..3789deebde 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -136,8 +136,7 @@ static void BM_PollEmptyPollset(benchmark::State& state) { gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); gpr_mu_lock(mu); while (state.KeepRunning()) { - grpc_pollset_worker* worker; - GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, NULL, now, deadline)); } grpc_closure shutdown_ps_closure; grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, @@ -233,8 +232,7 @@ static void BM_SingleThreadPollOneFd(benchmark::State& state) { grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure); gpr_mu_lock(mu); while (!done) { - grpc_pollset_worker* worker; - GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, NULL, now, deadline)); } grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done"); wakeup_fd.read_fd = 0; -- cgit v1.2.3 From cd4a214d8a0475d22147260dfd736cb7a915921b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 22:14:58 +0000 Subject: fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index cb996d8884..c75b59c4c4 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -796,8 +796,16 @@ static grpc_pollset_set *pollset_set_create(void) { return pss; } +static void pss_unref(grpc_pollset_set *pss); + static void pss_destroy(grpc_pollset_set *pss) { gpr_mu_destroy(&pss->po.mu); + GPR_ASSERT(pss->roots[PSS_FD] == NULL); + GPR_ASSERT(pss->roots[PSS_POLLSET] == NULL); + GPR_ASSERT(pss->roots[PSS_POLLSET_SET] == &pss->po); + for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; child = child->pss_next) { + pss_unref((grpc_pollset_set*)child); + } gpr_free(pss); } @@ -924,8 +932,11 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); b->po.pss_master = a; + a->roots[PSS_POLLSET_SET] = pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); + pss_unref(a); + /* a now owns a ref to b */ return; } } -- cgit v1.2.3 From 8c6878be3a5f748ccbc68be261303a3b57837854 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 15:15:30 -0700 Subject: clang-format --- src/core/lib/iomgr/ev_epollex_linux.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c75b59c4c4..89b416a60e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -404,7 +404,10 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { REF_BY(fd, 2, "return_workqueue"); return (grpc_workqueue*)fd; } +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + REF_BY(fd, 2, "return_workqueue"); + return (grpc_workqueue *)fd; +} #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, @@ -574,7 +577,8 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = {.clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS-1}; + static const gpr_timespec round_up = { + .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; @@ -634,9 +638,8 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } GRPC_SCHEDULING_START_BLOCKING_REGION; -int timeout=poll_deadline_to_millis_timeout(deadline, now); - int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, - timeout); + int timeout = poll_deadline_to_millis_timeout(deadline, now); + int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); @@ -803,8 +806,9 @@ static void pss_destroy(grpc_pollset_set *pss) { GPR_ASSERT(pss->roots[PSS_FD] == NULL); GPR_ASSERT(pss->roots[PSS_POLLSET] == NULL); GPR_ASSERT(pss->roots[PSS_POLLSET_SET] == &pss->po); - for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; child = child->pss_next) { - pss_unref((grpc_pollset_set*)child); + for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; + child = child->pss_next) { + pss_unref((grpc_pollset_set *)child); } gpr_free(pss); } @@ -932,7 +936,8 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); b->po.pss_master = a; - a->roots[PSS_POLLSET_SET] = pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); + a->roots[PSS_POLLSET_SET] = + pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); pss_unref(a); -- cgit v1.2.3 From 2636f43a6789f63148448fdce2877aff6b0aaade Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 15:16:56 -0700 Subject: less spam --- src/core/lib/iomgr/ev_epollex_linux.c | 36 +++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 89b416a60e..c71b1fe853 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1098,12 +1098,17 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ static bool is_epollex_available(void) { + static bool logged_why_not = false; + int fd = epoll_create1(EPOLL_CLOEXEC); if (fd < 0) { - gpr_log(GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epollex polling " - "engine.", - fd); + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } return false; } grpc_wakeup_fd wakeup; @@ -1119,19 +1124,26 @@ static bool is_epollex_available(void) { .data.ptr = NULL}; if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { if (errno != EINVAL) { - gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " - "%d. Not using epollex polling engine.", - errno); + if (!logged_why_not) { + gpr_log( + GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + logged_why_not = true; + } close(fd); grpc_wakeup_fd_destroy(&wakeup); return false; } } else { - gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " - "evidence of no EPOLLEXCLUSIVE support. Not using " - "epollex polling engine."); + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); + logged_why_not = true; + } close(fd); grpc_wakeup_fd_destroy(&wakeup); return false; -- cgit v1.2.3 From d9cd8f0abe6aa8562aaafb041b1db371247517d6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 16:26:02 -0700 Subject: shutdown progress --- src/core/lib/iomgr/ev_epollex_linux.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c71b1fe853..172847f6c8 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -601,11 +601,20 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } +static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0 && + pollset->po.pss_master == NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + } +} + /* pollset->po.mu lock must be held by the caller before calling this */ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { GPR_ASSERT(pollset->shutdown_closure == NULL); pollset->shutdown_closure = closure; + gpr_atm_no_barrier_store(&pollset->shutdown_atm, 1); if (pollset->num_pollers > 0) { struct epoll_event ev = {.events = EPOLLIN, .data.ptr = &pollset->pollset_wakeup}; @@ -613,9 +622,16 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, &ev); GRPC_LOG_IF_ERROR("pollset_shutdown", grpc_wakeup_fd_wakeup(&pollset->pollset_wakeup)); - } else { - grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } + if (pollset->root_worker != NULL) { + for (grpc_pollset_worker *worker = pollset->root_worker->next; + worker != pollset->root_worker; worker = worker->next) { + if (worker->initialized_cv) { + gpr_cv_signal(&worker->cv); + } + } + } + pollset_maybe_finish_shutdown(exec_ctx, pollset); } /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ @@ -719,6 +735,7 @@ static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, pollset->root_worker = worker->next; worker->prev->next = worker->next; worker->next->prev = worker->prev; + gpr_cv_signal(&pollset->root_worker->cv); } } else { worker->prev->next = worker->next; @@ -747,9 +764,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->po.mu); pollset->num_pollers--; - if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { - grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); - } + pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(pollset, &worker, worker_hdl); return error; @@ -979,12 +994,14 @@ static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, obj->pss_prev->pss_next = obj; obj->pss_next->pss_prev = obj; } - gpr_mu_unlock(&obj->mu); switch (type) { case PSS_FD: + REF_BY((grpc_fd *)obj, 2, "pollset_set"); + gpr_mu_unlock(&obj->mu); pss_broadcast_fd(exec_ctx, pss, obj); break; case PSS_POLLSET: + gpr_mu_unlock(&obj->mu); pss_broadcast_pollset(exec_ctx, pss, obj); break; case PSS_POLLSET_SET: @@ -998,6 +1015,7 @@ static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, pss_obj *obj, pss_obj_type type) { + bool unref = false; pss = pss_ref_and_lock_master(pss); gpr_mu_lock(&obj->mu); obj->pss_refs--; @@ -1012,10 +1030,12 @@ static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, obj->pss_next->pss_prev = obj->pss_prev; obj->pss_prev->pss_next = obj->pss_next; } + unref = true; } gpr_mu_unlock(&obj->mu); gpr_mu_unlock(&pss->po.mu); pss_unref(pss); + if (unref && type == PSS_FD) UNREF_BY((grpc_fd *)obj, 2, "pollset_set"); } static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, -- cgit v1.2.3 From bb93af6b86d0a097db109ab0aa0b0f7a43cb2a3e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 23:49:00 +0000 Subject: kick optimizations --- src/core/lib/iomgr/ev_epollex_linux.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c75b59c4c4..d3cc1d1952 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -165,6 +165,7 @@ struct grpc_pollset { pss_obj po; int epfd; int num_pollers; + bool kicked_without_poller; gpr_atm shutdown_atm; grpc_closure *shutdown_closure; grpc_wakeup_fd pollset_wakeup; @@ -507,7 +508,12 @@ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { - return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + if (p->num_pollers == 0) { + p->kicked_without_poller = true; + return GRPC_ERROR_NONE; + } else { + return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + } } else { return GRPC_ERROR_NONE; } @@ -528,6 +534,7 @@ static grpc_error *kick_poller(void) { static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pss_obj_init(&pollset->po); + pollset->kicked_without_poller = false; pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); @@ -736,13 +743,21 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; grpc_error *error = GRPC_ERROR_NONE; + if (pollset->kicked_without_poller) { + pollset->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } if (begin_worker(pollset, &worker, worker_hdl, deadline)) { + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); pollset->num_pollers++; gpr_mu_unlock(&pollset->po.mu); error = pollset_poll(exec_ctx, pollset, now, deadline); grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->po.mu); + gpr_tls_set(&g_current_thread_pollset, 0); + gpr_tls_set(&g_current_thread_worker, 0); pollset->num_pollers--; if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); -- cgit v1.2.3 From f18286bda6a7a117f5e8e33c6d65446c141d150f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 14:44:09 -0700 Subject: attempt #2 at pollset_set --- src/core/lib/iomgr/ev_epollex_linux.c | 508 +++++++++++++++++----------------- 1 file changed, 253 insertions(+), 255 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index de780ec417..958f4d5028 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -77,34 +77,48 @@ static grpc_wakeup_fd global_wakeup_fd; */ typedef enum { - PSS_FD, - PSS_POLLSET, - PSS_POLLSET_SET, - PSS_OBJ_TYPE_COUNT -} pss_obj_type; + PO_FD, + PO_POLLSET, + PO_POLLSET_SET, + PO_POLLING_GROUP, + PO_COUNT +} polling_obj_type; -typedef struct pss_obj { +typedef struct polling_obj polling_obj; +typedef struct polling_group polling_group; + +struct polling_obj { gpr_mu mu; - struct pss_obj *pss_next; - struct pss_obj *pss_prev; - int pss_refs; - grpc_pollset_set *pss_master; -} pss_obj; - -static void pss_obj_init(pss_obj *obj) { - gpr_mu_init(&obj->mu); - obj->pss_refs = 0; - obj->pss_next = NULL; - obj->pss_prev = NULL; - obj->pss_master = NULL; -} + polling_obj_type type; + polling_group *group; + struct polling_obj *next; + struct polling_obj *prev; +}; + +struct polling_group { + polling_obj po; + gpr_refcount refs; +}; + +static void po_init(polling_obj *po, polling_obj_type type); +static void po_destroy(polling_obj *po); +static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b); + +static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, + size_t initial_po_count); +static polling_group *pg_ref(polling_group *pg); +static void pg_unref(polling_group *pg); +static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, + polling_group *b); +static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, + polling_obj *po); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pss_obj po; + polling_obj po; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -162,7 +176,7 @@ struct grpc_pollset_worker { }; struct grpc_pollset { - pss_obj po; + polling_obj po; int epfd; int num_pollers; bool kicked_without_poller; @@ -181,11 +195,7 @@ struct grpc_pollset { * Pollset-set Declarations */ struct grpc_pollset_set { - pss_obj po; - gpr_refcount refs; - - /* roots are only used if master == self */ - pss_obj *roots[PSS_OBJ_TYPE_COUNT]; + polling_obj po; }; /******************************************************************************* @@ -257,10 +267,11 @@ static void unref_by(grpc_fd *fd, int n) { old = gpr_atm_full_fetch_add(&fd->refst, -n); if (old == n) { /* Add the fd to the freelist */ + grpc_iomgr_unregister_object(&fd->iomgr_object); + po_destroy(&fd->po); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); grpc_lfev_destroy(&fd->read_closure); grpc_lfev_destroy(&fd->write_closure); @@ -279,7 +290,6 @@ static void fd_global_shutdown(void) { while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->po.mu); gpr_free(fd); } gpr_mu_destroy(&fd_freelist_mu); @@ -299,7 +309,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = gpr_malloc(sizeof(grpc_fd)); } - pss_obj_init(&new_fd->po); + po_init(&new_fd->po, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -536,7 +546,7 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - pss_obj_init(&pollset->po); + po_init(&pollset->po, PO_POLLSET); pollset->kicked_without_poller = false; pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { @@ -610,8 +620,7 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0 && - pollset->po.pss_master == NULL) { + if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); } } @@ -643,7 +652,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->po.mu); + po_destroy(&pollset->po); if (pollset->epfd >= 0) close(pollset->epfd); grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); } @@ -821,268 +830,257 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static grpc_pollset_set *pollset_set_create(void) { grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); - pss_obj_init(&pss->po); - gpr_ref_init(&pss->refs, 1); - pss->roots[PSS_POLLSET_SET] = &pss->po; - pss->po.pss_master = pss; - pss->po.pss_next = pss->po.pss_prev = &pss->po; + po_init(&pss->po, PO_POLLSET_SET); return pss; } -static void pss_unref(grpc_pollset_set *pss); - -static void pss_destroy(grpc_pollset_set *pss) { - gpr_mu_destroy(&pss->po.mu); - GPR_ASSERT(pss->roots[PSS_FD] == NULL); - GPR_ASSERT(pss->roots[PSS_POLLSET] == NULL); - GPR_ASSERT(pss->roots[PSS_POLLSET_SET] == &pss->po); - for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; - child = child->pss_next) { - pss_unref((grpc_pollset_set *)child); - } - gpr_free(pss); +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + po_destroy(&pss->po); } -static grpc_pollset_set *pss_ref(grpc_pollset_set *pss) { - gpr_ref(&pss->refs); - return pss; +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + po_join(exec_ctx, &pss->po, &fd->po); } -static void pss_unref(grpc_pollset_set *pss) { - if (gpr_unref(&pss->refs)) pss_destroy(pss); -} +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} -static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss) { - pss_unref(pss); +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + po_join(exec_ctx, &pss->po, &ps->po); } -static grpc_pollset_set *pss_ref_and_lock_master( - grpc_pollset_set *master_or_slave) { - pss_ref(master_or_slave); - gpr_mu_lock(&master_or_slave->po.mu); - while (master_or_slave != master_or_slave->po.pss_master) { - grpc_pollset_set *master = pss_ref(master_or_slave->po.pss_master); - gpr_mu_unlock(&master_or_slave->po.mu); - pss_unref(master_or_slave); - master_or_slave = master; - gpr_mu_lock(&master_or_slave->po.mu); - } - return master_or_slave; -} +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} -static void pss_broadcast_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *dst, - pss_obj *obj) { - grpc_fd *fd = (grpc_fd *)obj; - if (dst->roots[PSS_POLLSET] == NULL) return; - pss_obj *tgt = dst->roots[PSS_POLLSET]; - do { - pollset_add_fd(exec_ctx, (grpc_pollset *)tgt, fd); - tgt = tgt->pss_next; - } while (tgt != dst->roots[PSS_POLLSET]); +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + po_join(exec_ctx, &bag->po, &item->po); } -static void pss_broadcast_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *dst, pss_obj *obj) { - grpc_pollset *pollset = (grpc_pollset *)obj; - if (dst->roots[PSS_FD] == NULL) return; - pss_obj *tgt = dst->roots[PSS_FD]; - do { - pollset_add_fd(exec_ctx, pollset, (grpc_fd *)tgt); - tgt = tgt->pss_next; - } while (tgt != dst->roots[PSS_FD]); -} +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} -static pss_obj *pss_splice(pss_obj *p, pss_obj *q) { - if (p == NULL) return q; - if (q == NULL) return p; - p->pss_next->pss_prev = q->pss_prev; - q->pss_prev->pss_next = p->pss_next; - p->pss_next = q; - q->pss_prev = p; - return p; +static void po_init(polling_obj *po, polling_obj_type type) { + gpr_mu_init(&po->mu); + po->type = type; + po->group = NULL; + po->next = po; + po->prev = po; } -static void (*const broadcast[PSS_OBJ_TYPE_COUNT])(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *dst, - pss_obj *obj) = { - pss_broadcast_fd, pss_broadcast_pollset, NULL}; - -static void pss_merge_broadcast_and_patch(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *a, - grpc_pollset_set *b, - pss_obj_type type) { - pss_obj *obj; - if (a->roots[type] != NULL) { - obj = a->roots[type]; - do { - broadcast[type](exec_ctx, b, obj); - obj = obj->pss_next; - } while (obj != a->roots[type]); - } - if (b->roots[type] != NULL) { - obj = b->roots[type]; - do { - broadcast[type](exec_ctx, a, obj); - gpr_mu_lock(&obj->mu); - obj->pss_master = a; - gpr_mu_unlock(&obj->mu); - obj = obj->pss_next; - } while (obj != b->roots[type]); +static polling_group *pg_lock_latest(polling_group *pg) { + /* assumes pg unlocked; consumes ref, returns ref */ + gpr_mu_lock(&pg->po.mu); + while (pg->po.group != NULL) { + polling_group *new_pg = pg_ref(pg->po.group); + gpr_mu_unlock(&pg->po.mu); + pg_unref(pg); + pg = new_pg; + gpr_mu_lock(&pg->po.mu); } - a->roots[type] = pss_splice(a->roots[type], b->roots[type]); + return pg; } -static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, - grpc_pollset_set *b) { - pss_ref(a); - pss_ref(b); - for (;;) { - if (a == b) { - pss_unref(a); - pss_unref(b); - return; - } else if (a < b) { - gpr_mu_lock(&a->po.mu); - gpr_mu_lock(&b->po.mu); - } else { - gpr_mu_lock(&b->po.mu); - gpr_mu_lock(&a->po.mu); - } - if (a != a->po.pss_master) { - grpc_pollset_set *master = pss_ref(a->po.pss_master); - gpr_mu_unlock(&a->po.mu); - gpr_mu_unlock(&b->po.mu); - pss_unref(a); - a = master; - } else if (b != b->po.pss_master) { - grpc_pollset_set *master = pss_ref(b->po.pss_master); - gpr_mu_unlock(&a->po.mu); - gpr_mu_unlock(&b->po.mu); - pss_unref(b); - b = master; - } else { - /* a, b locked and are at their respective masters */ - pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); - pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); - b->po.pss_master = a; - a->roots[PSS_POLLSET_SET] = - pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); - gpr_mu_unlock(&a->po.mu); - gpr_mu_unlock(&b->po.mu); - pss_unref(a); - /* a now owns a ref to b */ - return; - } +static void po_destroy(polling_obj *po) { + if (po->group != NULL) { + polling_group *pg = pg_lock_latest(po->group); + po->prev->next = po->next; + po->next->prev = po->prev; + pg_unref(pg); } + gpr_mu_destroy(&po->mu); } -static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - pss_obj *obj, pss_obj_type type) { - pss = pss_ref_and_lock_master(pss); - gpr_mu_lock(&obj->mu); - if (obj->pss_master == pss) { - /* obj is already a member -- just bump refcount */ - obj->pss_refs++; - gpr_mu_unlock(&obj->mu); - gpr_mu_unlock(&pss->po.mu); - pss_unref(pss); - return; - } else if (obj->pss_master != NULL) { - grpc_pollset_set *other_pss = pss_ref(obj->pss_master); - obj->pss_refs++; - gpr_mu_unlock(&obj->mu); - gpr_mu_unlock(&pss->po.mu); - pss_merge(exec_ctx, pss, other_pss); - pss_unref(other_pss); - pss_unref(pss); - } else { - GPR_ASSERT(obj->pss_refs == 0); - obj->pss_refs = 1; - obj->pss_master = pss; - if (pss->roots[type] == NULL) { - pss->roots[type] = obj; - obj->pss_next = obj->pss_prev = obj; - } else { - obj->pss_next = pss->roots[type]; - obj->pss_prev = obj->pss_next->pss_prev; - obj->pss_prev->pss_next = obj; - obj->pss_next->pss_prev = obj; - } - switch (type) { - case PSS_FD: - REF_BY((grpc_fd *)obj, 2, "pollset_set"); - gpr_mu_unlock(&obj->mu); - pss_broadcast_fd(exec_ctx, pss, obj); - break; - case PSS_POLLSET: - gpr_mu_unlock(&obj->mu); - pss_broadcast_pollset(exec_ctx, pss, obj); - break; - case PSS_POLLSET_SET: - case PSS_OBJ_TYPE_COUNT: - GPR_UNREACHABLE_CODE(break); - } - gpr_mu_unlock(&pss->po.mu); - pss_unref(pss); +static polling_group *pg_ref(polling_group *pg) { + gpr_ref(&pg->refs); + return pg; +} + +static void pg_unref(polling_group *pg) { + if (gpr_unref(&pg->refs)) { + po_destroy(&pg->po); + gpr_free(pg); } } -static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - pss_obj *obj, pss_obj_type type) { - bool unref = false; - pss = pss_ref_and_lock_master(pss); - gpr_mu_lock(&obj->mu); - obj->pss_refs--; - if (obj->pss_refs == 0) { - obj->pss_master = NULL; - if (obj == pss->roots[type]) { - pss->roots[type] = obj->pss_next; - } - if (obj->pss_next == obj) { - pss->roots[type] = NULL; +static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { + if (a == b) return; + if (a > b) GPR_SWAP(polling_obj *, a, b); + + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + + if (a->group == NULL) { + if (b->group == NULL) { + polling_obj *initial_po[] = {a, b}; + pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); } else { - obj->pss_next->pss_prev = obj->pss_prev; - obj->pss_prev->pss_next = obj->pss_next; + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + pg_join(exec_ctx, b_group, a); } - unref = true; + } else if (b->group == NULL) { + polling_group *a_group = pg_ref(a->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_join(exec_ctx, a_group, b); + } else if (a->group == b->group) { + /* nothing to do */ + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *a_group = pg_ref(a->group); + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_merge(exec_ctx, a_group, b_group); } - gpr_mu_unlock(&obj->mu); - gpr_mu_unlock(&pss->po.mu); - pss_unref(pss); - if (unref && type == PSS_FD) UNREF_BY((grpc_fd *)obj, 2, "pollset_set"); } -static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - grpc_fd *fd) { - pss_add_obj(exec_ctx, pss, &fd->po, PSS_FD); +#define POTYPES(a, b) (((a)*PO_COUNT) + (b)) + +static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { + GPR_ASSERT(a != b); + switch (POTYPES(a->type, b->type)) { + case POTYPES(PO_FD, PO_POLLSET): + pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); + break; + case POTYPES(PO_POLLSET, PO_FD): + pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); + break; + } } -static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - grpc_fd *fd) { - pss_del_obj(exec_ctx, pss, &fd->po, PSS_FD); +static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, + polling_group *to) { + for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { + for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { + pg_notify(exec_ctx, a, b); + } + } } -static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss, grpc_pollset *ps) { - pss_add_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); +static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, + size_t initial_po_count) { + /* assumes all polling objects in initial_po are locked */ + polling_group *pg = gpr_malloc(sizeof(*pg)); + po_init(&pg->po, PO_POLLING_GROUP); + gpr_ref_init(&pg->refs, initial_po_count); + GPR_ASSERT(initial_po[0]->group == NULL); + initial_po[0]->next = initial_po[0]->prev = initial_po[0]; + initial_po[0]->group = pg; + for (size_t i = 0; i < initial_po_count; i++) { + GPR_ASSERT(initial_po[i]->group == NULL); + initial_po[i]->group = pg; + } + for (size_t i = 1; i < initial_po_count; i++) { + initial_po[i]->prev = initial_po[i - 1]; + } + for (size_t i = 0; i < initial_po_count - 1; i++) { + initial_po[i]->next = initial_po[i + 1]; + } + initial_po[0]->prev = &pg->po; + initial_po[initial_po_count - 1]->next = &pg->po; + pg->po.next = initial_po[0]; + pg->po.prev = initial_po[initial_po_count - 1]; + for (size_t i = 1; i < initial_po_count; i++) { + for (size_t j = 0; j < i; j++) { + pg_notify(exec_ctx, initial_po[i], initial_po[j]); + } + } } -static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss, grpc_pollset *ps) { - pss_del_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); +static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, + polling_obj *po) { + /* assumes neither pg nor po are locked; consumes one ref to pg */ + pg = pg_lock_latest(pg); + /* pg locked */ + gpr_mu_lock(&po->mu); + if (po->group != NULL) { + gpr_mu_unlock(&pg->po.mu); + polling_group *po_group = pg_ref(po->group); + gpr_mu_unlock(&po->mu); + pg_merge(exec_ctx, pg, po_group); + /* early exit: polling obj picked up a group before joining: we now need + to do a full merge */ + return; + } + /* pg, po locked */ + for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; + existing != &pg->po; existing = existing->next) { + pg_notify(exec_ctx, po, existing); + } + po->group = pg; + po->next = &pg->po; + po->prev = pg->po.prev; + po->prev->next = po->next->prev = po; + gpr_mu_unlock(&pg->po.mu); + gpr_mu_unlock(&po->mu); } -static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) { - pss_merge(exec_ctx, bag, item); +static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, + polling_group *b) { + for (;;) { + if (a == b) return; + if (a > b) GPR_SWAP(polling_group *, a, b); + gpr_mu_lock(&a->po.mu); + gpr_mu_lock(&b->po.mu); + if (a->po.group != NULL) { + polling_group *m2 = pg_ref(a->po.group); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pg_unref(a); + a = m2; + } else if (b->po.group != NULL) { + polling_group *m2 = pg_ref(b->po.group); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pg_unref(b); + b = m2; + } else { + break; + } + } + polling_group **unref = NULL; + size_t unref_count = 0; + size_t unref_cap = 0; + b->po.group = a; + pg_broadcast(exec_ctx, a, b); + pg_broadcast(exec_ctx, b, a); + while (b->po.next != &b->po) { + polling_obj *po = b->po.next; + gpr_mu_lock(&po->mu); + if (unref_count == unref_cap) { + unref_cap = GPR_MAX(8, 3 * unref_cap / 2); + unref = gpr_realloc(unref, unref_cap * sizeof(*unref)); + } + unref[unref_count++] = po->group; + po->group = pg_ref(a); + // unlink from b + po->prev->next = po->next; + po->next->prev = po->prev; + // link to a + po->next = &a->po; + po->prev = a->po.prev; + po->next->prev = po->prev->next = po; + gpr_mu_unlock(&po->mu); + } + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + for (size_t i = 0; i < unref_count; i++) { + pg_unref(unref[i]); + } + gpr_free(unref); } -static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) {} - /******************************************************************************* * Event engine binding */ -- cgit v1.2.3 From ac328d2981ba0730ff4656106463282da4cbda80 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 14:45:15 -0700 Subject: fix compile --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 958f4d5028..d353eaf042 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -973,7 +973,7 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, /* assumes all polling objects in initial_po are locked */ polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); - gpr_ref_init(&pg->refs, initial_po_count); + gpr_ref_init(&pg->refs, (int)initial_po_count); GPR_ASSERT(initial_po[0]->group == NULL); initial_po[0]->next = initial_po[0]->prev = initial_po[0]; initial_po[0]->group = pg; -- cgit v1.2.3 From 105016680a8b50ad48182bf94e449db6d3a85bcf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 21:47:08 +0000 Subject: refine code --- src/core/lib/iomgr/ev_epollex_linux.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d353eaf042..4dbc8b127c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -974,9 +974,6 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); gpr_ref_init(&pg->refs, (int)initial_po_count); - GPR_ASSERT(initial_po[0]->group == NULL); - initial_po[0]->next = initial_po[0]->prev = initial_po[0]; - initial_po[0]->group = pg; for (size_t i = 0; i < initial_po_count; i++) { GPR_ASSERT(initial_po[i]->group == NULL); initial_po[i]->group = pg; -- cgit v1.2.3 From 3e21ec5c5ec4629c66078a44beb02c70aac1e5f9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 15:11:38 -0700 Subject: add missing unlock --- src/core/lib/iomgr/ev_epollex_linux.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4dbc8b127c..3330d86d85 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -891,6 +891,7 @@ static void po_destroy(polling_obj *po) { polling_group *pg = pg_lock_latest(po->group); po->prev->next = po->next; po->next->prev = po->prev; + gpr_mu_unlock(&pg->po.mu); pg_unref(pg); } gpr_mu_destroy(&po->mu); @@ -974,6 +975,9 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); gpr_ref_init(&pg->refs, (int)initial_po_count); + GPR_ASSERT(initial_po[0]->group == NULL); + initial_po[0]->next = initial_po[0]->prev = initial_po[0]; + initial_po[0]->group = pg; for (size_t i = 0; i < initial_po_count; i++) { GPR_ASSERT(initial_po[i]->group == NULL); initial_po[i]->group = pg; -- cgit v1.2.3 From 9b59114f73cc081716669e3866eb058ad69a69c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 22:12:44 +0000 Subject: huh --- src/core/lib/iomgr/ev_epollex_linux.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 3330d86d85..b4c879e431 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -975,9 +975,6 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); gpr_ref_init(&pg->refs, (int)initial_po_count); - GPR_ASSERT(initial_po[0]->group == NULL); - initial_po[0]->next = initial_po[0]->prev = initial_po[0]; - initial_po[0]->group = pg; for (size_t i = 0; i < initial_po_count; i++) { GPR_ASSERT(initial_po[i]->group == NULL); initial_po[i]->group = pg; -- cgit v1.2.3 From 83d5fb6e61a8f818c41929a81984b831244c3b67 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 22:13:43 +0000 Subject: fix memory leak --- src/core/lib/iomgr/ev_epollex_linux.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b4c879e431..c6505f6868 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -837,6 +837,7 @@ static grpc_pollset_set *pollset_set_create(void) { static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss) { po_destroy(&pss->po); + gpr_free(pss); } static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, -- cgit v1.2.3 From 59b86dd7957be126d542c9f46c24ce8b9a80bd98 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 23:11:08 +0000 Subject: debug --- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c6505f6868..2afc20daa5 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -519,6 +519,7 @@ static void pollset_global_shutdown(void) { /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { + gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d root_worker=%p", p, specific_worker, (void*)gpr_tls_get(&g_current_thread_pollset), (void*)gpr_tls_get(&g_current_thread_worker), p->num_pollers, p->root_worker); if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { if (p->num_pollers == 0) { @@ -771,6 +772,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; + gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%"PRId64".%09d deadline=%"PRId64".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, pollset->root_worker); grpc_error *error = GRPC_ERROR_NONE; if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; -- cgit v1.2.3 From 1814374c6958ca4a0a97f7a85e9cdcb7d5d30e82 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 23:14:29 +0000 Subject: debug --- src/core/lib/iomgr/ev_epollex_linux.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 2afc20daa5..468acd60c7 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -676,14 +676,19 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + grpc_error *error = GRPC_ERROR_NONE; for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { + gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else if (data_ptr == &pollset->pollset_wakeup) { + gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); /* once we start shutting down we stop consuming the wakeup: the fd is level triggered and non-exclusive, which should result in all pollers waking */ @@ -697,6 +702,9 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; + gpr_log(GPR_DEBUG, + "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", pollset, + is_workqueue, cancel, read_ev, write_ev); if (is_workqueue) { append_error(&error, grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), @@ -812,7 +820,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = fd}; + .data.ptr = (void*)(1+(intptr_t)fd)}; if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != 0) { switch (errno) { -- cgit v1.2.3 From cd0354b851390a4db72a02f389d8dfcd7cc7e3fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 16:19:18 -0700 Subject: Add tracer --- src/core/lib/iomgr/ev_epoll_linux.c | 1 - src/core/lib/iomgr/ev_epollex_linux.c | 39 +++++++++++++++++++++++++++-------- src/core/lib/iomgr/ev_posix.c | 5 +++++ src/core/lib/iomgr/ev_posix.h | 2 ++ 4 files changed, 37 insertions(+), 10 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index d41c164d71..613b435693 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -66,7 +66,6 @@ #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -static int grpc_polling_trace = 0; /* Disabled by default */ #define GRPC_POLLING_TRACE(fmt, ...) \ if (grpc_polling_trace) { \ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 468acd60c7..4c35dab0c3 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -519,7 +519,14 @@ static void pollset_global_shutdown(void) { /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { - gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d root_worker=%p", p, specific_worker, (void*)gpr_tls_get(&g_current_thread_pollset), (void*)gpr_tls_get(&g_current_thread_worker), p->num_pollers, p->root_worker); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, + "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d " + "root_worker=%p", + p, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset), + (void *)gpr_tls_get(&g_current_thread_worker), p->num_pollers, + p->root_worker); + } if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { if (p->num_pollers == 0) { @@ -676,19 +683,25 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); - gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + } grpc_error *error = GRPC_ERROR_NONE; for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + } grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else if (data_ptr == &pollset->pollset_wakeup) { - gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); + } /* once we start shutting down we stop consuming the wakeup: the fd is level triggered and non-exclusive, which should result in all pollers waking */ @@ -702,9 +715,11 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; - gpr_log(GPR_DEBUG, - "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", pollset, - is_workqueue, cancel, read_ev, write_ev); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, + "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", + pollset, is_workqueue, cancel, read_ev, write_ev); + } if (is_workqueue) { append_error(&error, grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), @@ -780,7 +795,13 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%"PRId64".%09d deadline=%"PRId64".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, pollset->root_worker); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 + ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", + pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, + deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, + pollset->root_worker); + } grpc_error *error = GRPC_ERROR_NONE; if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; @@ -820,7 +841,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = (void*)(1+(intptr_t)fd)}; + .data.ptr = (void *)(1 + (intptr_t)fd)}; if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != 0) { switch (errno) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 2663152aec..a8252b0162 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,11 +44,14 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/ev_epoll_linux.h" #include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" +int grpc_polling_trace = 0; /* Disabled by default */ + /** Default poll() function - a pointer so that it can be overridden by some * tests */ grpc_poll_function_type grpc_poll_function = poll; @@ -117,6 +120,8 @@ static void try_engine(const char *engine) { const char *grpc_get_poll_strategy_name() { return g_poll_strategy_name; } void grpc_event_engine_init(void) { + grpc_register_tracer("polling", &grpc_polling_trace); + char *s = gpr_getenv("GRPC_POLL_STRATEGY"); if (s == NULL) { s = gpr_strdup("all"); diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1a9e5c115a..7ce1e06f20 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -42,6 +42,8 @@ #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" +extern int grpc_polling_trace; /* Disabled by default */ + typedef struct grpc_fd grpc_fd; typedef struct grpc_event_engine_vtable { -- cgit v1.2.3 From 1ad9477da37b8aa92459c75ee22c9841075baaf0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 16:15:19 +0000 Subject: More debug --- src/core/lib/iomgr/ev_epollex_linux.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4c35dab0c3..35e4e0543f 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -530,20 +530,38 @@ static grpc_error *pollset_kick(grpc_pollset *p, if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { if (p->num_pollers == 0) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_without_poller", p); + } p->kicked_without_poller = true; return GRPC_ERROR_NONE; } else { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + } return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); } } else { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + } return GRPC_ERROR_NONE; } } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + } return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + } return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); } else { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_via_cv", p); + } gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } @@ -716,9 +734,11 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, - "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", - pollset, is_workqueue, cancel, read_ev, write_ev); + gpr_log( + GPR_DEBUG, + "PS:%p poll got fd %p(%d/%d): is_wq=%d cancel=%d read=%d write=%d", + pollset, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, is_workqueue, + cancel, read_ev, write_ev); } if (is_workqueue) { append_error(&error, -- cgit v1.2.3 From 66030c14cdd8710a9220d5fec45f0dd006c10fd6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 16:41:14 +0000 Subject: Fix broken test --- src/core/lib/iomgr/ev_epollex_linux.c | 5 +++++ test/core/end2end/tests/resource_quota_server.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 35e4e0543f..5eb92e310a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -697,6 +697,11 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GRPC_SCHEDULING_START_BLOCKING_REGION; int timeout = poll_deadline_to_millis_timeout(deadline, now); + + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll for %dms", pollset, timeout); + } + int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c index 4d3ce1c937..d6d640b8dd 100644 --- a/test/core/end2end/tests/resource_quota_server.c +++ b/test/core/end2end/tests/resource_quota_server.c @@ -203,7 +203,7 @@ void resource_quota_server(grpc_end2end_test_config config) { op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 0; - op->flags = 0; + op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY; op->reserved = NULL; op++; op->op = GRPC_OP_SEND_MESSAGE; -- cgit v1.2.3 From 6462fd81b3ae4d2805ab87110238f62996ccb0c5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 17:32:07 +0000 Subject: Debug --- src/core/lib/iomgr/ev_epollex_linux.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 5eb92e310a..88b7c83933 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1003,17 +1003,11 @@ static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { } } -#define POTYPES(a, b) (((a)*PO_COUNT) + (b)) - static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { - GPR_ASSERT(a != b); - switch (POTYPES(a->type, b->type)) { - case POTYPES(PO_FD, PO_POLLSET): - pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); - break; - case POTYPES(PO_POLLSET, PO_FD): - pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); - break; + if (a->type == PO_FD && b->type == PO_POLLSET) { + pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); + } else if (a->type == PO_POLLSET && b->type == PO_FD) { + pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); } } @@ -1064,7 +1058,7 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, polling_group *po_group = pg_ref(po->group); gpr_mu_unlock(&po->mu); pg_merge(exec_ctx, pg, po_group); - /* early exit: polling obj picked up a group before joining: we now need + /* early exit: polling obj picked up a group before joining: we needed to do a full merge */ return; } -- cgit v1.2.3 From 361489496541ce51dab11599b441d36852720093 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 17:45:08 +0000 Subject: Fix mem leaks --- src/core/lib/iomgr/ev_epollex_linux.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 88b7c83933..db35e0e11f 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1078,7 +1078,11 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, polling_group *b) { for (;;) { - if (a == b) return; + if (a == b) { + pg_unref(a); + pg_unref(b); + return; + } if (a > b) GPR_SWAP(polling_group *, a, b); gpr_mu_lock(&a->po.mu); gpr_mu_lock(&b->po.mu); @@ -1128,6 +1132,7 @@ static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, pg_unref(unref[i]); } gpr_free(unref); + pg_unref(b); } /******************************************************************************* -- cgit v1.2.3 From 3594aa896d2d5a1b0ab56d1d2e6d7998c1cee073 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 14:03:53 -0700 Subject: Delete unused fields --- src/core/lib/iomgr/ev_epollex_linux.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index db35e0e11f..a8d7b54887 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -184,11 +184,6 @@ struct grpc_pollset { grpc_closure *shutdown_closure; grpc_wakeup_fd pollset_wakeup; grpc_pollset_worker *root_worker; - - grpc_pollset *pss_next; - grpc_pollset *pss_prev; - int pss_refs; - grpc_pollset_set *pss_master; }; /******************************************************************************* -- cgit v1.2.3 From 7a59b56e53020e655f83d65c165dfb8c228ddf4d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 14:20:47 -0700 Subject: sketching pollable set --- src/core/lib/iomgr/ev_epollex_linux.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index a8d7b54887..0b3e2b13fc 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -177,7 +177,11 @@ struct grpc_pollset_worker { struct grpc_pollset { polling_obj po; - int epfd; + /* Pollable set - possible values: + 0 - nothing is pollable + pointer | 1 - a single pollable file descriptor + (fd << 1) | 0 - an epoll fd */ + gpr_atm pollable_set_atm; int num_pollers; bool kicked_without_poller; gpr_atm shutdown_atm; -- cgit v1.2.3 From 9f012514738c560df9ccd72971390a5b097256c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 15:37:14 -0700 Subject: Working towards a single-fd optimized data structure --- src/core/lib/iomgr/ev_epollex_linux.c | 161 +++++++++++++++++++++++----------- 1 file changed, 109 insertions(+), 52 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0b3e2b13fc..6e6e4460c0 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -170,24 +170,27 @@ static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { struct grpc_pollset_worker { bool kicked; bool initialized_cv; + bool inserted; gpr_cv cv; grpc_pollset_worker *next; grpc_pollset_worker *prev; + grpc_pollset *pollset; }; -struct grpc_pollset { +struct pollable { polling_obj po; - /* Pollable set - possible values: - 0 - nothing is pollable - pointer | 1 - a single pollable file descriptor - (fd << 1) | 0 - an epoll fd */ - gpr_atm pollable_set_atm; + int epfd; + grpc_wakeup_fd wakeup; + grpc_pollset_worker *root_worker; int num_pollers; - bool kicked_without_poller; gpr_atm shutdown_atm; +}; + +struct grpc_pollset { + pollable pollable; + pollable *current_pollable; + bool kicked_without_poller; grpc_closure *shutdown_closure; - grpc_wakeup_fd pollset_wakeup; - grpc_pollset_worker *root_worker; }; /******************************************************************************* @@ -573,32 +576,53 @@ static grpc_error *kick_poller(void) { static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { po_init(&pollset->po, PO_POLLSET); pollset->kicked_without_poller = false; - pollset->epfd = epoll_create1(EPOLL_CLOEXEC); - if (pollset->epfd < 0) { - GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); + gpr_atm_no_barrier_store(&pollset->pollable_set_atm, 0); + pollset->num_pollers = 0; + gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); + pollset->shutdown_closure = NULL; + pollset->root_worker = NULL; + *mu = &pollset->po.mu; +} + +static grpc_error *multipoller_create(multipoller **out) { + multipoller *p = gpr_malloc(sizeof(*p)); + p->epfd = epoll_create1(EPOLL_CLOEXEC); + if (p->epfd < 0) { + grpc_error *err = GRPC_OS_ERROR(errno, "epoll_create1"); + gpr_free(p); + return err; } else { struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, .data.ptr = &global_wakeup_fd}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, - &ev) != 0) { - GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); + if (epoll_ctl(p->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { + grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(p->epfd); + gpr_free(p); + return err; } } - pollset->num_pollers = 0; - gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); - pollset->shutdown_closure = NULL; - if (GRPC_LOG_IF_ERROR("pollset_init", - grpc_wakeup_fd_init(&pollset->pollset_wakeup)) && - pollset->epfd >= 0) { - struct epoll_event ev = {.events = EPOLLIN | EPOLLET, - .data.ptr = &pollset->pollset_wakeup}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, pollset->pollset_wakeup.read_fd, - &ev) != 0) { - GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); - } + grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); + if (err != GRPC_ERROR_NONE) { + close(p->epfd); + gpr_free(p); + return err; } - pollset->root_worker = NULL; - *mu = &pollset->po.mu; + struct epoll_event ev = {.events = EPOLLIN | EPOLLET, .data.ptr = &p->wakeup}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { + err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + gpr_free(p); + return err; + } + *out = p; + return GRPC_ERROR_NONE; +} + +static void multipoller_destroy(multipoller *p) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + gpr_free(p); } /* Convert a timespec to milliseconds: @@ -678,31 +702,40 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { po_destroy(&pollset->po); - if (pollset->epfd >= 0) close(pollset->epfd); - grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); + switch (pollset->occupancy) { + case POLLSET_EMPTY: + break; + case POLLSET_UNARY_FD: + UNREF_BY(pollset->pollable.unary_fd, 2); + break; + case POLLSET_MULTIPOLLER: + multipoller_destroy(pollset->pollable.multipoller); + break; + } } #define MAX_EPOLL_EVENTS 100 -static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - gpr_timespec now, gpr_timespec deadline) { +static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { struct epoll_event events[MAX_EPOLL_EVENTS]; static const char *err_desc = "pollset_poll"; - if (pollset->epfd < 0) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "epoll fd failed to initialize"); - } - - GRPC_SCHEDULING_START_BLOCKING_REGION; int timeout = poll_deadline_to_millis_timeout(deadline, now); if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p poll for %dms", pollset, timeout); } - int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, timeout); - GRPC_SCHEDULING_END_BLOCKING_REGION; + if (timeout != 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; + } + int r = epoll_wait(pollset->pollable.multipoller->epfd, events, + MAX_EPOLL_EVENTS, timeout); + if (timeout != 0) { + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); if (grpc_polling_trace) { @@ -720,7 +753,7 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); - } else if (data_ptr == &pollset->pollset_wakeup) { + } else if (data_ptr == &pollset->pollable.multipoller->pollset_wakeup) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); } @@ -728,7 +761,9 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, the fd is level triggered and non-exclusive, which should result in all pollers waking */ if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { - append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + append_error(&error, + grpc_wakeup_fd_consume_wakeup( + &pollset->pollable.multipoller->pollset_wakeup), err_desc); } } else { @@ -767,20 +802,29 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec deadline) { - if (worker_hdl != NULL) { - *worker_hdl = worker; + worker->initialized_cv = false; + worker->inserted = false; + if (worker_hdl != NULL || pollset->occupancy != POLLSET_MULTIPOLLER) { + if (worker_hdl != NULL) *worker_hdl = worker; worker->kicked = false; + worker->inserted = true; if (pollset->root_worker == NULL) { pollset->root_worker = worker; worker->next = worker->prev = worker; - worker->initialized_cv = false; + if (pollset->occupancy == POLLSET_EMPTY) { + worker->initialized_cv = true; + } } else { worker->next = pollset->root_worker; worker->prev = worker->next->prev; worker->next->prev = worker->prev->next = worker; worker->initialized_cv = true; + } + if (worker->initialized_cv) { + GPR_ASSERT(worker->inserted); gpr_cv_init(&worker->cv); - while (pollset->root_worker != worker) { + while (pollset->root_worker != worker || + pollset->occupancy == POLLSET_EMPTY) { if (gpr_cv_wait(&worker->cv, &pollset->po.mu, deadline)) return false; if (worker->kicked) return false; } @@ -791,7 +835,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (worker_hdl != NULL) { + if (worker->inserted) { if (worker == pollset->root_worker) { if (worker == worker->next) { pollset->root_worker = NULL; @@ -819,6 +863,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; + grpc_pollset_worker *fake_worker_hdl; if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", @@ -836,10 +881,22 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); pollset->num_pollers++; - gpr_mu_unlock(&pollset->po.mu); - error = pollset_poll(exec_ctx, pollset, now, deadline); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); + switch (pollset->occupancy) { + case POLLSET_EMPTY: + GPR_UNREACHABLE_CODE(break); + case POLLSET_UNARY_FD: + gpr_mu_unlock(&pollset->po.mu); + error = pollset_poll(exec_ctx, pollset, now, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + break; + case POLLSET_MULTIPOLLER: + gpr_mu_unlock(&pollset->po.mu); + error = pollset_epoll(exec_ctx, pollset, now, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + break; + } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset->num_pollers--; -- cgit v1.2.3 From d1d7fdd791e2ad62427a1bbe7d918d8fd822d8d4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 16:16:24 -0700 Subject: Sketch turnstile with single-fd-poller optimization --- src/core/lib/iomgr/ev_epollex_linux.c | 594 ++++++++++++++++++++-------------- 1 file changed, 358 insertions(+), 236 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 6e6e4460c0..d49654bbc2 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -77,10 +77,11 @@ static grpc_wakeup_fd global_wakeup_fd; */ typedef enum { - PO_FD, - PO_POLLSET, - PO_POLLSET_SET, PO_POLLING_GROUP, + PO_POLLSET_SET, + PO_POLLSET, + PO_FD, /* ordering is important: we always want to lock pollsets before fds: + this guarantees that using an fd as a pollable is safe */ PO_COUNT } polling_obj_type; @@ -103,6 +104,7 @@ struct polling_group { static void po_init(polling_obj *po, polling_obj_type type); static void po_destroy(polling_obj *po); static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b); +static int po_cmp(polling_obj *a, polling_obj *b); static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, size_t initial_po_count); @@ -113,12 +115,30 @@ static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, polling_obj *po); +/******************************************************************************* + * pollable Declarations + */ + +typedef struct pollable { + polling_obj po; + int epfd; + grpc_wakeup_fd wakeup; + grpc_pollset_worker *root_worker; +} pollable; + +static pollable g_empty_pollable; + +static void pollable_init(pollable *p, polling_obj_type type); +static void pollable_destroy(pollable *p); +/* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ +static grpc_error *pollable_materialize(pollable *p); + /******************************************************************************* * Fd Declarations */ struct grpc_fd { - polling_obj po; + pollable pollable; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -167,23 +187,25 @@ static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { /******************************************************************************* * Pollset Declarations */ + +typedef struct pollset_worker_link { + grpc_pollset_worker *next; + grpc_pollset_worker *prev; +} pollset_worker_link; + +typedef enum { + PWL_POLLSET, + PWL_POLLABLE, + POLLSET_WORKER_LINK_COUNT +} pollset_worker_links; + struct grpc_pollset_worker { bool kicked; bool initialized_cv; - bool inserted; + pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; - grpc_pollset_worker *next; - grpc_pollset_worker *prev; grpc_pollset *pollset; -}; - -struct pollable { - polling_obj po; - int epfd; - grpc_wakeup_fd wakeup; - grpc_pollset_worker *root_worker; - int num_pollers; - gpr_atm shutdown_atm; + pollable *pollable; }; struct grpc_pollset { @@ -191,6 +213,7 @@ struct grpc_pollset { pollable *current_pollable; bool kicked_without_poller; grpc_closure *shutdown_closure; + grpc_pollset_worker *root_worker; }; /******************************************************************************* @@ -270,7 +293,7 @@ static void unref_by(grpc_fd *fd, int n) { if (old == n) { /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); - po_destroy(&fd->po); + pollable_destroy(&fd->pollable); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; @@ -311,7 +334,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = gpr_malloc(sizeof(grpc_fd)); } - po_init(&new_fd->po, PO_FD); + pollable_init(&new_fd->pollable, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -342,11 +365,11 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { int ret_fd = -1; - gpr_mu_lock(&fd->po.mu); + gpr_mu_lock(&fd->pollable.po.mu); if (!fd->orphaned) { ret_fd = fd->fd; } - gpr_mu_unlock(&fd->po.mu); + gpr_mu_unlock(&fd->pollable.po.mu); return ret_fd; } @@ -357,7 +380,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->po.mu); + gpr_mu_lock(&fd->pollable.po.mu); fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file @@ -381,7 +404,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); - gpr_mu_unlock(&fd->po.mu); + gpr_mu_unlock(&fd->pollable.po.mu); UNREF_BY(fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -499,9 +522,94 @@ static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { return &((grpc_fd *)workqueue)->workqueue_scheduler; } +/******************************************************************************* + * Pollable Definitions + */ + +static void pollable_init(pollable *p, polling_obj_type type) { + po_init(&p->po, type); + p->root_worker = NULL; + p->epfd = -1; +} + +static void pollable_destroy(pollable *p) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + po_destroy(&p->po); +} + +/* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ +static grpc_error *pollable_materialize(pollable *p) { + if (p->epfd == -1) { + int new_epfd = epoll_create1(EPOLL_CLOEXEC); + if (new_epfd < 0) { + return GRPC_OS_ERROR(errno, "epoll_create1"); + } else { + struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != + 0) { + grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(new_epfd); + return err; + } + } + grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); + if (err != GRPC_ERROR_NONE) { + close(new_epfd); + return err; + } + struct epoll_event ev = {.events = EPOLLIN | EPOLLET, + .data.ptr = &p->wakeup}; + if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { + err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(new_epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + return err; + } + + p->epfd = new_epfd; + } + return GRPC_ERROR_NONE; +} + +/* pollable must be materialized */ +static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollable_add_fd"; + const int epfd = p->epfd; + + struct epoll_event ev_fd = { + .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { + switch (errno) { + case EEXIST: /* if this fd is already in the epoll set, the workqueue fd + must also be - just return */ + return GRPC_ERROR_NONE; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, + .data.ptr = (void *)(1 + (intptr_t)fd)}; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != + 0) { + switch (errno) { + case EEXIST: /* if the workqueue fd is already in the epoll set we're ok + - no need to do anything special */ + break; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + + return error; +} + /******************************************************************************* * Pollset Definitions */ + GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); @@ -518,51 +626,78 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } +static grpc_error *pollset_kick_all(grpc_pollset *pollset) { + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->root_worker != NULL) { + grpc_pollset_worker *worker = pollset->root_worker; + do { + if (worker->initialized_cv) { + gpr_cv_signal(&worker->cv); + } else { + append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), + "pollset_shutdown"); + } + + worker = worker->links[PWL_POLLSET].next; + } while (worker != pollset->root_worker); + } + return error; +} + /* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable) { + gpr_mu_lock(&p->po.mu); + } + if (grpc_polling_trace) { gpr_log(GPR_DEBUG, - "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d " - "root_worker=%p", + "PS:%p kick %p tls_pollset=%p tls_worker=%p " + "root_worker=(pollset:%p pollable:%p)", p, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset), - (void *)gpr_tls_get(&g_current_thread_worker), p->num_pollers, + (void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker, p->root_worker); } if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { - if (p->num_pollers == 0) { + if (pollset->root_worker == NULL) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_without_poller", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); } - p->kicked_without_poller = true; + pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; } else { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_any_via_wakeup_fd", p); } - return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + grpc_error *err = pollable_materialize(p); + if (err != GRPC_ERROR_NONE) return err; + return grpc_wakeup_fd_wakeup(&p->wakeup); } } else { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_any_but_awake", p); } return GRPC_ERROR_NONE; } } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); } return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_wakeup_fd", p); } - return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + grpc_error *err = pollable_materialize(p); + if (err != GRPC_ERROR_NONE) return err; + return grpc_wakeup_fd_wakeup(&p->wakeup); } else { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_via_cv", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); } gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; @@ -574,55 +709,12 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - po_init(&pollset->po, PO_POLLSET); + pollable_init(&pollset->pollable, PO_POLLSET); + pollset->current_pollable = &g_empty_pollable; pollset->kicked_without_poller = false; - gpr_atm_no_barrier_store(&pollset->pollable_set_atm, 0); - pollset->num_pollers = 0; - gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); pollset->shutdown_closure = NULL; pollset->root_worker = NULL; - *mu = &pollset->po.mu; -} - -static grpc_error *multipoller_create(multipoller **out) { - multipoller *p = gpr_malloc(sizeof(*p)); - p->epfd = epoll_create1(EPOLL_CLOEXEC); - if (p->epfd < 0) { - grpc_error *err = GRPC_OS_ERROR(errno, "epoll_create1"); - gpr_free(p); - return err; - } else { - struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, - .data.ptr = &global_wakeup_fd}; - if (epoll_ctl(p->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { - grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); - close(p->epfd); - gpr_free(p); - return err; - } - } - grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); - if (err != GRPC_ERROR_NONE) { - close(p->epfd); - gpr_free(p); - return err; - } - struct epoll_event ev = {.events = EPOLLIN | EPOLLET, .data.ptr = &p->wakeup}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { - err = GRPC_OS_ERROR(errno, "epoll_ctl"); - close(p->epfd); - grpc_wakeup_fd_destroy(&p->wakeup); - gpr_free(p); - return err; - } - *out = p; - return GRPC_ERROR_NONE; -} - -static void multipoller_destroy(multipoller *p) { - close(p->epfd); - grpc_wakeup_fd_destroy(&p->wakeup); - gpr_free(p); + *mu = &pollset->pollable.po.mu; } /* Convert a timespec to milliseconds: @@ -667,9 +759,20 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } +static grpc_error *fd_become_pollable(grpc_fd *fd) { + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "fd_become_pollable"; + gpr_mu_lock(&fd->pollable.po.mu); + if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { + append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); + } + gpr_mu_unlock(&fd->pollable.po.mu); + return error; +} + static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); } } @@ -679,45 +782,27 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { GPR_ASSERT(pollset->shutdown_closure == NULL); pollset->shutdown_closure = closure; - gpr_atm_no_barrier_store(&pollset->shutdown_atm, 1); - if (pollset->num_pollers > 0) { - struct epoll_event ev = {.events = EPOLLIN, - .data.ptr = &pollset->pollset_wakeup}; - epoll_ctl(pollset->epfd, EPOLL_CTL_MOD, pollset->pollset_wakeup.read_fd, - &ev); - GRPC_LOG_IF_ERROR("pollset_shutdown", - grpc_wakeup_fd_wakeup(&pollset->pollset_wakeup)); - } - if (pollset->root_worker != NULL) { - for (grpc_pollset_worker *worker = pollset->root_worker->next; - worker != pollset->root_worker; worker = worker->next) { - if (worker->initialized_cv) { - gpr_cv_signal(&worker->cv); - } - } - } + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); pollset_maybe_finish_shutdown(exec_ctx, pollset); } +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { + return p != &g_empty_pollable && p != &pollset->pollable; +} + /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { - po_destroy(&pollset->po); - switch (pollset->occupancy) { - case POLLSET_EMPTY: - break; - case POLLSET_UNARY_FD: - UNREF_BY(pollset->pollable.unary_fd, 2); - break; - case POLLSET_MULTIPOLLER: - multipoller_destroy(pollset->pollable.multipoller); - break; + pollable_destroy(&pollset->pollable); + if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { + UNREF_BY((grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); } } #define MAX_EPOLL_EVENTS 100 static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - gpr_timespec now, gpr_timespec deadline) { + pollable *p, gpr_timespec now, + gpr_timespec deadline) { struct epoll_event events[MAX_EPOLL_EVENTS]; static const char *err_desc = "pollset_poll"; @@ -730,8 +815,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (timeout != 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; } - int r = epoll_wait(pollset->pollable.multipoller->epfd, events, - MAX_EPOLL_EVENTS, timeout); + int r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); if (timeout != 0) { GRPC_SCHEDULING_END_BLOCKING_REGION; } @@ -753,19 +837,11 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); - } else if (data_ptr == &pollset->pollable.multipoller->pollset_wakeup) { + } else if (data_ptr == &p->wakeup) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); } - /* once we start shutting down we stop consuming the wakeup: - the fd is level triggered and non-exclusive, which should result in all - pollers waking */ - if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { - append_error(&error, - grpc_wakeup_fd_consume_wakeup( - &pollset->pollable.multipoller->pollset_wakeup), - err_desc); - } + append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); } else { grpc_fd *fd = (grpc_fd *)(((intptr_t)data_ptr) & ~(intptr_t)1); bool is_workqueue = (((intptr_t)data_ptr) & 1) != 0; @@ -798,60 +874,89 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } +/* Return true if first in list */ +static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, + grpc_pollset_worker *worker) { + if (*root == NULL) { + *root = worker; + worker->links[link].next = worker->links[link].prev = worker; + return true; + } else { + worker->links[link].next = *root; + worker->links[link].prev = worker->links[link].next->links[link].prev; + worker->links[link].next->links[link].prev = worker; + worker->links[link].prev->links[link].next = worker; + return false; + } +} + +/* Return true if last in list */ +typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; + +static worker_remove_result worker_remove(grpc_pollset_worker **root, + pollset_worker_links link, + grpc_pollset_worker *worker) { + if (worker == *root) { + if (worker == worker->links[link].next) { + *root = NULL; + return EMPTIED; + } else { + *root = worker->links[link].next; + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return NEW_ROOT; + } + } else { + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return REMOVED; + } +} + /* Return true if this thread should poll */ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec deadline) { + if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; - worker->inserted = false; - if (worker_hdl != NULL || pollset->occupancy != POLLSET_MULTIPOLLER) { - if (worker_hdl != NULL) *worker_hdl = worker; - worker->kicked = false; - worker->inserted = true; - if (pollset->root_worker == NULL) { - pollset->root_worker = worker; - worker->next = worker->prev = worker; - if (pollset->occupancy == POLLSET_EMPTY) { - worker->initialized_cv = true; + worker->kicked = false; + worker->pollset = pollset; + worker->pollable = pollset->current_pollable; + + if (pollset_is_pollable_fd(pollset, worker->pollable)) { + REF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + } + + worker_insert(&pollset->root_worker, PWL_POLLSET, worker); + if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (pollset->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &pollset->current_pollable->po.mu, + deadline)) { + return false; } - } else { - worker->next = pollset->root_worker; - worker->prev = worker->next->prev; - worker->next->prev = worker->prev->next = worker; - worker->initialized_cv = true; - } - if (worker->initialized_cv) { - GPR_ASSERT(worker->inserted); - gpr_cv_init(&worker->cv); - while (pollset->root_worker != worker || - pollset->occupancy == POLLSET_EMPTY) { - if (gpr_cv_wait(&worker->cv, &pollset->po.mu, deadline)) return false; - if (worker->kicked) return false; + if (worker->kicked) { + return false; } } } + return pollset->shutdown_closure == NULL; } static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (worker->inserted) { - if (worker == pollset->root_worker) { - if (worker == worker->next) { - pollset->root_worker = NULL; - } else { - pollset->root_worker = worker->next; - worker->prev->next = worker->next; - worker->next->prev = worker->prev; - gpr_cv_signal(&pollset->root_worker->cv); - } - } else { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; - } - if (worker->initialized_cv) { - gpr_cv_destroy(&worker->cv); - } + worker_remove(&pollset->root_worker, PWL_POLLSET, worker); + if (NEW_ROOT == + worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&worker->pollable->root_worker->cv); + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + if (pollset_is_pollable_fd(pollset, worker->pollable)) { + UNREF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); } } @@ -863,7 +968,6 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - grpc_pollset_worker *fake_worker_hdl; if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", @@ -872,67 +976,71 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->root_worker); } grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_work"; if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } + if (pollset->current_pollable != &pollset->pollable) { + gpr_mu_lock(&pollset->current_pollable->po.mu); + } if (begin_worker(pollset, &worker, worker_hdl, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - pollset->num_pollers++; - switch (pollset->occupancy) { - case POLLSET_EMPTY: - GPR_UNREACHABLE_CODE(break); - case POLLSET_UNARY_FD: - gpr_mu_unlock(&pollset->po.mu); - error = pollset_poll(exec_ctx, pollset, now, deadline); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); - break; - case POLLSET_MULTIPOLLER: - gpr_mu_unlock(&pollset->po.mu); - error = pollset_epoll(exec_ctx, pollset, now, deadline); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); - break; + append_error(&error, pollable_materialize(worker.pollable), err_desc); + if (worker.pollable != &pollset->pollable) { + gpr_mu_unlock(&worker.pollable->po.mu); + } + gpr_mu_unlock(&pollset->pollable.po.mu); + append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable, now, + deadline), + err_desc); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->pollable.po.mu); + if (worker.pollable != &pollset->pollable) { + gpr_mu_lock(&worker.pollable->po.mu); } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); - pollset->num_pollers--; pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(pollset, &worker, worker_hdl); + if (worker.pollable != &pollset->pollable) { + gpr_mu_unlock(&worker.pollable->po.mu); + } return error; } static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; + grpc_fd *unref_fd = NULL; static const char *err_desc = "pollset_add_fd"; - struct epoll_event ev_fd = { - .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { - switch (errno) { - case EEXIST: /* if this fd is already in the epoll set, the workqueue fd - must also be - just return */ - return; - default: - append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + gpr_mu_lock(&pollset->pollable.po.mu); + if (pollset->current_pollable == &g_empty_pollable) { + /* empty pollable --> single fd pollable */ + append_error(&error, pollset_kick_all(pollset), err_desc); + pollset->current_pollable = &fd->pollable; + append_error(&error, fd_become_pollable(fd), err_desc); + REF_BY(fd, 2, "pollset_pollable"); + } else if (pollset->current_pollable == &pollset->pollable) { + append_error(&error, pollable_add_fd(pollset->current_pollable, fd), + err_desc); + } else if (pollset->current_pollable != &fd->pollable) { + unref_fd = (grpc_fd *)pollset->current_pollable; + pollset->current_pollable = &pollset->pollable; + if (append_error(&error, pollable_materialize(&pollset->pollable), + err_desc)) { + pollable_add_fd(&pollset->pollable, unref_fd); + pollable_add_fd(&pollset->pollable, fd); } } - struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = (void *)(1 + (intptr_t)fd)}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, - &ev_wq) != 0) { - switch (errno) { - case EEXIST: /* if the workqueue fd is already in the epoll set we're ok - - no need to do anything special */ - break; - default: - append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); - } + gpr_mu_unlock(&pollset->pollable.po.mu); + if (unref_fd) { + UNREF_BY(unref_fd, 2, "pollset_pollable"); } + GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -954,7 +1062,7 @@ static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - po_join(exec_ctx, &pss->po, &fd->po); + po_join(exec_ctx, &pss->po, &fd->pollable.po); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, @@ -962,7 +1070,7 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - po_join(exec_ctx, &pss->po, &ps->po); + po_join(exec_ctx, &pss->po, &ps->pollable.po); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, @@ -1022,40 +1130,54 @@ static void pg_unref(polling_group *pg) { } } +static int po_cmp(polling_obj *a, polling_obj *b) { + if (a == b) return 0; + if (a->type < b->type) return -1; + if (a->type > b->type) return 1; + if (a < b) return -1; + assert(a > b); + return 1; +} + static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { - if (a == b) return; - if (a > b) GPR_SWAP(polling_obj *, a, b); - - gpr_mu_lock(&a->mu); - gpr_mu_lock(&b->mu); - - if (a->group == NULL) { - if (b->group == NULL) { - polling_obj *initial_po[] = {a, b}; - pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - } else { - polling_group *b_group = pg_ref(b->group); - gpr_mu_unlock(&b->mu); - gpr_mu_unlock(&a->mu); - pg_join(exec_ctx, b_group, a); - } - } else if (b->group == NULL) { - polling_group *a_group = pg_ref(a->group); - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - pg_join(exec_ctx, a_group, b); - } else if (a->group == b->group) { - /* nothing to do */ - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - } else { - polling_group *a_group = pg_ref(a->group); - polling_group *b_group = pg_ref(b->group); - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - pg_merge(exec_ctx, a_group, b_group); + switch (po_cmp(a, b)) { + case 0: + return; + case 1: + GPR_SWAP(polling_obj *, a, b); + /* fall through */ + case -1: + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + + if (a->group == NULL) { + if (b->group == NULL) { + polling_obj *initial_po[] = {a, b}; + pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + pg_join(exec_ctx, b_group, a); + } + } else if (b->group == NULL) { + polling_group *a_group = pg_ref(a->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_join(exec_ctx, a_group, b); + } else if (a->group == b->group) { + /* nothing to do */ + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *a_group = pg_ref(a->group); + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_merge(exec_ctx, a_group, b_group); + } } } -- cgit v1.2.3 From 911490c0d4edf4804304121cf63f81a6336b66cf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 16:19:57 -0700 Subject: Finish initialization code --- src/core/lib/iomgr/ev_epollex_linux.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d49654bbc2..d01aeb9c82 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -82,6 +82,7 @@ typedef enum { PO_POLLSET, PO_FD, /* ordering is important: we always want to lock pollsets before fds: this guarantees that using an fd as a pollable is safe */ + PO_EMPTY_POLLABLE, PO_COUNT } polling_obj_type; @@ -612,16 +613,23 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); +static bool global_wakeup_fd_initialized = false; /* Global state management */ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_global_init"; + global_wakeup_fd_initialized = + append_error(&error, grpc_wakeup_fd_init(&global_wakeup_fd), err_desc); + pollable_init(&g_empty_pollable, PO_EMPTY_POLLABLE); + return error; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); + if (global_wakeup_fd_initialized) grpc_wakeup_fd_destroy(&global_wakeup_fd); + pollable_destroy(&g_empty_pollable); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } -- cgit v1.2.3 From 54d388d0af69aae8ba7716da7f6c3bc8b1768664 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 16:20:45 -0700 Subject: init tweaks --- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d01aeb9c82..0918b22e46 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1438,6 +1438,8 @@ const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + pollset_global_shutdown(); + fd_global_shutdown(); return NULL; } -- cgit v1.2.3 From 29a256773a23a530814d9b28d26ab99674de59c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 15 Apr 2017 00:00:39 +0000 Subject: Consume the orrect fd --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index db35e0e11f..4ec9774b0c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -729,7 +729,7 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, the fd is level triggered and non-exclusive, which should result in all pollers waking */ if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { - append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + append_error(&error, grpc_wakeup_fd_consume_wakeup(&pollset->pollset_wakeup), err_desc); } } else { -- cgit v1.2.3 From 26017de5d55f3ec5002afa0b97c7fe660d6144bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 15 Apr 2017 00:07:25 +0000 Subject: Fix leaked lock --- src/core/lib/iomgr/ev_epollex_linux.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0918b22e46..2c0fe42acc 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -652,14 +652,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { return error; } -/* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *pollset, - grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; - if (p != &pollset->pollable) { - gpr_mu_lock(&p->po.mu); - } - +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p " @@ -710,6 +703,21 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } + +} + +/* p->po.mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable) { + gpr_mu_lock(&p->po.mu); + } + grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + if (p != &pollset->pollable) { + gpr_mu_unlock(&p->po.mu); + } + return error; } static grpc_error *kick_poller(void) { -- cgit v1.2.3 From e310f5b6230ab38b0c2e9aafb4caf1aa1fc45faa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 15:43:58 +0000 Subject: Fix scheduled bit --- src/core/lib/iomgr/ev_epollex_linux.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 2c0fe42acc..eba1bf920c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -509,6 +509,9 @@ static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); } else if (gpr_atm_no_barrier_load(&fd->workqueue_item_count) > 0) { -- cgit v1.2.3 From 50480b2a058fc74c3cf934ae5ae9981b2417005e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 16:34:52 +0000 Subject: Fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 130 ++++++++++++++++++++++------------ src/core/lib/iomgr/pollset.h | 2 +- 2 files changed, 86 insertions(+), 46 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index eba1bf920c..fdde21756b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -265,7 +265,7 @@ static gpr_mu fd_freelist_mu; #ifdef GRPC_FD_REF_COUNT_DEBUG #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n, reason, __FILE__, __LINE__) static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, int line) { gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, @@ -273,25 +273,14 @@ static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); #else #define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) +#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n) static void ref_by(grpc_fd *fd, int n) { #endif GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); } -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { +static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { +grpc_fd *fd = arg; /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); pollable_destroy(&fd->pollable); @@ -303,6 +292,22 @@ static void unref_by(grpc_fd *fd, int n) { grpc_lfev_destroy(&fd->write_closure); gpr_mu_unlock(&fd_freelist_mu); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); } else { GPR_ASSERT(old > n); } @@ -406,7 +411,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->pollable.po.mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ + UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); } @@ -459,7 +464,7 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, const char *file, int line, const char *reason) { if (workqueue != NULL) { - unref_by((grpc_fd *)workqueue, 2, file, line, reason); + unref_by(exec_ctx, (grpc_fd *)workqueue, 2, file, line, reason); } } #else @@ -473,7 +478,7 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) { if (workqueue != NULL) { - unref_by((grpc_fd *)workqueue, 2); + unref_by(exec_ctx, (grpc_fd *)workqueue, 2); } } #endif @@ -495,7 +500,7 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, if (last == 0) { workqueue_wakeup(fd); } - UNREF_BY(fd, 2, "workqueue_enqueue"); + UNREF_BY(exec_ctx, fd, 2, "workqueue_enqueue"); } static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { @@ -778,14 +783,12 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static grpc_error *fd_become_pollable(grpc_fd *fd) { +static grpc_error *fd_become_pollable_locked(grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "fd_become_pollable"; - gpr_mu_lock(&fd->pollable.po.mu); if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); } - gpr_mu_unlock(&fd->pollable.po.mu); return error; } @@ -810,10 +813,10 @@ static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { } /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { pollable_destroy(&pollset->pollable); if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY((grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); } } @@ -975,7 +978,7 @@ static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_cv_destroy(&worker->cv); } if (pollset_is_pollable_fd(pollset, worker->pollable)) { - UNREF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); } } @@ -1031,35 +1034,45 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { - grpc_error *error = GRPC_ERROR_NONE; - grpc_fd *unref_fd = NULL; +static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + grpc_fd *fd = arg; + UNREF_BY(exec_ctx, fd, 2, "pollset_pollable"); +} + +/* expects pollsets locked, flag whether fd is locked or not */ +static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd, bool fd_locked) { static const char *err_desc = "pollset_add_fd"; - gpr_mu_lock(&pollset->pollable.po.mu); + grpc_error *error = GRPC_ERROR_NONE; if (pollset->current_pollable == &g_empty_pollable) { /* empty pollable --> single fd pollable */ append_error(&error, pollset_kick_all(pollset), err_desc); pollset->current_pollable = &fd->pollable; - append_error(&error, fd_become_pollable(fd), err_desc); + if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); + append_error(&error, fd_become_pollable_locked(fd), err_desc); + if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); REF_BY(fd, 2, "pollset_pollable"); } else if (pollset->current_pollable == &pollset->pollable) { append_error(&error, pollable_add_fd(pollset->current_pollable, fd), err_desc); } else if (pollset->current_pollable != &fd->pollable) { - unref_fd = (grpc_fd *)pollset->current_pollable; + grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; pollset->current_pollable = &pollset->pollable; if (append_error(&error, pollable_materialize(&pollset->pollable), err_desc)) { - pollable_add_fd(&pollset->pollable, unref_fd); + pollable_add_fd(&pollset->pollable, had_fd); pollable_add_fd(&pollset->pollable, fd); } + grpc_closure_sched(exec_ctx, grpc_closure_create(unref_fd_no_longer_poller, had_fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); } - gpr_mu_unlock(&pollset->pollable.po.mu); - if (unref_fd) { - UNREF_BY(unref_fd, 2, "pollset_pollable"); - } + return error; +} +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + gpr_mu_lock(&pollset->pollable.po.mu); + grpc_error *error = pollset_add_fd_locked(exec_ctx, pollset, fd, false); + gpr_mu_unlock(&pollset->pollable.po.mu); GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -1202,9 +1215,9 @@ static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { if (a->type == PO_FD && b->type == PO_POLLSET) { - pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); + pollset_add_fd_locked(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a, true); } else if (a->type == PO_POLLSET && b->type == PO_FD) { - pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); + pollset_add_fd_locked(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b, true); } } @@ -1212,7 +1225,14 @@ static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, polling_group *to) { for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { + if (po_cmp(a, b) < 0) { + gpr_mu_lock(&a->mu); gpr_mu_lock(&b->mu); + } else { + GPR_ASSERT(po_cmp(a, b) != 0); + gpr_mu_lock(&b->mu); gpr_mu_lock(&a->mu); + } pg_notify(exec_ctx, a, b); + gpr_mu_unlock(&a->mu); gpr_mu_unlock(&b->mu); } } } @@ -1249,21 +1269,41 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, /* assumes neither pg nor po are locked; consumes one ref to pg */ pg = pg_lock_latest(pg); /* pg locked */ + for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; + existing != &pg->po; existing = existing->next) { + if (po_cmp(po, existing) < 0) { + gpr_mu_lock(&po->mu); + gpr_mu_lock(&existing->mu); + } else { + GPR_ASSERT(po_cmp(po, existing) != 0); + gpr_mu_lock(&existing->mu); + gpr_mu_lock(&po->mu); + } + /* pg, po, existing locked */ + if (po->group != NULL) { + gpr_mu_unlock(&pg->po.mu); + polling_group *po_group = pg_ref(po->group); + gpr_mu_unlock(&po->mu); + gpr_mu_unlock(&existing->mu); + pg_merge(exec_ctx, pg, po_group); + /* early exit: polling obj picked up a group during joining: we needed + to do a full merge */ + return; + } + pg_notify(exec_ctx, po, existing); + gpr_mu_unlock(&po->mu); + gpr_mu_unlock(&existing->mu); + } gpr_mu_lock(&po->mu); if (po->group != NULL) { gpr_mu_unlock(&pg->po.mu); polling_group *po_group = pg_ref(po->group); gpr_mu_unlock(&po->mu); pg_merge(exec_ctx, pg, po_group); - /* early exit: polling obj picked up a group before joining: we needed + /* early exit: polling obj picked up a group during joining: we needed to do a full merge */ return; } - /* pg, po locked */ - for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; - existing != &pg->po; existing = existing->next) { - pg_notify(exec_ctx, po, existing); - } po->group = pg; po->next = &pg->po; po->prev = pg->po.prev; diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index 6f3a51e717..69e20098d7 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -57,7 +57,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); * pollset's mutex must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); -void grpc_pollset_destroy(grpc_pollset *pollset); +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); /* Do some work on a pollset. May involve invoking asynchronous callbacks, or actually polling file -- cgit v1.2.3 From f840110de243ae225f5825c0dda47e9fe89b582c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 09:47:28 -0700 Subject: Fixup tests --- src/core/lib/iomgr/ev_epoll_linux.c | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 65 +++++++++++++--------- src/core/lib/iomgr/ev_poll_posix.c | 2 +- src/core/lib/iomgr/ev_posix.c | 4 +- src/core/lib/iomgr/ev_posix.h | 2 +- .../google_default/google_default_credentials.c | 2 +- src/core/lib/surface/alarm.c | 4 +- src/core/lib/surface/call.c | 2 +- src/core/lib/surface/completion_queue.c | 19 ++++--- src/core/lib/surface/completion_queue.h | 12 ++-- src/core/lib/surface/server.c | 2 +- test/core/end2end/fixtures/http_proxy_fixture.c | 2 +- test/core/http/httpcli_test.c | 2 +- test/core/http/httpscli_test.c | 2 +- test/core/iomgr/endpoint_pair_test.c | 2 +- test/core/iomgr/ev_epoll_linux_test.c | 2 +- test/core/iomgr/fd_posix_test.c | 2 +- test/core/iomgr/pollset_set_test.c | 2 +- test/core/iomgr/resolve_address_posix_test.c | 2 +- test/core/iomgr/resolve_address_test.c | 2 +- test/core/iomgr/tcp_client_posix_test.c | 2 +- test/core/iomgr/tcp_posix_test.c | 2 +- test/core/iomgr/tcp_server_posix_test.c | 2 +- test/core/iomgr/udp_server_test.c | 2 +- test/core/security/secure_endpoint_test.c | 2 +- test/core/surface/concurrent_connectivity_test.c | 2 +- test/core/util/port_server_client.c | 2 +- test/core/util/test_tcp_server.c | 9 ++- 28 files changed, 90 insertions(+), 67 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 766891169f..0f1bd833f7 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1333,7 +1333,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. So other * than destroying the mutexes, there is nothing special that needs to be done * here */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); gpr_mu_destroy(&pollset->po.mu); } diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index fdde21756b..f0b9ee39f6 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -265,7 +265,8 @@ static gpr_mu fd_freelist_mu; #ifdef GRPC_FD_REF_COUNT_DEBUG #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(ec, fd, n, reason) \ + unref_by(ec, fd, n, reason, __FILE__, __LINE__) static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, int line) { gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, @@ -280,23 +281,23 @@ static void ref_by(grpc_fd *fd, int n) { } static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { -grpc_fd *fd = arg; - /* Add the fd to the freelist */ - grpc_iomgr_unregister_object(&fd->iomgr_object); - pollable_destroy(&fd->pollable); - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; + grpc_fd *fd = arg; + /* Add the fd to the freelist */ + grpc_iomgr_unregister_object(&fd->iomgr_object); + pollable_destroy(&fd->pollable); + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); - gpr_mu_unlock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); } #ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, const char *reason, const char *file, - int line) { +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, + const char *reason, const char *file, int line) { gpr_atm old; gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), @@ -307,7 +308,9 @@ static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n) { #endif old = gpr_atm_full_fetch_add(&fd->refst, -n); if (old == n) { - grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); + grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); } else { GPR_ASSERT(old > n); } @@ -660,7 +663,8 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { return error; } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, + grpc_pollset_worker *specific_worker) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p " @@ -711,7 +715,6 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_p gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } - } /* p->po.mu must be held before calling this function */ @@ -816,7 +819,8 @@ static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { pollable_destroy(&pollset->pollable); if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, + "pollset_pollable"); } } @@ -967,7 +971,8 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, return pollset->shutdown_closure == NULL; } -static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, +static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { worker_remove(&pollset->root_worker, PWL_POLLSET, worker); if (NEW_ROOT == @@ -1027,21 +1032,23 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); } - end_worker(pollset, &worker, worker_hdl); + end_worker(exec_ctx, pollset, &worker, worker_hdl); if (worker.pollable != &pollset->pollable) { gpr_mu_unlock(&worker.pollable->po.mu); } return error; } -static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { +static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { grpc_fd *fd = arg; UNREF_BY(exec_ctx, fd, 2, "pollset_pollable"); } /* expects pollsets locked, flag whether fd is locked or not */ -static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd, bool fd_locked) { +static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, + bool fd_locked) { static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; if (pollset->current_pollable == &g_empty_pollable) { @@ -1063,7 +1070,10 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_pollset * pollable_add_fd(&pollset->pollable, had_fd); pollable_add_fd(&pollset->pollable, fd); } - grpc_closure_sched(exec_ctx, grpc_closure_create(unref_fd_no_longer_poller, had_fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); + grpc_closure_sched(exec_ctx, + grpc_closure_create(unref_fd_no_longer_poller, had_fd, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); } return error; } @@ -1226,13 +1236,16 @@ static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { if (po_cmp(a, b) < 0) { - gpr_mu_lock(&a->mu); gpr_mu_lock(&b->mu); + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); } else { GPR_ASSERT(po_cmp(a, b) != 0); - gpr_mu_lock(&b->mu); gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + gpr_mu_lock(&a->mu); } pg_notify(exec_ctx, a, b); - gpr_mu_unlock(&a->mu); gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); } } } diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index b13ec2cbc1..a83bad319a 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -817,7 +817,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->pollset_set_count = 0; } -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); while (pollset->local_wakeup_cache) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 620bbde65d..a01de93e9b 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -204,8 +204,8 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_event_engine->pollset_shutdown(exec_ctx, pollset, closure); } -void grpc_pollset_destroy(grpc_pollset *pollset) { - g_event_engine->pollset_destroy(pollset); +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + g_event_engine->pollset_destroy(exec_ctx, pollset); } grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index e84a86082a..59832aaf24 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -66,7 +66,7 @@ typedef struct grpc_event_engine_vtable { void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu); void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); - void (*pollset_destroy)(grpc_pollset *pollset); + void (*pollset_destroy)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index 97501e6788..4d8c451ea8 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -99,7 +99,7 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *e) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static int is_stack_running_on_compute_engine(grpc_exec_ctx *exec_ctx) { diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c index e71c0ebfc5..b72d534b7e 100644 --- a/src/core/lib/surface/alarm.c +++ b/src/core/lib/surface/alarm.c @@ -81,7 +81,9 @@ void grpc_alarm_cancel(grpc_alarm *alarm) { } void grpc_alarm_destroy(grpc_alarm *alarm) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_alarm_cancel(alarm); - GRPC_CQ_INTERNAL_UNREF(alarm->cq, "alarm"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm"); gpr_free(alarm); + grpc_exec_ctx_finish(&exec_ctx); } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 3e96d09798..6bd481d3bc 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -509,7 +509,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, } } if (c->cq) { - GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } get_final_status(call, set_status_value_directly, &c->final_info.final_status, diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 35e9f7eb30..366842d121 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -182,20 +182,21 @@ void grpc_cq_internal_ref(grpc_completion_queue *cc) { static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_completion_queue *cc = arg; - GRPC_CQ_INTERNAL_UNREF(cc, "pollset_destroy"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, cc, "pollset_destroy"); } #ifdef GRPC_CQ_REF_COUNT_DEBUG -void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason, - const char *file, int line) { +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, + const char *reason, const char *file, int line) { gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "CQ:%p unref %d -> %d %s", cc, (int)cc->owning_refs.count, (int)cc->owning_refs.count - 1, reason); #else -void grpc_cq_internal_unref(grpc_completion_queue *cc) { +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, + grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_destroy(POLLSET_FROM_CQ(cc)); + grpc_pollset_destroy(exec_ctx, POLLSET_FROM_CQ(cc)); #ifndef NDEBUG gpr_free(cc->outstanding_tags); #endif @@ -469,7 +470,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, is_finished_arg.first_loop = false; } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); - GRPC_CQ_INTERNAL_UNREF(cc, "next"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "next"); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(is_finished_arg.stolen_completion == NULL); @@ -664,7 +665,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, } done: GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); - GRPC_CQ_INTERNAL_UNREF(cc, "pluck"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "pluck"); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(is_finished_arg.stolen_completion == NULL); @@ -701,7 +702,9 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { GRPC_API_TRACE("grpc_completion_queue_destroy(cc=%p)", 1, (cc)); GPR_TIMER_BEGIN("grpc_completion_queue_destroy", 0); grpc_completion_queue_shutdown(cc); - GRPC_CQ_INTERNAL_UNREF(cc, "destroy"); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "destroy"); + grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_destroy", 0); } diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index 1ff3d64293..a2e7b2d6a4 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -65,17 +65,17 @@ typedef struct grpc_cq_completion { #ifdef GRPC_CQ_REF_COUNT_DEBUG void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason, const char *file, int line); -void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason, - const char *file, int line); +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, + const char *reason, const char *file, int line); #define GRPC_CQ_INTERNAL_REF(cc, reason) \ grpc_cq_internal_ref(cc, reason, __FILE__, __LINE__) -#define GRPC_CQ_INTERNAL_UNREF(cc, reason) \ - grpc_cq_internal_unref(cc, reason, __FILE__, __LINE__) +#define GRPC_CQ_INTERNAL_UNREF(ec, cc, reason) \ + grpc_cq_internal_unref(ec, cc, reason, __FILE__, __LINE__) #else void grpc_cq_internal_ref(grpc_completion_queue *cc); -void grpc_cq_internal_unref(grpc_completion_queue *cc); +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc); #define GRPC_CQ_INTERNAL_REF(cc, reason) grpc_cq_internal_ref(cc) -#define GRPC_CQ_INTERNAL_UNREF(cc, reason) grpc_cq_internal_unref(cc) +#define GRPC_CQ_INTERNAL_UNREF(ec, cc, reason) grpc_cq_internal_unref(ec, cc) #endif /* Flag that an operation is beginning: the completion channel will not finish diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 38800ea37b..b0328cec98 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -408,7 +408,7 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { request_matcher_destroy(&server->unregistered_request_matcher); } for (i = 0; i < server->cq_count; i++) { - GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, server->cqs[i], "server"); if (server->started) { gpr_stack_lockfree_destroy(server->request_freelist_per_cq[i]); gpr_free(server->requested_calls_per_cq[i]); diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index 451ed268d3..259abfd151 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -469,7 +469,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { static void destroy_pollset(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { grpc_pollset* pollset = arg; - grpc_pollset_destroy(pollset); + grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); } diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index d3b45c4505..21135ddf6b 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -155,7 +155,7 @@ static void test_post(int port) { } static void destroy_pops(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(grpc_polling_entity_pollset(p)); + grpc_pollset_destroy(exec_ctx, grpc_polling_entity_pollset(p)); } int main(int argc, char **argv) { diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index acc94091f4..73eaae87d7 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -157,7 +157,7 @@ static void test_post(int port) { } static void destroy_pops(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(grpc_polling_entity_pollset(p)); + grpc_pollset_destroy(exec_ctx, grpc_polling_entity_pollset(p)); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index c8a60776b9..4561c3846e 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -70,7 +70,7 @@ static grpc_endpoint_test_config configs[] = { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 0856023b14..7693b5a398 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -113,7 +113,7 @@ static void test_pollset_init(test_pollset *pollsets, int num_pollsets) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx, diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 81d2692a08..d6cee8e43a 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -535,7 +535,7 @@ static void test_grpc_fd_change(void) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 3a9d459579..469551c827 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -86,7 +86,7 @@ static void init_test_pollsets(test_pollset *pollsets, const int num_pollsets) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static void cleanup_test_pollsets(grpc_exec_ctx *exec_ctx, diff --git a/test/core/iomgr/resolve_address_posix_test.c b/test/core/iomgr/resolve_address_posix_test.c index fa88aca431..7fba0b92be 100644 --- a/test/core/iomgr/resolve_address_posix_test.c +++ b/test/core/iomgr/resolve_address_posix_test.c @@ -81,7 +81,7 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) { grpc_pollset_shutdown(exec_ctx, args->pollset, &do_nothing_cb); // exec_ctx needs to be flushed before calling grpc_pollset_destroy() grpc_exec_ctx_flush(exec_ctx); - grpc_pollset_destroy(args->pollset); + grpc_pollset_destroy(exec_ctx, args->pollset); gpr_free(args->pollset); } diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index ea79adc090..0425c3b3f5 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -76,7 +76,7 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) { grpc_pollset_shutdown(exec_ctx, args->pollset, &do_nothing_cb); // exec_ctx needs to be flushed before calling grpc_pollset_destroy() grpc_exec_ctx_flush(exec_ctx); - grpc_pollset_destroy(args->pollset); + grpc_pollset_destroy(exec_ctx, args->pollset); gpr_free(args->pollset); } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 2fae6774e8..6e1bb43eb5 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -197,7 +197,7 @@ void test_fails(void) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index 2c53a003d2..a1c54e19c1 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -562,7 +562,7 @@ static grpc_endpoint_test_config configs[] = { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 112743b95b..88aead1dd0 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -444,7 +444,7 @@ static void test_connect(size_t num_connects, static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index 1f1696a7a7..ee78d6b4ad 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -307,7 +307,7 @@ static void test_receive(int number_of_clients) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index 71d8057ac3..cd6ff2ceac 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -185,7 +185,7 @@ static void test_leftover(grpc_endpoint_test_config config, size_t slice_size) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index d6841ea1f8..f0e3394b2e 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -162,7 +162,7 @@ void bad_server_thread(void *vargs) { static void done_pollset_shutdown(grpc_exec_ctx *exec_ctx, void *pollset, grpc_error *error) { - grpc_pollset_destroy(pollset); + grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 38054dd1e7..2cd50a4ff7 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -58,7 +58,7 @@ typedef struct freereq { static void destroy_pops_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { grpc_pollset *pollset = grpc_polling_entity_pollset(p); - grpc_pollset_destroy(pollset); + grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); grpc_shutdown(); } diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 496e579bc3..1908698009 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -106,6 +106,10 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { } static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} +static void finish_pollset(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_pollset_destroy(exec_ctx, arg); +} void test_tcp_server_destroy(test_tcp_server *server) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -120,9 +124,10 @@ void test_tcp_server_destroy(test_tcp_server *server) { gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), shutdown_deadline) < 0) { test_tcp_server_poll(server, 1); } - grpc_pollset_shutdown(&exec_ctx, server->pollset, &do_nothing_cb); + grpc_pollset_shutdown(&exec_ctx, server->pollset, + grpc_closure_create(finish_pollset, server->pollset, + grpc_schedule_on_exec_ctx)); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(server->pollset); gpr_free(server->pollset); grpc_shutdown(); } -- cgit v1.2.3 From 79d24fb8eb6a6dfe51aaea6de4bf388ba55bdde6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 19:35:19 +0000 Subject: Fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 22 ++++++++++++++++++---- test/core/util/port_server_client.c | 6 +++--- test/cpp/microbenchmarks/bm_pollset.cc | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index f0b9ee39f6..31316cfc4d 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -161,6 +161,7 @@ struct grpc_fd { /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer valid */ + gpr_mu orphaned_mu; bool orphaned; gpr_atm read_closure; @@ -285,6 +286,7 @@ static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); pollable_destroy(&fd->pollable); + gpr_mu_destroy(&fd->orphaned_mu); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; @@ -347,6 +349,7 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; + gpr_mu_init(&new_fd->orphaned_mu); new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); @@ -374,11 +377,11 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { int ret_fd = -1; - gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->orphaned_mu); if (!fd->orphaned) { ret_fd = fd->fd; } - gpr_mu_unlock(&fd->pollable.po.mu); + gpr_mu_unlock(&fd->orphaned_mu); return ret_fd; } @@ -390,6 +393,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *error = GRPC_ERROR_NONE; gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->orphaned_mu); fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file @@ -413,6 +417,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + gpr_mu_unlock(&fd->orphaned_mu); gpr_mu_unlock(&fd->pollable.po.mu); UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); @@ -545,9 +550,11 @@ static void pollable_init(pollable *p, polling_obj_type type) { } static void pollable_destroy(pollable *p) { - close(p->epfd); - grpc_wakeup_fd_destroy(&p->wakeup); po_destroy(&p->po); + if (p->epfd != -1) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + } } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ @@ -590,7 +597,13 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; + GPR_ASSERT(epfd != -1); + gpr_mu_lock(&fd->orphaned_mu); + if (fd->orphaned) { + gpr_mu_unlock(&fd->orphaned_mu); + return GRPC_ERROR_NONE; + } struct epoll_event ev_fd = { .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { @@ -614,6 +627,7 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); } } + gpr_mu_unlock(&fd->orphaned_mu); return error; } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 2cd50a4ff7..a388d5f27b 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -60,7 +60,6 @@ static void destroy_pops_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, grpc_pollset *pollset = grpc_polling_entity_pollset(p); grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); - grpc_shutdown(); } static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, @@ -122,12 +121,13 @@ void grpc_free_port_using_server(int port) { gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&exec_ctx, &context); - grpc_exec_ctx_finish(&exec_ctx); grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&pr.pops), shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); gpr_free(path); grpc_http_response_destroy(&rsp); + + grpc_shutdown(); } typedef struct portreq { @@ -239,7 +239,6 @@ int grpc_pick_port_using_server(void) { grpc_closure_create(got_port_from_server, &pr, grpc_schedule_on_exec_ctx), &pr.response); grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); - grpc_exec_ctx_finish(&exec_ctx); gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; @@ -258,6 +257,7 @@ int grpc_pick_port_using_server(void) { grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&pr.pops), shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); + grpc_shutdown(); return pr.port; } diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 2a0e96ddf1..19b3ba0a3c 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -59,7 +59,7 @@ extern "C" { auto& force_library_initialization = Library::get(); static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) { - grpc_pollset_destroy(static_cast(ps)); + grpc_pollset_destroy(exec_ctx, static_cast(ps)); } static void BM_CreateDestroyPollset(benchmark::State& state) { -- cgit v1.2.3 From 8a2984c4b66b812a1e8519f32406358e44781536 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 19:48:27 +0000 Subject: add missing unlock --- src/core/lib/iomgr/ev_epollex_linux.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 31316cfc4d..a6fec5d49a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -610,6 +610,7 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { switch (errno) { case EEXIST: /* if this fd is already in the epoll set, the workqueue fd must also be - just return */ + gpr_mu_unlock(&fd->orphaned_mu); return GRPC_ERROR_NONE; default: append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); -- cgit v1.2.3 From 144521fc76b2c4af303ffd70e3fcd946866ee994 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 20:19:04 +0000 Subject: sync fixes --- include/grpc++/server_builder.h | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/core') diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index d707100a52..a0ba4fc8d2 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,7 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(1), + : num_cqs(gpr_cpu_num_cores()), min_pollers(1), max_pollers(INT_MAX), cq_timeout_msec(1000) {} diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index a6fec5d49a..af2908f16d 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -972,7 +972,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (pollset->root_worker != worker) { + while (worker->pollable->root_worker != worker) { if (gpr_cv_wait(&worker->cv, &pollset->current_pollable->po.mu, deadline)) { return false; -- cgit v1.2.3 From e3a69338c38fc63a8a7e3d0e61802b9dff13439b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 18 Apr 2017 16:45:40 +0000 Subject: Fix wakeup bugs --- src/core/lib/iomgr/ev_epollex_linux.c | 73 +++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 21 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index af2908f16d..4f221a5f22 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -666,6 +666,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { grpc_pollset_worker *worker = pollset->root_worker; do { if (worker->initialized_cv) { + worker->kicked = true; gpr_cv_signal(&worker->cv); } else { append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), @@ -727,6 +728,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); } + specific_worker->kicked = true; gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } @@ -850,13 +852,16 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, int timeout = poll_deadline_to_millis_timeout(deadline, now); if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll for %dms", pollset, timeout); + gpr_log(GPR_DEBUG, "PS:%p poll %p for %dms", pollset, p, timeout); } if (timeout != 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; } - int r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); + int r; + do { + r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); + } while (r < 0 && errno == EINTR); if (timeout != 0) { GRPC_SCHEDULING_END_BLOCKING_REGION; } @@ -864,7 +869,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + gpr_log(GPR_DEBUG, "PS:%p poll %p got %d events", pollset, p, r); } grpc_error *error = GRPC_ERROR_NONE; @@ -872,7 +877,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + gpr_log(GPR_DEBUG, "PS:%p poll %p got global_wakeup_fd", pollset, p); } grpc_timer_consume_kick(); @@ -880,7 +885,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, err_desc); } else if (data_ptr == &p->wakeup) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); + gpr_log(GPR_DEBUG, "PS:%p poll %p got pollset_wakeup", pollset, p); } append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); } else { @@ -890,11 +895,11 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; if (grpc_polling_trace) { - gpr_log( - GPR_DEBUG, - "PS:%p poll got fd %p(%d/%d): is_wq=%d cancel=%d read=%d write=%d", - pollset, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, is_workqueue, - cancel, read_ev, write_ev); + gpr_log(GPR_DEBUG, + "PS:%p poll %p got fd %p(%d/%d): is_wq=%d cancel=%d read=%d " + "write=%d", + pollset, p, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, + is_workqueue, cancel, read_ev, write_ev); } if (is_workqueue) { append_error(&error, @@ -956,8 +961,9 @@ static worker_remove_result worker_remove(grpc_pollset_worker **root, /* Return true if this thread should poll */ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, - grpc_pollset_worker **worker_hdl, + grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { + bool do_poll = true; if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; worker->kicked = false; @@ -972,18 +978,43 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (worker->pollable->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &pollset->current_pollable->po.mu, - deadline)) { - return false; - } - if (worker->kicked) { - return false; + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&pollset->pollable.po.mu); + } + if (grpc_polling_trace && worker->pollable->root_worker != worker) { + gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, + worker->pollable, worker, + poll_deadline_to_millis_timeout(deadline, *now)); + } + while (do_poll && worker->pollable->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, + worker->pollable, worker); + } + do_poll = false; + } else if (worker->kicked) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, + worker); + } + do_poll = false; + } else if (grpc_polling_trace && + worker->pollable->root_worker != worker) { + gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, + worker->pollable, worker); } } + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&worker->pollable->po.mu); + gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&worker->pollable->po.mu); + } + *now = gpr_now(now->clock_type); } - return pollset->shutdown_closure == NULL; + return do_poll && pollset->shutdown_closure == NULL && + pollset->current_pollable == worker->pollable; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -1010,7 +1041,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - if (grpc_polling_trace) { + if (0 && grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, @@ -1026,7 +1057,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (pollset->current_pollable != &pollset->pollable) { gpr_mu_lock(&pollset->current_pollable->po.mu); } - if (begin_worker(pollset, &worker, worker_hdl, deadline)) { + if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); -- cgit v1.2.3 From d37d142a6339ba1ea600fe4c2002971dc8337949 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 18 Apr 2017 19:41:07 +0000 Subject: sync fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4f221a5f22..d6504f5bac 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -711,11 +711,17 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } return GRPC_ERROR_NONE; } + } else if (specific_worker->kicked) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_already_kicked", p); + } + return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); } + specific_worker->kicked = true; return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { if (grpc_polling_trace) { @@ -723,6 +729,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } grpc_error *err = pollable_materialize(p); if (err != GRPC_ERROR_NONE) return err; + specific_worker->kicked = true; return grpc_wakeup_fd_wakeup(&p->wakeup); } else { if (grpc_polling_trace) { -- cgit v1.2.3 From 1d15194027649936ab579aeeba81240252e13d2f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 23:22:48 +0000 Subject: Fix race --- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d6504f5bac..0839a40429 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -665,6 +665,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + gpr_mu_lock(&worker->pollable->po.mu); if (worker->initialized_cv) { worker->kicked = true; gpr_cv_signal(&worker->cv); @@ -672,6 +673,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), "pollset_shutdown"); } + gpr_mu_unlock(&worker->pollable->po.mu); worker = worker->links[PWL_POLLSET].next; } while (worker != pollset->root_worker); -- cgit v1.2.3 From 227e11bdee93bcc88abf664be23aab8cafbaf653 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 18:25:02 +0000 Subject: Fix missing edge on shutdown path --- src/core/lib/iomgr/ev_epollex_linux.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0839a40429..adbdaf6e5b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -825,6 +825,7 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + pollset->shutdown_closure = NULL; } } @@ -1029,7 +1030,6 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - worker_remove(&pollset->root_worker, PWL_POLLSET, worker); if (NEW_ROOT == worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { gpr_cv_signal(&worker->pollable->root_worker->cv); @@ -1040,6 +1040,9 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (pollset_is_pollable_fd(pollset, worker->pollable)) { UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); } + if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } } /* pollset->po.mu lock must be held by the caller before calling this. -- cgit v1.2.3 From ffe51bd8ad96f5b26c6a09e039e49538a7d140b5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 08:30:24 -0700 Subject: Fix? uv portability --- src/core/lib/iomgr/pollset_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index a2f81bcd78..5923da98e2 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -106,7 +106,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } -void grpc_pollset_destroy(grpc_pollset *pollset) { +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { uv_close((uv_handle_t *)&pollset->timer, timer_close_cb); // timer.data is a boolean indicating that the timer has finished closing pollset->timer.data = (void *)0; -- cgit v1.2.3 From 0bf244769910fdc9f467bd531c82d58bc7941500 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 08:31:20 -0700 Subject: Fix? older compiler portability --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index adbdaf6e5b..4000d0445f 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -64,7 +64,7 @@ #include "src/core/lib/support/spinlock.h" #ifndef EPOLLEXCLUSIVE -#define EPOLLEXCLUSIVE (1u << 28) +#define EPOLLEXCLUSIVE (1 << 28) #endif /* TODO: sreek: Right now, this wakes up all pollers. In future we should make -- cgit v1.2.3 From c8d9b9c3437ad799a718db171a665b0bc094ad38 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 16:41:34 +0000 Subject: Fix deadlock --- src/core/lib/iomgr/ev_epollex_linux.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4000d0445f..3a0335ace0 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -665,7 +665,9 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable != &pollset->pollable) { + gpr_mu_lock(&worker->pollable->po.mu); + } if (worker->initialized_cv) { worker->kicked = true; gpr_cv_signal(&worker->cv); @@ -673,7 +675,9 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), "pollset_shutdown"); } - gpr_mu_unlock(&worker->pollable->po.mu); + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&worker->pollable->po.mu); + } worker = worker->links[PWL_POLLSET].next; } while (worker != pollset->root_worker); -- cgit v1.2.3 From 1898a29f7ecf2ac4f706351d68dedd5767f02ec9 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 19 Apr 2017 19:37:39 -0700 Subject: Fixed pollers --- include/grpc++/server_builder.h | 2 +- src/core/lib/iomgr/ev_epoll_linux.c | 362 ++++++++++++++++++++++++++-------- test/core/iomgr/ev_epoll_linux_test.c | 46 ++++- 3 files changed, 320 insertions(+), 90 deletions(-) (limited to 'src/core') diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 7ac23349c8..68c5344e2a 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,7 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(1), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(gpr_cpu_num_cores()), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index e603a75593..16a9199ab9 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -97,6 +97,9 @@ void grpc_use_signal(int signum) { } } +/* The maximum number of polling threads per polling island */ +#define GRPC_MAX_POLLERS_PER_ISLAND 1 + struct polling_island; typedef enum { @@ -195,6 +198,11 @@ static void fd_global_shutdown(void); #endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ +typedef struct worker_node { + struct worker_node *next; + struct worker_node *prev; +} worker_node; + /* This is also used as grpc_workqueue (by directly casing it) */ typedef struct polling_island { grpc_closure_scheduler workqueue_scheduler; @@ -229,6 +237,9 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; + gpr_mu worker_list_mu; + worker_node worker_list_head; + /* The fd of the underlying epoll set */ int epoll_fd; @@ -241,14 +252,21 @@ typedef struct polling_island { /******************************************************************************* * Pollset Declarations */ +#define WORKER_FROM_WORKER_LIST_NODE(p) \ + (struct grpc_pollset_worker *)(((char *)(p)) - \ + offsetof(grpc_pollset_worker, pi_list_link)) struct grpc_pollset_worker { /* Thread id of this worker */ pthread_t pt_id; /* Used to prevent a worker from getting kicked multiple times */ gpr_atm is_kicked; + struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; + + gpr_atm is_polling_turn; + worker_node pi_list_link; }; struct grpc_pollset { @@ -392,7 +410,47 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { } } -/* The caller is expected to hold pi->mu lock before calling this function */ +static void worker_node_init(worker_node *node) { + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void push_back_worker_node(worker_node *head, worker_node *node) { + node->next = head; + node->prev = head->prev; + head->prev->next = node; + head->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void remove_worker_node(worker_node *node) { + node->next->prev = node->prev; + node->prev->next = node->next; + /* If node's next and prev point to itself, the node is considered detached + * from the list*/ + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static worker_node *pop_front_worker_node(worker_node *head) { + worker_node *node = head->next; + if (node != head) { + remove_worker_node(node); + } else { + node = NULL; + } + + return node; +} + +/* Returns true if the node's next and prev are pointing to itself (which + indicates that the node is not in the list */ +static bool is_worker_node_detached(worker_node *node) { + return (node->next == node->prev && node->next == node); +} + +/* The caller is expected to hold pi->mu lock before calling this function + */ static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, size_t fd_count, bool add_fd_refs, grpc_error **error) { @@ -546,6 +604,9 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, gpr_atm_rel_store(&pi->poller_count, 0); gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); + gpr_mu_init(&pi->worker_list_mu); + worker_node_init(&pi->worker_list_head); + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), err_desc)) { goto done; @@ -584,6 +645,9 @@ static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { gpr_mpscq_destroy(&pi->workqueue_items); gpr_mu_destroy(&pi->mu); grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + gpr_mu_destroy(&pi->worker_list_mu); + GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); + gpr_free(pi->fds); gpr_free(pi); } @@ -1102,6 +1166,7 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static __thread bool g_initialized_sigmask; static __thread sigset_t g_orig_sigmask; +static __thread sigset_t g_wakeup_sig_set; static void sig_handler(int sig_num) { #ifdef GRPC_EPOLL_DEBUG @@ -1109,6 +1174,14 @@ static void sig_handler(int sig_num) { #endif } +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->pt_id = pthread_self(); + worker->next = worker->prev = NULL; + gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); + gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); + worker_node_init(&worker->pi_list_link); +} + static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } /* Global state management */ @@ -1125,11 +1198,12 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { +static grpc_error *worker_kick(grpc_pollset_worker *worker, + gpr_atm *is_kicked) { grpc_error *err = GRPC_ERROR_NONE; /* Kick the worker only if it was not already kicked */ - if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { + if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { GRPC_POLLING_TRACE( "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", (void *)worker, (long int)worker->pt_id); @@ -1141,6 +1215,14 @@ static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { return err; } +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_kicked); +} + +static grpc_error *poller_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_polling_turn); +} + /* Return 1 if the pollset has active threads in pollset_work (pollset must * be locked) */ static int pollset_has_workers(grpc_pollset *p) { @@ -1246,6 +1328,22 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->shutdown_done = NULL; } +/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + /* Convert a timespec to milliseconds: - Very small or negative poll times are clamped to zero to do a non-blocking poll (which becomes spin polling) @@ -1364,35 +1462,190 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, return false; } +/* NOTE: May modify 'now' */ +static bool acquire_polling_lease(grpc_pollset_worker *worker, + polling_island *pi, gpr_timespec deadline, + gpr_timespec *now) { + bool is_lease_acquired = false; + + gpr_mu_lock(&pi->worker_list_mu); // Lock + long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + + if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { + push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); + gpr_mu_unlock(&pi->worker_list_mu); // Unlock + + bool is_timeout = false; + int ret; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); + if (timeout_ms == -1) { + ret = sigwaitinfo(&g_wakeup_sig_set, NULL); + } else { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + } + + if (ret == -1) { + if (errno == EAGAIN) { + // gpr_log(GPR_INFO, "timeout"); // TODO: sreek remove this + // log-line + } else { + gpr_log(GPR_ERROR, "Failed with retcode: %d (timeout_ms: %d)", errno, + timeout_ms); + } + is_timeout = true; + } + + bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); + /* + if (is_polling_turn) { +/ gpr_log(GPR_ERROR, "do epoll is true (timeout_ms:%d)", + timeout_ms); // TODO: sreek remove this logline + } + */ + + bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + if (is_kicked || is_timeout) { + *now = deadline; + } else if (is_polling_turn) { + *now = gpr_now(GPR_CLOCK_MONOTONIC); + } + + gpr_mu_lock(&pi->worker_list_mu); // Lock + /* The node might have already been removed from the list by the poller + that kicked this. However it is safe to call 'remove_worker_node' on + an already detached node */ + remove_worker_node(&worker->pi_list_link); + num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + } + + if (num_pollers < GRPC_MAX_POLLERS_PER_ISLAND) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); // Add a poller + is_lease_acquired = true; + } + + gpr_mu_unlock(&pi->worker_list_mu); // Unlock + return is_lease_acquired; +} + +static void release_polling_lease(polling_island *pi, grpc_error **error) { + gpr_mu_lock(&pi->worker_list_mu); // Lock + + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); // Remove poller + worker_node *node = pop_front_worker_node(&pi->worker_list_head); + if (node != NULL) { + grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); + append_error(error, poller_kick(next_worker), "poller kick error"); + } + + gpr_mu_unlock(&pi->worker_list_mu); +} + #define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + if (!acquire_polling_lease(worker, pi, deadline, &now)) { + return; + } + + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = + epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + release_polling_lease(pi, error); + + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + /* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, int timeout_ms, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, sigset_t *sig_mask, grpc_error **error) { - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int epoll_fd = -1; - int ep_rv; polling_island *pi = NULL; - char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the latest polling island pointed by pollset->po.pi - Since epoll_fd is immutable, we can read it without obtaining the polling - island lock. There is however a possibility that the polling island (from - which we got the epoll_fd) got merged with another island while we are - in this function. This is still okay because in such a case, we will wakeup - right-away from epoll_wait() and pick up the latest polling_island the next - this function (i.e pollset_work_and_unlock()) is called */ + Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. There is however a possibility that the polling island from + which we got the epoll_fd, got merged with another island in the meantime. + This is okay because in such a case, we will wakeup right-away from + epoll_pwait() (because any merge will poison the old polling island's epoll + set 'polling_island_wakeup_fd') and then pick up the latest polling_island + the next time this function - pollset_work_and_unlock()) is called */ if (pollset->po.pi == NULL) { pollset->po.pi = polling_island_create(exec_ctx, NULL, error); if (pollset->po.pi == NULL) { GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. We cannot continue */ + return; /* Fatal error. Cannot continue */ } PI_ADD_REF(pollset->po.pi, "ps"); @@ -1423,70 +1676,10 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ if (!maybe_do_workqueue_work(exec_ctx, pi)) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); g_current_thread_polling_island = pi; - - GRPC_SCHEDULING_START_BLOCKING_REGION; - ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, - sig_mask); - GRPC_SCHEDULING_END_BLOCKING_REGION; - if (ep_rv < 0) { - if (errno != EINTR) { - gpr_asprintf(&err_msg, - "epoll_wait() epoll fd: %d failed with error: %d (%s)", - epoll_fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - } else { - /* We were interrupted. Save an interation by doing a zero timeout - epoll_wait to see if there are any other events of interest */ - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p received kick", - (void *)pollset, (void *)worker); - ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - } - } - -#ifdef GRPC_TSAN - /* See the definition of g_poll_sync for more details */ - gpr_atm_acq_load(&g_epoll_sync); -#endif /* defined(GRPC_TSAN) */ - - for (int i = 0; i < ep_rv; ++i) { - void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { - append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), - err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " - "%d) got merged", - (void *)pollset, (void *)worker, epoll_fd); - /* This means that our polling island is merged with a different - island. We do not have to do anything here since the subsequent call - to the function pollset_work_and_unlock() will pick up the correct - epoll_fd */ - } else { - grpc_fd *fd = data_ptr; - int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write_ev = ep_ev[i].events & EPOLLOUT; - if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); - } - if (write_ev || cancel) { - fd_become_writable(exec_ctx, fd); - } - } - } - + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, + deadline, sig_mask, error); g_current_thread_polling_island = NULL; - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); } GPR_ASSERT(pi != NULL); @@ -1510,14 +1703,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec now, gpr_timespec deadline) { GPR_TIMER_BEGIN("pollset_work", 0); grpc_error *error = GRPC_ERROR_NONE; - int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); - - sigset_t new_mask; grpc_pollset_worker worker; - worker.next = worker.prev = NULL; - worker.pt_id = pthread_self(); - gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); + pollset_worker_init(&worker); if (worker_hdl) *worker_hdl = &worker; @@ -1551,9 +1739,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, misses acting on a kick */ if (!g_initialized_sigmask) { - sigemptyset(&new_mask); - sigaddset(&new_mask, grpc_wakeup_signal); - pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); + sigemptyset(&g_wakeup_sig_set); + sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); sigdelset(&g_orig_sigmask, grpc_wakeup_signal); g_initialized_sigmask = true; /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. @@ -1568,7 +1756,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, push_front_worker(pollset, &worker); /* Add worker to pollset */ - pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms, + pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, &g_orig_sigmask, &error); grpc_exec_ctx_flush(exec_ctx); diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 0856023b14..f7745cddb4 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -38,7 +38,10 @@ #include "src/core/lib/iomgr/ev_posix.h" #include +#include +#include #include +#include #include #include @@ -327,7 +330,7 @@ static __thread int thread_wakeups = 0; static void test_threading_loop(void *arg) { threading_shared *shared = arg; - while (thread_wakeups < 1000000) { + while (thread_wakeups < 20000) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker; gpr_mu_lock(shared->mu); @@ -360,7 +363,7 @@ static void test_threading(void) { shared.pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(shared.pollset, &shared.mu); - gpr_thd_id thds[10]; + gpr_thd_id thds[20]; for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); @@ -399,6 +402,44 @@ static void test_threading(void) { gpr_free(shared.pollset); } +/* Convert milliseconds into 'struct timespec' struct. millis == -1 is + * * considered as an infinity-time in future */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + +void test_sigwait() { + sigset_t wakeup_sig_set; + sigemptyset(&wakeup_sig_set); + sigaddset(&wakeup_sig_set, SIGRTMIN + 6); + int timeout_ms[] = {10, 1400}; + + for (size_t i = 0; i < GPR_ARRAY_SIZE(timeout_ms); i++) { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms[i]); + gpr_log(GPR_ERROR, "sigwait_timeout: %ld, %ld", sigwait_timeout.tv_sec, + sigwait_timeout.tv_nsec); + + gpr_log(GPR_ERROR, "Waiting for %d ms...", timeout_ms[i]); + gpr_timespec bef = gpr_now(GPR_CLOCK_REALTIME); + sigtimedwait(&wakeup_sig_set, NULL, &sigwait_timeout); + gpr_timespec af = gpr_now(GPR_CLOCK_REALTIME); + + gpr_log(GPR_ERROR, "Bef: %ld, %d", bef.tv_sec, bef.tv_nsec); + gpr_log(GPR_ERROR, "Aft: %ld, %d", af.tv_sec, af.tv_nsec); + } +} + int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); @@ -409,6 +450,7 @@ int main(int argc, char **argv) { test_add_fd_to_pollset(); test_pollset_queue_merge_items(); test_threading(); + test_sigwait(); } else { gpr_log(GPR_INFO, "Skipping the test. The test is only relevant for 'epoll' " -- cgit v1.2.3 From 92290167d63fd8b591e0028ba89a14546783b6b1 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 21 Apr 2017 15:40:27 -0700 Subject: Minor changes --- include/grpc++/server_builder.h | 5 ++++- src/core/lib/iomgr/ev_epoll_linux.c | 41 ++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) (limited to 'src/core') diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 68c5344e2a..a56f81dc4b 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,10 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(gpr_cpu_num_cores()), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(gpr_cpu_num_cores()), + min_pollers(1), + max_pollers(2), + cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 16a9199ab9..a686d9e0ac 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -237,6 +237,7 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; + /* The list of workers waiting to do polling on this polling island */ gpr_mu worker_list_mu; worker_node worker_list_head; @@ -265,7 +266,10 @@ struct grpc_pollset_worker { struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; + /* Indicates if it is this worker's turn to do epoll */ gpr_atm is_polling_turn; + + /* Node in the polling island's worker list. */ worker_node pi_list_link; }; @@ -1468,12 +1472,12 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, gpr_timespec *now) { bool is_lease_acquired = false; - gpr_mu_lock(&pi->worker_list_mu); // Lock + gpr_mu_lock(&pi->worker_list_mu); // LOCK long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); - gpr_mu_unlock(&pi->worker_list_mu); // Unlock + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK bool is_timeout = false; int ret; @@ -1487,51 +1491,54 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, if (ret == -1) { if (errno == EAGAIN) { - // gpr_log(GPR_INFO, "timeout"); // TODO: sreek remove this - // log-line + is_timeout = true; } else { + /* TODO: sreek This should not happen. If we see these log messages, it + * means we are most likely something incorrect in the setup needed for + * sigwaitinfo/sigtimedwait */ gpr_log(GPR_ERROR, "Failed with retcode: %d (timeout_ms: %d)", errno, timeout_ms); } - is_timeout = true; } + /* Did the worker come out of sigtimedwait due to a thread that just + * exited epoll and kicking it (in release_polling_lease function). */ bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); - /* - if (is_polling_turn) { -/ gpr_log(GPR_ERROR, "do epoll is true (timeout_ms:%d)", - timeout_ms); // TODO: sreek remove this logline - } - */ + /* Did the worker come out of sigtimedwait due to a thread alerting it that + * some completion event was (likely) available in the completion queue */ bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + if (is_kicked || is_timeout) { - *now = deadline; + *now = deadline; /* Essentially make the epoll timeout = 0 */ } else if (is_polling_turn) { - *now = gpr_now(GPR_CLOCK_MONOTONIC); + *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ } - gpr_mu_lock(&pi->worker_list_mu); // Lock + gpr_mu_lock(&pi->worker_list_mu); // LOCK /* The node might have already been removed from the list by the poller that kicked this. However it is safe to call 'remove_worker_node' on an already detached node */ remove_worker_node(&worker->pi_list_link); + /* It is important to read the num_pollers again under the lock so that we + * have the latest num_pollers value that doesn't change while we are doing + * the "(num_pollers < GPRC_MAX_POLLERS_PER_ISLAND)" a a few lines below */ num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); } if (num_pollers < GRPC_MAX_POLLERS_PER_ISLAND) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); // Add a poller + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); is_lease_acquired = true; } - gpr_mu_unlock(&pi->worker_list_mu); // Unlock + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK return is_lease_acquired; } static void release_polling_lease(polling_island *pi, grpc_error **error) { gpr_mu_lock(&pi->worker_list_mu); // Lock - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); // Remove poller + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); worker_node *node = pop_front_worker_node(&pi->worker_list_head); if (node != NULL) { grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); -- cgit v1.2.3 From f3e94ecaa13433bb5310df716420da6624238f9d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 21 Apr 2017 15:48:33 -0700 Subject: Add TODOs to ev_epoll_linux_test --- src/core/lib/iomgr/ev_epoll_linux.c | 4 ++-- test/core/iomgr/ev_epoll_linux_test.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index a686d9e0ac..3e13e657d3 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1472,12 +1472,12 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, gpr_timespec *now) { bool is_lease_acquired = false; - gpr_mu_lock(&pi->worker_list_mu); // LOCK + gpr_mu_lock(&pi->worker_list_mu); // LOCK long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); - gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK bool is_timeout = false; int ret; diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index f7745cddb4..3813fb9cb0 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -419,11 +419,13 @@ static struct timespec millis_to_timespec(int millis) { return linux_ts; } +/* TODO (sreek) - Remove this test before merging. This is written just to + * understand the functionality of sigtimedwait and serves no other purpose */ void test_sigwait() { sigset_t wakeup_sig_set; sigemptyset(&wakeup_sig_set); sigaddset(&wakeup_sig_set, SIGRTMIN + 6); - int timeout_ms[] = {10, 1400}; + int timeout_ms[] = {10, 100}; for (size_t i = 0; i < GPR_ARRAY_SIZE(timeout_ms); i++) { struct timespec sigwait_timeout = millis_to_timespec(timeout_ms[i]); -- cgit v1.2.3 From 334b3548b26404bd66ddaf10c9492accc684c6f5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 17:11:29 -0700 Subject: Add casts --- src/core/lib/iomgr/ev_epollex_linux.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 3a0335ace0..d12c7ffe90 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -564,7 +564,7 @@ static grpc_error *pollable_materialize(pollable *p) { if (new_epfd < 0) { return GRPC_OS_ERROR(errno, "epoll_create1"); } else { - struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), .data.ptr = &global_wakeup_fd}; if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { @@ -578,7 +578,7 @@ static grpc_error *pollable_materialize(pollable *p) { close(new_epfd); return err; } - struct epoll_event ev = {.events = EPOLLIN | EPOLLET, + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &p->wakeup}; if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { err = GRPC_OS_ERROR(errno, "epoll_ctl"); @@ -605,7 +605,8 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { return GRPC_ERROR_NONE; } struct epoll_event ev_fd = { - .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE), + .data.ptr = fd}; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { switch (errno) { case EEXIST: /* if this fd is already in the epoll set, the workqueue fd @@ -616,8 +617,9 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); } } - struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = (void *)(1 + (intptr_t)fd)}; + struct epoll_event ev_wq = { + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE), + .data.ptr = (void *)(1 + (intptr_t)fd)}; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != 0) { switch (errno) { @@ -1519,7 +1521,7 @@ static bool is_epollex_available(void) { /* choose events that should cause an error on EPOLLEXCLUSIVE enabled kernels - specifically the combination of EPOLLONESHOT and EPOLLEXCLUSIVE */ - .events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT, + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), .data.ptr = NULL}; if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { if (errno != EINVAL) { -- cgit v1.2.3 From e0e8ed9b5bc16111057a2760b235b22bfa084483 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 18:32:11 +0000 Subject: Fix compile --- src/core/lib/iomgr/pollset_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 6dca37b481..d0387d210b 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -116,7 +116,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } -void grpc_pollset_destroy(grpc_pollset *pollset) {} +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {} grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, -- cgit v1.2.3 From 956920d84ead617222fe073f2e877211dfb9ce81 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 13:09:12 -0700 Subject: clang-format --- src/core/lib/iomgr/ev_epollex_linux.c | 5 +++-- src/core/lib/surface/completion_queue.c | 3 ++- test/cpp/microbenchmarks/bm_pollset.cc | 11 ++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d12c7ffe90..ee944bcfad 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -564,8 +564,9 @@ static grpc_error *pollable_materialize(pollable *p) { if (new_epfd < 0) { return GRPC_OS_ERROR(errno, "epoll_create1"); } else { - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), - .data.ptr = &global_wakeup_fd}; + struct epoll_event ev = { + .events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), + .data.ptr = &global_wakeup_fd}; if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 7f950cdad5..a739af2fe9 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -98,7 +98,8 @@ static void non_polling_poller_init(grpc_pollset *pollset, gpr_mu **mu) { *mu = &npp->mu; } -static void non_polling_poller_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { +static void non_polling_poller_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { non_polling_poller *npp = (non_polling_poller *)pollset; gpr_mu_destroy(&npp->mu); } diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 19b3ba0a3c..f5e8d13881 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -157,17 +157,18 @@ static void BM_PollAddFd(benchmark::State& state) { grpc_pollset_init(ps, &mu); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_wakeup_fd wakeup_fd; - GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&wakeup_fd))); - grpc_fd *fd = grpc_fd_create(wakeup_fd.read_fd, "xxx"); + GPR_ASSERT( + GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&wakeup_fd))); + grpc_fd* fd = grpc_fd_create(wakeup_fd.read_fd, "xxx"); while (state.KeepRunning()) { - grpc_pollset_add_fd(&exec_ctx, ps, fd); - grpc_exec_ctx_flush(&exec_ctx); + grpc_pollset_add_fd(&exec_ctx, ps, fd); + grpc_exec_ctx_flush(&exec_ctx); } grpc_fd_orphan(&exec_ctx, fd, NULL, NULL, "xxx"); grpc_closure shutdown_ps_closure; grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, grpc_schedule_on_exec_ctx); - gpr_mu_lock(mu); + gpr_mu_lock(mu); grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); gpr_mu_unlock(mu); grpc_exec_ctx_finish(&exec_ctx); -- cgit v1.2.3 From c3c9bf2442664493f081460c10f98ea2c43e9496 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 13:15:05 -0700 Subject: Fix tracing related TSAN race --- src/core/lib/iomgr/ev_epollex_linux.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index ee944bcfad..64afc7b804 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -913,10 +913,9 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool write_ev = (events[i].events & EPOLLOUT) != 0; if (grpc_polling_trace) { gpr_log(GPR_DEBUG, - "PS:%p poll %p got fd %p(%d/%d): is_wq=%d cancel=%d read=%d " + "PS:%p poll %p got fd %p: is_wq=%d cancel=%d read=%d " "write=%d", - pollset, p, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, - is_workqueue, cancel, read_ev, write_ev); + pollset, p, fd, is_workqueue, cancel, read_ev, write_ev); } if (is_workqueue) { append_error(&error, -- cgit v1.2.3 From 819cd88bd7e95092f6f46fada403f86fc0dcacec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 13:18:22 -0700 Subject: Allow EPOLLEXCLUSIVE tests to be omitted outside of CI --- BUILD | 3 + CMakeLists.txt | 44 ++++++ Makefile | 42 ++++- binding.gyp | 1 + build.yaml | 11 ++ config.m4 | 1 + gRPC-Core.podspec | 5 + grpc.gemspec | 3 + package.xml | 3 + src/core/lib/iomgr/ev_epollex_linux.c | 69 +-------- src/core/lib/iomgr/is_epollexclusive_available.c | 116 ++++++++++++++ src/core/lib/iomgr/is_epollexclusive_available.h | 41 +++++ src/core/lib/iomgr/sys_epoll_wrapper.h | 43 ++++++ src/python/grpcio/grpc_core_dependencies.py | 1 + test/build/check_epollexclusive.c | 38 +++++ tools/doxygen/Doxyfile.c++.internal | 3 + tools/doxygen/Doxyfile.core.internal | 3 + tools/run_tests/generated/sources_and_headers.json | 20 +++ tools/run_tests/run_tests.py | 12 ++ vsprojects/buildtests_c.sln | 25 +++ vsprojects/grpc.sln | 25 +++ .../check_epollexclusive.vcxproj | 170 +++++++++++++++++++++ .../check_epollexclusive.vcxproj.filters | 18 +++ vsprojects/vcxproj/grpc++/grpc++.vcxproj | 4 + vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 9 ++ .../grpc++_unsecure/grpc++_unsecure.vcxproj | 4 + .../grpc++_unsecure.vcxproj.filters | 9 ++ vsprojects/vcxproj/grpc/grpc.vcxproj | 4 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 9 ++ .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 4 + .../grpc_test_util/grpc_test_util.vcxproj.filters | 9 ++ .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 4 + .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 9 ++ 33 files changed, 696 insertions(+), 66 deletions(-) create mode 100644 src/core/lib/iomgr/is_epollexclusive_available.c create mode 100644 src/core/lib/iomgr/is_epollexclusive_available.h create mode 100644 src/core/lib/iomgr/sys_epoll_wrapper.h create mode 100644 test/build/check_epollexclusive.c create mode 100644 vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj create mode 100644 vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters (limited to 'src/core') diff --git a/BUILD b/BUILD index b3ad629272..5944095b35 100644 --- a/BUILD +++ b/BUILD @@ -463,6 +463,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epollex_linux.c", + "src/core/lib/iomgr/is_epollexclusive_available.c", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_posix.c", "src/core/lib/iomgr/exec_ctx.c", @@ -586,6 +587,8 @@ grpc_cc_library( "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", + "src/core/lib/iomgr/is_epollexclusive_available.h", + "src/core/lib/iomgr/sys_epoll_wrapper.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index e1513ee48f..a3fd5b4851 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,6 +346,7 @@ add_custom_target(plugins add_custom_target(tools_c DEPENDS + check_epollexclusive gen_hpack_tables gen_legal_metadata_characters gen_percent_encoding_tables @@ -946,6 +947,7 @@ add_library(grpc src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -1272,6 +1274,7 @@ add_library(grpc_cronet src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -1583,6 +1586,7 @@ add_library(grpc_test_util src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -1840,6 +1844,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -2260,6 +2265,7 @@ add_library(grpc++ src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -2586,6 +2592,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -3351,6 +3358,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -5045,6 +5053,42 @@ target_link_libraries(channel_create_test ) endif (gRPC_BUILD_TESTS) + +add_executable(check_epollexclusive + test/build/check_epollexclusive.c +) + + +target_include_directories(check_epollexclusive + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(check_epollexclusive + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc + gpr +) + + +if (gRPC_INSTALL) + install(TARGETS check_epollexclusive EXPORT gRPCTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif() + if (gRPC_BUILD_TESTS) add_executable(chttp2_hpack_encoder_test diff --git a/Makefile b/Makefile index 1f356ceef3..3540732df7 100644 --- a/Makefile +++ b/Makefile @@ -976,6 +976,7 @@ census_context_test: $(BINDIR)/$(CONFIG)/census_context_test census_resource_test: $(BINDIR)/$(CONFIG)/census_resource_test census_trace_context_test: $(BINDIR)/$(CONFIG)/census_trace_context_test channel_create_test: $(BINDIR)/$(CONFIG)/channel_create_test +check_epollexclusive: $(BINDIR)/$(CONFIG)/check_epollexclusive chttp2_hpack_encoder_test: $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test @@ -2133,7 +2134,7 @@ test_python: static_c tools: tools_c tools_cxx -tools_c: privatelibs_c $(BINDIR)/$(CONFIG)/gen_hpack_tables $(BINDIR)/$(CONFIG)/gen_legal_metadata_characters $(BINDIR)/$(CONFIG)/gen_percent_encoding_tables $(BINDIR)/$(CONFIG)/grpc_create_jwt $(BINDIR)/$(CONFIG)/grpc_print_google_default_creds_token $(BINDIR)/$(CONFIG)/grpc_verify_jwt +tools_c: privatelibs_c $(BINDIR)/$(CONFIG)/check_epollexclusive $(BINDIR)/$(CONFIG)/gen_hpack_tables $(BINDIR)/$(CONFIG)/gen_legal_metadata_characters $(BINDIR)/$(CONFIG)/gen_percent_encoding_tables $(BINDIR)/$(CONFIG)/grpc_create_jwt $(BINDIR)/$(CONFIG)/grpc_print_google_default_creds_token $(BINDIR)/$(CONFIG)/grpc_verify_jwt tools_cxx: privatelibs_cxx @@ -2920,6 +2921,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -3244,6 +3246,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -3554,6 +3557,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -3783,6 +3787,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -4180,6 +4185,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -4514,6 +4520,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -5276,6 +5283,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -8991,6 +8999,38 @@ endif endif +CHECK_EPOLLEXCLUSIVE_SRC = \ + test/build/check_epollexclusive.c \ + +CHECK_EPOLLEXCLUSIVE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CHECK_EPOLLEXCLUSIVE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/check_epollexclusive: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/check_epollexclusive: $(CHECK_EPOLLEXCLUSIVE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(CHECK_EPOLLEXCLUSIVE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/check_epollexclusive + +endif + +$(OBJDIR)/$(CONFIG)/test/build/check_epollexclusive.o: $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_check_epollexclusive: $(CHECK_EPOLLEXCLUSIVE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CHECK_EPOLLEXCLUSIVE_OBJS:.o=.dep) +endif +endif + + CHTTP2_HPACK_ENCODER_TEST_SRC = \ test/core/transport/chttp2/hpack_encoder_test.c \ diff --git a/binding.gyp b/binding.gyp index 82242c057c..d39665272c 100644 --- a/binding.gyp +++ b/binding.gyp @@ -683,6 +683,7 @@ 'src/core/lib/iomgr/iomgr_posix.c', 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', + 'src/core/lib/iomgr/is_epollexclusive_available.c', 'src/core/lib/iomgr/load_file.c', 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', diff --git a/build.yaml b/build.yaml index 80ad5498f1..856104dbe2 100644 --- a/build.yaml +++ b/build.yaml @@ -203,6 +203,7 @@ filegroups: - src/core/lib/iomgr/iomgr.h - src/core/lib/iomgr/iomgr_internal.h - src/core/lib/iomgr/iomgr_posix.h + - src/core/lib/iomgr/is_epollexclusive_available.h - src/core/lib/iomgr/load_file.h - src/core/lib/iomgr/lockfree_event.h - src/core/lib/iomgr/network_status_tracker.h @@ -224,6 +225,7 @@ filegroups: - src/core/lib/iomgr/socket_utils.h - src/core/lib/iomgr/socket_utils_posix.h - src/core/lib/iomgr/socket_windows.h + - src/core/lib/iomgr/sys_epoll_wrapper.h - src/core/lib/iomgr/tcp_client.h - src/core/lib/iomgr/tcp_client_posix.h - src/core/lib/iomgr/tcp_posix.h @@ -312,6 +314,7 @@ filegroups: - src/core/lib/iomgr/iomgr_posix.c - src/core/lib/iomgr/iomgr_uv.c - src/core/lib/iomgr/iomgr_windows.c + - src/core/lib/iomgr/is_epollexclusive_available.c - src/core/lib/iomgr/load_file.c - src/core/lib/iomgr/lockfree_event.c - src/core/lib/iomgr/network_status_tracker.c @@ -1658,6 +1661,14 @@ targets: - grpc - gpr_test_util - gpr +- name: check_epollexclusive + build: tool + language: c + src: + - test/build/check_epollexclusive.c + deps: + - grpc + - gpr - name: chttp2_hpack_encoder_test build: test language: c diff --git a/config.m4 b/config.m4 index 73c0e01a38..8d07fa931e 100644 --- a/config.m4 +++ b/config.m4 @@ -119,6 +119,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c5ff33ef87..3a1741cc85 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -285,6 +285,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_internal.h', 'src/core/lib/iomgr/iomgr_posix.h', + 'src/core/lib/iomgr/is_epollexclusive_available.h', 'src/core/lib/iomgr/load_file.h', 'src/core/lib/iomgr/lockfree_event.h', 'src/core/lib/iomgr/network_status_tracker.h', @@ -306,6 +307,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils.h', 'src/core/lib/iomgr/socket_utils_posix.h', 'src/core/lib/iomgr/socket_windows.h', + 'src/core/lib/iomgr/sys_epoll_wrapper.h', 'src/core/lib/iomgr/tcp_client.h', 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', @@ -492,6 +494,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr_posix.c', 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', + 'src/core/lib/iomgr/is_epollexclusive_available.c', 'src/core/lib/iomgr/load_file.c', 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', @@ -742,6 +745,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_internal.h', 'src/core/lib/iomgr/iomgr_posix.h', + 'src/core/lib/iomgr/is_epollexclusive_available.h', 'src/core/lib/iomgr/load_file.h', 'src/core/lib/iomgr/lockfree_event.h', 'src/core/lib/iomgr/network_status_tracker.h', @@ -763,6 +767,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils.h', 'src/core/lib/iomgr/socket_utils_posix.h', 'src/core/lib/iomgr/socket_windows.h', + 'src/core/lib/iomgr/sys_epoll_wrapper.h', 'src/core/lib/iomgr/tcp_client.h', 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index 33822cc649..b810d02d73 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -201,6 +201,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/iomgr.h ) s.files += %w( src/core/lib/iomgr/iomgr_internal.h ) s.files += %w( src/core/lib/iomgr/iomgr_posix.h ) + s.files += %w( src/core/lib/iomgr/is_epollexclusive_available.h ) s.files += %w( src/core/lib/iomgr/load_file.h ) s.files += %w( src/core/lib/iomgr/lockfree_event.h ) s.files += %w( src/core/lib/iomgr/network_status_tracker.h ) @@ -222,6 +223,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/socket_utils.h ) s.files += %w( src/core/lib/iomgr/socket_utils_posix.h ) s.files += %w( src/core/lib/iomgr/socket_windows.h ) + s.files += %w( src/core/lib/iomgr/sys_epoll_wrapper.h ) s.files += %w( src/core/lib/iomgr/tcp_client.h ) s.files += %w( src/core/lib/iomgr/tcp_client_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_posix.h ) @@ -408,6 +410,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/iomgr_posix.c ) s.files += %w( src/core/lib/iomgr/iomgr_uv.c ) s.files += %w( src/core/lib/iomgr/iomgr_windows.c ) + s.files += %w( src/core/lib/iomgr/is_epollexclusive_available.c ) s.files += %w( src/core/lib/iomgr/load_file.c ) s.files += %w( src/core/lib/iomgr/lockfree_event.c ) s.files += %w( src/core/lib/iomgr/network_status_tracker.c ) diff --git a/package.xml b/package.xml index 3f438a51ce..9c52f5cbe8 100644 --- a/package.xml +++ b/package.xml @@ -210,6 +210,7 @@ + @@ -231,6 +232,7 @@ + @@ -417,6 +419,7 @@ + diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 64afc7b804..b7c3e2e959 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -43,7 +43,6 @@ #include #include #include -#include #include #include @@ -55,7 +54,9 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/is_epollexclusive_available.h" #include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/sys_epoll_wrapper.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" @@ -63,10 +64,6 @@ #include "src/core/lib/support/block_annotate.h" #include "src/core/lib/support/spinlock.h" -#ifndef EPOLLEXCLUSIVE -#define EPOLLEXCLUSIVE (1 << 28) -#endif - /* TODO: sreek: Right now, this wakes up all pollers. In future we should make * sure to wake up one polling thread (which can wake up other threads if * needed) */ @@ -1496,70 +1493,12 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -/* It is possible that GLIBC has epoll but the underlying kernel doesn't. - * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epollex_available(void) { - static bool logged_why_not = false; - - int fd = epoll_create1(EPOLL_CLOEXEC); - if (fd < 0) { - if (!logged_why_not) { - gpr_log(GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epollex polling " - "engine.", - fd); - logged_why_not = true; - } - return false; - } - grpc_wakeup_fd wakeup; - if (!GRPC_LOG_IF_ERROR("check_wakeupfd_for_epollex", - grpc_wakeup_fd_init(&wakeup))) { - return false; - } - struct epoll_event ev = { - /* choose events that should cause an error on - EPOLLEXCLUSIVE enabled kernels - specifically the combination of - EPOLLONESHOT and EPOLLEXCLUSIVE */ - .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), - .data.ptr = NULL}; - if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { - if (errno != EINVAL) { - if (!logged_why_not) { - gpr_log( - GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " - "%d. Not using epollex polling engine.", - errno); - logged_why_not = true; - } - close(fd); - grpc_wakeup_fd_destroy(&wakeup); - return false; - } - } else { - if (!logged_why_not) { - gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " - "evidence of no EPOLLEXCLUSIVE support. Not using " - "epollex polling engine."); - logged_why_not = true; - } - close(fd); - grpc_wakeup_fd_destroy(&wakeup); - return false; - } - grpc_wakeup_fd_destroy(&wakeup); - close(fd); - return true; -} - const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { if (!grpc_has_wakeup_fd()) { return NULL; } - if (!is_epollex_available()) { + if (!grpc_is_epollexclusive_available()) { return NULL; } diff --git a/src/core/lib/iomgr/is_epollexclusive_available.c b/src/core/lib/iomgr/is_epollexclusive_available.c new file mode 100644 index 0000000000..34fcf313e5 --- /dev/null +++ b/src/core/lib/iomgr/is_epollexclusive_available.c @@ -0,0 +1,116 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#include "src/core/lib/iomgr/is_epollexclusive_available.h" + +#ifdef GRPC_LINUX_EPOLL + +#include + +#include +#include +#include + +#include "src/core/lib/iomgr/sys_epoll_wrapper.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +bool grpc_is_epollexclusive_available(void) { + static bool logged_why_not = false; + + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } + return false; + } + int evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (evfd < 0) { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "eventfd failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } + close(fd); + return false; + } + struct epoll_event ev = { + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), + .data.ptr = NULL}; + if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) { + if (errno != EINVAL) { + if (!logged_why_not) { + gpr_log( + GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + logged_why_not = true; + } + close(fd); + close(evfd); + return false; + } + } else { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); + logged_why_not = true; + } + close(fd); + close(evfd); + return false; + } + close(evfd); + close(fd); + return true; +} + +#else + +bool grpc_is_epollexclusive_available(void) { return false; } + +#endif diff --git a/src/core/lib/iomgr/is_epollexclusive_available.h b/src/core/lib/iomgr/is_epollexclusive_available.h new file mode 100644 index 0000000000..b65b819e74 --- /dev/null +++ b/src/core/lib/iomgr/is_epollexclusive_available.h @@ -0,0 +1,41 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H +#define GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H + +#include + +bool grpc_is_epollexclusive_available(void); + +#endif /* GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H */ diff --git a/src/core/lib/iomgr/sys_epoll_wrapper.h b/src/core/lib/iomgr/sys_epoll_wrapper.h new file mode 100644 index 0000000000..2f08423193 --- /dev/null +++ b/src/core/lib/iomgr/sys_epoll_wrapper.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H +#define GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H + +#include + +#ifndef EPOLLEXCLUSIVE +#define EPOLLEXCLUSIVE (1 << 28) +#endif + +#endif /* GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d3af7cace3..ced072bf59 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -108,6 +108,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/iomgr_posix.c', 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', + 'src/core/lib/iomgr/is_epollexclusive_available.c', 'src/core/lib/iomgr/load_file.c', 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', diff --git a/test/build/check_epollexclusive.c b/test/build/check_epollexclusive.c new file mode 100644 index 0000000000..fb512c3ae1 --- /dev/null +++ b/test/build/check_epollexclusive.c @@ -0,0 +1,38 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/is_epollexclusive_available.h" + +int main(int argc, char **argv) { + return grpc_is_epollexclusive_available() ? 0 : 1; +} diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index f0768464d8..4cf5bef747 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -960,6 +960,8 @@ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_posix.h \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ +src/core/lib/iomgr/is_epollexclusive_available.c \ +src/core/lib/iomgr/is_epollexclusive_available.h \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/load_file.h \ src/core/lib/iomgr/lockfree_event.c \ @@ -1002,6 +1004,7 @@ src/core/lib/iomgr/socket_utils_uv.c \ src/core/lib/iomgr/socket_utils_windows.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/socket_windows.h \ +src/core/lib/iomgr/sys_epoll_wrapper.h \ src/core/lib/iomgr/tcp_client.h \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_posix.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 02af1d69cb..d43e4bcbc5 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1095,6 +1095,8 @@ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_posix.h \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ +src/core/lib/iomgr/is_epollexclusive_available.c \ +src/core/lib/iomgr/is_epollexclusive_available.h \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/load_file.h \ src/core/lib/iomgr/lockfree_event.c \ @@ -1137,6 +1139,7 @@ src/core/lib/iomgr/socket_utils_uv.c \ src/core/lib/iomgr/socket_utils_windows.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/socket_windows.h \ +src/core/lib/iomgr/sys_epoll_wrapper.h \ src/core/lib/iomgr/tcp_client.h \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_posix.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6ef1c13b10..36305a003a 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -232,6 +232,21 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "grpc" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "check_epollexclusive", + "src": [ + "test/build/check_epollexclusive.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -7711,6 +7726,7 @@ "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_internal.h", "src/core/lib/iomgr/iomgr_posix.h", + "src/core/lib/iomgr/is_epollexclusive_available.h", "src/core/lib/iomgr/load_file.h", "src/core/lib/iomgr/lockfree_event.h", "src/core/lib/iomgr/network_status_tracker.h", @@ -7732,6 +7748,7 @@ "src/core/lib/iomgr/socket_utils.h", "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", + "src/core/lib/iomgr/sys_epoll_wrapper.h", "src/core/lib/iomgr/tcp_client.h", "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", @@ -7864,6 +7881,8 @@ "src/core/lib/iomgr/iomgr_posix.h", "src/core/lib/iomgr/iomgr_uv.c", "src/core/lib/iomgr/iomgr_windows.c", + "src/core/lib/iomgr/is_epollexclusive_available.c", + "src/core/lib/iomgr/is_epollexclusive_available.h", "src/core/lib/iomgr/load_file.c", "src/core/lib/iomgr/load_file.h", "src/core/lib/iomgr/lockfree_event.c", @@ -7906,6 +7925,7 @@ "src/core/lib/iomgr/socket_utils_windows.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/socket_windows.h", + "src/core/lib/iomgr/sys_epoll_wrapper.h", "src/core/lib/iomgr/tcp_client.h", "src/core/lib/iomgr/tcp_client_posix.c", "src/core/lib/iomgr/tcp_client_posix.h", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d2e3506fe5..3698d1b861 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1432,6 +1432,14 @@ class BuildAndRunError(object): POST_TEST = object() +def _has_epollexclusive(): + try: + subprocess.check_call('bins/%s/check_epollexclusive' % args.config) + return True + except subprocess.CalledProcessError, e: + return False + + # returns a list of things that failed (or an empty list on success) def _build_and_run( check_cancelled, newline_on_success, xml_report=None, build_only=False): @@ -1449,6 +1457,10 @@ def _build_and_run( suite_name=args.report_suite_name) return [] + if not args.travis and not _has_epollexclusive() and 'epollex' in _POLLING_STRATEGIES[platform_string()]: + print('\n\nOmitting EPOLLEXCLUSIVE tests\n\n') + _POLLING_STRATEGIES[platform_string()].remove('epollex') + # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 539f474f9e..2e8ccf812b 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -162,6 +162,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel_create_test", "vcxp {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "check_epollexclusive", "vcxproj\.\check_epollexclusive\check_epollexclusive.vcxproj", "{03306445-5BA0-289C-02AB-513DE6C52124}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "chttp2_hpack_encoder_test", "vcxproj\test\chttp2_hpack_encoder_test\chttp2_hpack_encoder_test.vcxproj", "{19F92966-3B0E-4FF8-CD7C-435D353E079E}" ProjectSection(myProperties) = preProject lib = "False" @@ -1914,6 +1923,22 @@ Global {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release-DLL|Win32.Build.0 = Release|Win32 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release-DLL|x64.ActiveCfg = Release|x64 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release-DLL|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.Build.0 = Release|x64 {19F92966-3B0E-4FF8-CD7C-435D353E079E}.Debug|Win32.ActiveCfg = Debug|Win32 {19F92966-3B0E-4FF8-CD7C-435D353E079E}.Debug|x64.ActiveCfg = Debug|x64 {19F92966-3B0E-4FF8-CD7C-435D353E079E}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/grpc.sln b/vsprojects/grpc.sln index 97378e0a0f..307ae4b599 100644 --- a/vsprojects/grpc.sln +++ b/vsprojects/grpc.sln @@ -8,6 +8,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ares", "vcxproj\.\ares\ares lib = "True" EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "check_epollexclusive", "vcxproj\.\check_epollexclusive\check_epollexclusive.vcxproj", "{03306445-5BA0-289C-02AB-513DE6C52124}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "gen_hpack_tables", "vcxproj\.\gen_hpack_tables\gen_hpack_tables.vcxproj", "{FCDEA4C7-7F26-05DB-D08F-A08F499026E6}" ProjectSection(myProperties) = preProject lib = "False" @@ -190,6 +199,22 @@ Global {1769D06D-F18C-B4C2-B019-31D7F83F3C9A}.Release-DLL|Win32.Build.0 = Release|Win32 {1769D06D-F18C-B4C2-B019-31D7F83F3C9A}.Release-DLL|x64.ActiveCfg = Release|x64 {1769D06D-F18C-B4C2-B019-31D7F83F3C9A}.Release-DLL|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.Build.0 = Release|x64 {FCDEA4C7-7F26-05DB-D08F-A08F499026E6}.Debug|Win32.ActiveCfg = Debug|Win32 {FCDEA4C7-7F26-05DB-D08F-A08F499026E6}.Debug|x64.ActiveCfg = Debug|x64 {FCDEA4C7-7F26-05DB-D08F-A08F499026E6}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj new file mode 100644 index 0000000000..36e6fee0db --- /dev/null +++ b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj @@ -0,0 +1,170 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {03306445-5BA0-289C-02AB-513DE6C52124} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + StaticLibrary + true + Unicode + + + StaticLibrary + false + true + Unicode + + + + + + + + + + + + check_epollexclusive + + + check_epollexclusive + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + diff --git a/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters new file mode 100644 index 0000000000..5572541d1d --- /dev/null +++ b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters @@ -0,0 +1,18 @@ + + + + + test\build + + + + + + {98195cc4-af1b-3ef2-80a8-86b96fa1c488} + + + {c32fe5ee-bf78-58e4-fa70-b9294e48a136} + + + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index fa4610974a..3648bd67a2 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -406,6 +406,7 @@ + @@ -427,6 +428,7 @@ + @@ -633,6 +635,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index e94aee357e..2703372d1d 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -217,6 +217,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -941,6 +944,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -1004,6 +1010,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 5513816d77..b6bfe437be 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -400,6 +400,7 @@ + @@ -421,6 +422,7 @@ + @@ -617,6 +619,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 646e35bbe8..6a86c11a3e 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -202,6 +202,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -908,6 +911,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -971,6 +977,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index d6881a98c0..7e864c10b2 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -330,6 +330,7 @@ + @@ -351,6 +352,7 @@ + @@ -571,6 +573,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 2a694cd4f3..ba17b67ed3 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -97,6 +97,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -899,6 +902,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -962,6 +968,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 1a3120c798..55d53eec47 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -225,6 +225,7 @@ + @@ -246,6 +247,7 @@ + @@ -406,6 +408,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index d750cf78d9..0a12bc9ee1 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -154,6 +154,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -656,6 +659,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -719,6 +725,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0716de4a9c..bfa01f1a3c 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -320,6 +320,7 @@ + @@ -341,6 +342,7 @@ + @@ -539,6 +541,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index ebbd03b5fb..e23e2cf091 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -100,6 +100,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -812,6 +815,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -875,6 +881,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr -- cgit v1.2.3 From 22b8bb1e0824eb21b3f244e543589639a85fb36d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 26 Apr 2017 20:08:30 -0700 Subject: Make max-pollers configurable via GRPC_MAX_POLLERS_PER_PI env variable --- src/core/lib/iomgr/ev_epoll_linux.c | 53 +++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 14 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 3e13e657d3..3a5f48cb8f 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -62,6 +63,7 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/env.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ static int grpc_polling_trace = 0; /* Disabled by default */ @@ -73,6 +75,10 @@ static int grpc_polling_trace = 0; /* Disabled by default */ /* Uncomment the following to enable extra checks on poll_object operations */ /* #define PO_DEBUG */ +/* The maximum number of polling threads per polling island. By default no + limit */ +static int g_max_pollers_per_pi = INT_MAX; + static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; @@ -97,9 +103,6 @@ void grpc_use_signal(int signum) { } } -/* The maximum number of polling threads per polling island */ -#define GRPC_MAX_POLLERS_PER_ISLAND 1 - struct polling_island; typedef enum { @@ -1466,7 +1469,7 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, return false; } -/* NOTE: May modify 'now' */ +/* NOTE: This function may modify 'now' */ static bool acquire_polling_lease(grpc_pollset_worker *worker, polling_island *pi, gpr_timespec deadline, gpr_timespec *now) { @@ -1475,7 +1478,7 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, gpr_mu_lock(&pi->worker_list_mu); // LOCK long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { + if (num_pollers >= g_max_pollers_per_pi) { push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK @@ -1493,20 +1496,21 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, if (errno == EAGAIN) { is_timeout = true; } else { - /* TODO: sreek This should not happen. If we see these log messages, it - * means we are most likely something incorrect in the setup needed for - * sigwaitinfo/sigtimedwait */ - gpr_log(GPR_ERROR, "Failed with retcode: %d (timeout_ms: %d)", errno, + /* NOTE: This should not happen. If we see these log messages, it means + we are most likely doing something incorrect in the setup * needed + for sigwaitinfo/sigtimedwait */ + gpr_log(GPR_ERROR, + "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, timeout_ms); } } /* Did the worker come out of sigtimedwait due to a thread that just - * exited epoll and kicking it (in release_polling_lease function). */ + exited epoll and kicking it (in release_polling_lease function). */ bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); /* Did the worker come out of sigtimedwait due to a thread alerting it that - * some completion event was (likely) available in the completion queue */ + some completion event was (likely) available in the completion queue */ bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); if (is_kicked || is_timeout) { @@ -1522,11 +1526,11 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, remove_worker_node(&worker->pi_list_link); /* It is important to read the num_pollers again under the lock so that we * have the latest num_pollers value that doesn't change while we are doing - * the "(num_pollers < GPRC_MAX_POLLERS_PER_ISLAND)" a a few lines below */ + * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); } - if (num_pollers < GRPC_MAX_POLLERS_PER_ISLAND) { + if (num_pollers < g_max_pollers_per_pi) { gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); is_lease_acquired = true; } @@ -1536,7 +1540,7 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, } static void release_polling_lease(polling_island *pi, grpc_error **error) { - gpr_mu_lock(&pi->worker_list_mu); // Lock + gpr_mu_lock(&pi->worker_list_mu); gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); worker_node *node = pop_front_worker_node(&pi->worker_list_head); @@ -1554,6 +1558,8 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, grpc_pollset_worker *worker, gpr_timespec now, gpr_timespec deadline, sigset_t *sig_mask, grpc_error **error) { + /* Only g_max_pollers_per_pi threads can be doing polling in parallel. + If we cannot get a lease, we cannot continue to do epoll_pwait() */ if (!acquire_polling_lease(worker, pi, deadline, &now)) { return; } @@ -1563,6 +1569,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, char *err_msg; const char *err_desc = "pollset_work_and_unlock"; + /* timeout_ms is the time between 'now' and 'deadline' */ int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); GRPC_SCHEDULING_START_BLOCKING_REGION; @@ -1570,6 +1577,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); GRPC_SCHEDULING_END_BLOCKING_REGION; + /* Give back the lease right away so that some other thread can enter */ release_polling_lease(pi, error); if (ep_rv < 0) { @@ -2116,6 +2124,21 @@ static bool is_epoll_available() { return true; } +/* This is mainly for testing purposes. Checks to see if environment variable + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to the */ +static void set_max_pollers_per_island() { + char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); + if (s) { + int max_pollers = (int)strtol(s, NULL, 10); + if (max_pollers > 0) { + g_max_pollers_per_pi = max_pollers; + } + } + + gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", + g_max_pollers_per_pi); +} + const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { @@ -2134,6 +2157,8 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { grpc_use_signal(SIGRTMIN + 6); } + set_max_pollers_per_island(); + fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { -- cgit v1.2.3 From 93478976b7bc72e1cf44c843fa5505b0e5caf97d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 26 Apr 2017 20:39:03 -0700 Subject: Cleanup ev_epoll_linux_test and add some log statements in ev_epoll_linux.c --- src/core/lib/iomgr/ev_epoll_linux.c | 11 +++++--- test/core/iomgr/ev_epoll_linux_test.c | 48 ++--------------------------------- 2 files changed, 9 insertions(+), 50 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 3a5f48cb8f..1234828063 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -2125,14 +2125,17 @@ static bool is_epoll_available() { } /* This is mainly for testing purposes. Checks to see if environment variable - * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to the */ + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to + * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ static void set_max_pollers_per_island() { char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); if (s) { - int max_pollers = (int)strtol(s, NULL, 10); - if (max_pollers > 0) { - g_max_pollers_per_pi = max_pollers; + g_max_pollers_per_pi = (int)strtol(s, NULL, 10); + if (g_max_pollers_per_pi < 0) { + g_max_pollers_per_pi = INT_MAX; } + } else { + g_max_pollers_per_pi = INT_MAX; } gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 3813fb9cb0..0856023b14 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -38,10 +38,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include -#include -#include #include -#include #include #include @@ -330,7 +327,7 @@ static __thread int thread_wakeups = 0; static void test_threading_loop(void *arg) { threading_shared *shared = arg; - while (thread_wakeups < 20000) { + while (thread_wakeups < 1000000) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker; gpr_mu_lock(shared->mu); @@ -363,7 +360,7 @@ static void test_threading(void) { shared.pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(shared.pollset, &shared.mu); - gpr_thd_id thds[20]; + gpr_thd_id thds[10]; for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); @@ -402,46 +399,6 @@ static void test_threading(void) { gpr_free(shared.pollset); } -/* Convert milliseconds into 'struct timespec' struct. millis == -1 is - * * considered as an infinity-time in future */ -static struct timespec millis_to_timespec(int millis) { - struct timespec linux_ts; - gpr_timespec gpr_ts; - - if (millis == -1) { - gpr_ts = gpr_inf_future(GPR_TIMESPAN); - } else { - gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); - } - - linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; - linux_ts.tv_nsec = gpr_ts.tv_nsec; - return linux_ts; -} - -/* TODO (sreek) - Remove this test before merging. This is written just to - * understand the functionality of sigtimedwait and serves no other purpose */ -void test_sigwait() { - sigset_t wakeup_sig_set; - sigemptyset(&wakeup_sig_set); - sigaddset(&wakeup_sig_set, SIGRTMIN + 6); - int timeout_ms[] = {10, 100}; - - for (size_t i = 0; i < GPR_ARRAY_SIZE(timeout_ms); i++) { - struct timespec sigwait_timeout = millis_to_timespec(timeout_ms[i]); - gpr_log(GPR_ERROR, "sigwait_timeout: %ld, %ld", sigwait_timeout.tv_sec, - sigwait_timeout.tv_nsec); - - gpr_log(GPR_ERROR, "Waiting for %d ms...", timeout_ms[i]); - gpr_timespec bef = gpr_now(GPR_CLOCK_REALTIME); - sigtimedwait(&wakeup_sig_set, NULL, &sigwait_timeout); - gpr_timespec af = gpr_now(GPR_CLOCK_REALTIME); - - gpr_log(GPR_ERROR, "Bef: %ld, %d", bef.tv_sec, bef.tv_nsec); - gpr_log(GPR_ERROR, "Aft: %ld, %d", af.tv_sec, af.tv_nsec); - } -} - int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); @@ -452,7 +409,6 @@ int main(int argc, char **argv) { test_add_fd_to_pollset(); test_pollset_queue_merge_items(); test_threading(); - test_sigwait(); } else { gpr_log(GPR_INFO, "Skipping the test. The test is only relevant for 'epoll' " -- cgit v1.2.3 From c67cc999e35061873b4ba8bd72ebfb4ff8b7c0b6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 10:15:51 -0700 Subject: Initial fork for singleton epoll poller --- CMakeLists.txt | 21 +- Makefile | 21 +- binding.gyp | 3 +- build.yaml | 6 +- config.m4 | 3 +- gRPC-Core.podspec | 9 +- grpc.gemspec | 6 +- package.xml | 6 +- src/core/lib/iomgr/ev_epoll1_linux.c | 579 ++++++ src/core/lib/iomgr/ev_epoll1_linux.h | 50 + src/core/lib/iomgr/ev_epoll_linux.c | 1965 -------------------- src/core/lib/iomgr/ev_epoll_linux.h | 48 - src/core/lib/iomgr/ev_epollsig_linux.c | 1965 ++++++++++++++++++++ src/core/lib/iomgr/ev_epollsig_linux.h | 48 + src/core/lib/iomgr/ev_posix.c | 6 +- src/python/grpcio/grpc_core_dependencies.py | 3 +- tools/doxygen/Doxyfile.c++.internal | 6 +- tools/doxygen/Doxyfile.core.internal | 6 +- tools/run_tests/generated/sources_and_headers.json | 9 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 7 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 10 +- .../grpc++_unsecure/grpc++_unsecure.vcxproj | 7 +- .../grpc++_unsecure.vcxproj.filters | 10 +- vsprojects/vcxproj/grpc/grpc.vcxproj | 7 +- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 10 +- .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 7 +- .../grpc_test_util/grpc_test_util.vcxproj.filters | 10 +- .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 7 +- .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 10 +- 29 files changed, 2777 insertions(+), 2068 deletions(-) create mode 100644 src/core/lib/iomgr/ev_epoll1_linux.c create mode 100644 src/core/lib/iomgr/ev_epoll1_linux.h delete mode 100644 src/core/lib/iomgr/ev_epoll_linux.c delete mode 100644 src/core/lib/iomgr/ev_epoll_linux.h create mode 100644 src/core/lib/iomgr/ev_epollsig_linux.c create mode 100644 src/core/lib/iomgr/ev_epollsig_linux.h (limited to 'src/core') diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fd3b40cfa..2debe0422e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -936,7 +936,8 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1261,7 +1262,8 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1571,7 +1573,8 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1827,7 +1830,8 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2246,7 +2250,8 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2571,7 +2576,8 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -3335,7 +3341,8 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c diff --git a/Makefile b/Makefile index 04e681480b..b5d8b5b254 100644 --- a/Makefile +++ b/Makefile @@ -2916,7 +2916,8 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3239,7 +3240,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3548,7 +3550,8 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3776,7 +3779,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4172,7 +4176,8 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4505,7 +4510,8 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -5266,7 +5272,8 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/binding.gyp b/binding.gyp index 7974a7eb04..70538f9b43 100644 --- a/binding.gyp +++ b/binding.gyp @@ -672,7 +672,8 @@ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll1_linux.c', + 'src/core/lib/iomgr/ev_epollsig_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/build.yaml b/build.yaml index f5ada4f34f..89b356c8f2 100644 --- a/build.yaml +++ b/build.yaml @@ -197,7 +197,8 @@ filegroups: - src/core/lib/iomgr/endpoint_pair.h - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h - - src/core/lib/iomgr/ev_epoll_linux.h + - src/core/lib/iomgr/ev_epoll1_linux.h + - src/core/lib/iomgr/ev_epollsig_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -304,7 +305,8 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_uv.c - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - - src/core/lib/iomgr/ev_epoll_linux.c + - src/core/lib/iomgr/ev_epoll1_linux.c + - src/core/lib/iomgr/ev_epollsig_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c - src/core/lib/iomgr/exec_ctx.c diff --git a/config.m4 b/config.m4 index 98e48e5454..ddd260816c 100644 --- a/config.m4 +++ b/config.m4 @@ -108,7 +108,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index a81d37cd63..f970a8ec01 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -279,7 +279,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', - 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll1_linux.h', + 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -484,7 +485,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll1_linux.c', + 'src/core/lib/iomgr/ev_epollsig_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', @@ -738,7 +740,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', - 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll1_linux.h', + 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/grpc.gemspec b/grpc.gemspec index 677839d495..c23bcd7083 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -195,7 +195,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair.h ) s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) - s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -400,7 +401,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_uv.c ) s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) - s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) diff --git a/package.xml b/package.xml index 771e34cffc..2081b0c3b7 100644 --- a/package.xml +++ b/package.xml @@ -204,7 +204,8 @@ - + + @@ -409,7 +410,8 @@ - + + diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c new file mode 100644 index 0000000000..3c7a16b5ed --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -0,0 +1,579 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; +static int g_epfd; + +/******************************************************************************* + * Fd Declarations + */ + +struct grpc_fd { + int fd; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Pollset Declarations + */ + +typedef struct pollset_worker_link { + grpc_pollset_worker *next; + grpc_pollset_worker *prev; +} pollset_worker_link; + +typedef enum { + PWL_POLLSET, + PWL_POLLABLE, + POLLSET_WORKER_LINK_COUNT +} pollset_worker_links; + +struct grpc_pollset_worker { + bool kicked; + bool initialized_cv; + pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; + gpr_cv cv; +}; + +struct grpc_pollset { + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set {}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->po.mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->po.mu); + } + + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->po.mu); + new_fd->po.pi = NULL; +#ifdef PO_DEBUG + new_fd->po.obj_type = POLL_OBJ_FD; +#endif + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->po.mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->po.mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->po.mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->po.mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->po.pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->po.pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->po.pi != NULL) { + polling_island *pi_latest = polling_island_lock(fd->po.pi); + polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi_latest->mu); + + unref_pi = fd->po.pi; + fd->po.pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->po.mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + gpr_mu_lock(&fd->po.mu); + grpc_workqueue *workqueue = + GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); + gpr_mu_unlock(&fd->po.mu); + return workqueue; +} + +/******************************************************************************* + * Pollset Definitions + */ + +/* Return true if first in list */ +static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, + grpc_pollset_worker *worker) { + if (*root == NULL) { + *root = worker; + worker->links[link].next = worker->links[link].prev = worker; + return true; + } else { + worker->links[link].next = *root; + worker->links[link].prev = worker->links[link].next->links[link].prev; + worker->links[link].next->links[link].prev = worker; + worker->links[link].prev->links[link].next = worker; + return false; + } +} + +/* Return true if last in list */ +typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; + +static worker_remove_result worker_remove(grpc_pollset_worker **root, + pollset_worker_links link, + grpc_pollset_worker *worker) { + if (worker == *root) { + if (worker == worker->links[link].next) { + *root = NULL; + return EMPTIED; + } else { + *root = worker->links[link].next; + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return NEW_ROOT; + } + } else { + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return REMOVED; + } +} + +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + return (grpc_pollset_set *)((intptr_t)0xdeafbeef); +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) {} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { + /* If use of signals is disabled, we cannot use epoll engine*/ + if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { + return NULL; + } + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + if (!is_grpc_wakeup_signal_initialized) { + grpc_use_signal(SIGRTMIN + 6); + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h new file mode 100644 index 0000000000..5d3651019b --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -0,0 +1,50 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +// a polling engine that utilizes a singleton epoll set and turnstile polling + +const grpc_event_engine_vtable *grpc_init_epoll1_linux(void); + +#ifdef GRPC_LINUX_EPOLL +void *grpc_fd_get_polling_island(grpc_fd *fd); +void *grpc_pollset_get_polling_island(grpc_pollset *ps); +bool grpc_are_polling_islands_equal(void *p, void *q); +#endif /* defined(GRPC_LINUX_EPOLL) */ + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c deleted file mode 100644 index e603a75593..0000000000 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ /dev/null @@ -1,1965 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/lib/iomgr/port.h" - -/* This polling engine is only relevant on linux kernels supporting epoll() */ -#ifdef GRPC_LINUX_EPOLL - -#include "src/core/lib/iomgr/ev_epoll_linux.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "src/core/lib/iomgr/ev_posix.h" -#include "src/core/lib/iomgr/iomgr_internal.h" -#include "src/core/lib/iomgr/lockfree_event.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/iomgr/wakeup_fd_posix.h" -#include "src/core/lib/iomgr/workqueue.h" -#include "src/core/lib/profiling/timers.h" -#include "src/core/lib/support/block_annotate.h" - -/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -static int grpc_polling_trace = 0; /* Disabled by default */ -#define GRPC_POLLING_TRACE(fmt, ...) \ - if (grpc_polling_trace) { \ - gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ - } - -/* Uncomment the following to enable extra checks on poll_object operations */ -/* #define PO_DEBUG */ - -static int grpc_wakeup_signal = -1; -static bool is_grpc_wakeup_signal_initialized = false; - -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - -/* Implements the function defined in grpc_posix.h. This function might be - * called before even calling grpc_init() to set either a different signal to - * use. If signum == -1, then the use of signals is disabled */ -void grpc_use_signal(int signum) { - grpc_wakeup_signal = signum; - is_grpc_wakeup_signal_initialized = true; - - if (grpc_wakeup_signal < 0) { - gpr_log(GPR_INFO, - "Use of signals is disabled. Epoll engine will not be used"); - } else { - gpr_log(GPR_INFO, "epoll engine will be using signal: %d", - grpc_wakeup_signal); - } -} - -struct polling_island; - -typedef enum { - POLL_OBJ_FD, - POLL_OBJ_POLLSET, - POLL_OBJ_POLLSET_SET -} poll_obj_type; - -typedef struct poll_obj { -#ifdef PO_DEBUG - poll_obj_type obj_type; -#endif - gpr_mu mu; - struct polling_island *pi; -} poll_obj; - -const char *poll_obj_string(poll_obj_type po_type) { - switch (po_type) { - case POLL_OBJ_FD: - return "fd"; - case POLL_OBJ_POLLSET: - return "pollset"; - case POLL_OBJ_POLLSET_SET: - return "pollset_set"; - } - - GPR_UNREACHABLE_CODE(return "UNKNOWN"); -} - -/******************************************************************************* - * Fd Declarations - */ - -#define FD_FROM_PO(po) ((grpc_fd *)(po)) - -struct grpc_fd { - poll_obj po; - - int fd; - /* refst format: - bit 0 : 1=Active / 0=Orphaned - bits 1-n : refcount - Ref/Unref by two to avoid altering the orphaned bit */ - gpr_atm refst; - - /* The fd is either closed or we relinquished control of it. In either - cases, this indicates that the 'fd' on this structure is no longer - valid */ - bool orphaned; - - gpr_atm read_closure; - gpr_atm write_closure; - - struct grpc_fd *freelist_next; - grpc_closure *on_done_closure; - - /* The pollset that last noticed that the fd is readable. The actual type - * stored in this is (grpc_pollset *) */ - gpr_atm read_notifier_pollset; - - grpc_iomgr_object iomgr_object; -}; - -/* Reference counting for fds */ -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd *fd); -static void fd_unref(grpc_fd *fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - -static void fd_global_init(void); -static void fd_global_shutdown(void); - -/******************************************************************************* - * Polling island Declarations - */ - -#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG - -#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) -#define PI_UNREF(exec_ctx, p, r) \ - pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) - -#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ - -#define PI_ADD_REF(p, r) pi_add_ref((p)) -#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) - -#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ - -/* This is also used as grpc_workqueue (by directly casing it) */ -typedef struct polling_island { - grpc_closure_scheduler workqueue_scheduler; - - gpr_mu mu; - /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement - the refcount. - Once the ref count becomes zero, this structure is destroyed which means - we should ensure that there is never a scenario where a PI_ADD_REF() is - racing with a PI_UNREF() that just made the ref_count zero. */ - gpr_atm ref_count; - - /* Pointer to the polling_island this merged into. - * merged_to value is only set once in polling_island's lifetime (and that too - * only if the island is merged with another island). Because of this, we can - * use gpr_atm type here so that we can do atomic access on this and reduce - * lock contention on 'mu' mutex. - * - * Note that if this field is not NULL (i.e not 0), all the remaining fields - * (except mu and ref_count) are invalid and must be ignored. */ - gpr_atm merged_to; - - /* Number of threads currently polling on this island */ - gpr_atm poller_count; - /* Mutex guarding the read end of the workqueue (must be held to pop from - * workqueue_items) */ - gpr_mu workqueue_read_mu; - /* Queue of closures to be executed */ - gpr_mpscq workqueue_items; - /* Count of items in workqueue_items */ - gpr_atm workqueue_item_count; - /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ - grpc_wakeup_fd workqueue_wakeup_fd; - - /* The fd of the underlying epoll set */ - int epoll_fd; - - /* The file descriptors in the epoll set */ - size_t fd_cnt; - size_t fd_capacity; - grpc_fd **fds; -} polling_island; - -/******************************************************************************* - * Pollset Declarations - */ -struct grpc_pollset_worker { - /* Thread id of this worker */ - pthread_t pt_id; - - /* Used to prevent a worker from getting kicked multiple times */ - gpr_atm is_kicked; - struct grpc_pollset_worker *next; - struct grpc_pollset_worker *prev; -}; - -struct grpc_pollset { - poll_obj po; - - grpc_pollset_worker root_worker; - bool kicked_without_pollers; - - bool shutting_down; /* Is the pollset shutting down ? */ - bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ - grpc_closure *shutdown_done; /* Called after after shutdown is complete */ -}; - -/******************************************************************************* - * Pollset-set Declarations - */ -struct grpc_pollset_set { - poll_obj po; -}; - -/******************************************************************************* - * Common helpers - */ - -static bool append_error(grpc_error **composite, grpc_error *error, - const char *desc) { - if (error == GRPC_ERROR_NONE) return true; - if (*composite == GRPC_ERROR_NONE) { - *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); - } - *composite = grpc_error_add_child(*composite, error); - return false; -} - -/******************************************************************************* - * Polling island Definitions - */ - -/* The wakeup fd that is used to wake up all threads in a Polling island. This - is useful in the polling island merge operation where we need to wakeup all - the threads currently polling the smaller polling island (so that they can - start polling the new/merged polling island) - - NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the - threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ -static grpc_wakeup_fd polling_island_wakeup_fd; - -/* The polling island being polled right now. - See comments in workqueue_maybe_wakeup for why this is tracked. */ -static __thread polling_island *g_current_thread_polling_island; - -/* Forward declaration */ -static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); -static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, - grpc_error *error); - -#ifdef GRPC_TSAN -/* Currently TSAN may incorrectly flag data races between epoll_ctl and - epoll_wait for any grpc_fd structs that are added to the epoll set via - epoll_ctl and are returned (within a very short window) via epoll_wait(). - - To work-around this race, we establish a happens-before relation between - the code just-before epoll_ctl() and the code after epoll_wait() by using - this atomic */ -gpr_atm g_epoll_sync; -#endif /* defined(GRPC_TSAN) */ - -static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { - workqueue_enqueue, workqueue_enqueue, "workqueue"}; - -static void pi_add_ref(polling_island *pi); -static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); - -#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -static void pi_add_ref_dbg(polling_island *pi, const char *reason, - const char *file, int line) { - long old_cnt = gpr_atm_acq_load(&pi->ref_count); - pi_add_ref(pi); - gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", - (void *)pi, old_cnt, old_cnt + 1, reason, file, line); -} - -static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, - const char *reason, const char *file, int line) { - long old_cnt = gpr_atm_acq_load(&pi->ref_count); - pi_unref(exec_ctx, pi); - gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", - (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); -} - -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, - const char *file, int line, - const char *reason) { - if (workqueue != NULL) { - pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); - } - return workqueue; -} - -static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, - const char *file, int line, const char *reason) { - if (workqueue != NULL) { - pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); - } -} -#else -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { - if (workqueue != NULL) { - pi_add_ref((polling_island *)workqueue); - } - return workqueue; -} - -static void workqueue_unref(grpc_exec_ctx *exec_ctx, - grpc_workqueue *workqueue) { - if (workqueue != NULL) { - pi_unref(exec_ctx, (polling_island *)workqueue); - } -} -#endif - -static void pi_add_ref(polling_island *pi) { - gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); -} - -static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { - /* If ref count went to zero, delete the polling island. - Note that this deletion not be done under a lock. Once the ref count goes - to zero, we are guaranteed that no one else holds a reference to the - polling island (and that there is no racing pi_add_ref() call either). - - Also, if we are deleting the polling island and the merged_to field is - non-empty, we should remove a ref to the merged_to polling island - */ - if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { - polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); - polling_island_delete(exec_ctx, pi); - if (next != NULL) { - PI_UNREF(exec_ctx, next, "pi_delete"); /* Recursive call */ - } - } -} - -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, - size_t fd_count, bool add_fd_refs, - grpc_error **error) { - int err; - size_t i; - struct epoll_event ev; - char *err_msg; - const char *err_desc = "polling_island_add_fds"; - -#ifdef GRPC_TSAN - /* See the definition of g_epoll_sync for more context */ - gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); -#endif /* defined(GRPC_TSAN) */ - - for (i = 0; i < fd_count; i++) { - ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); - ev.data.ptr = fds[i]; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); - - if (err < 0) { - if (errno != EEXIST) { - gpr_asprintf( - &err_msg, - "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", - pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - continue; - } - - if (pi->fd_cnt == pi->fd_capacity) { - pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); - pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); - } - - pi->fds[pi->fd_cnt++] = fds[i]; - if (add_fd_refs) { - GRPC_FD_REF(fds[i], "polling_island"); - } - } -} - -/* The caller is expected to hold pi->mu before calling this */ -static void polling_island_add_wakeup_fd_locked(polling_island *pi, - grpc_wakeup_fd *wakeup_fd, - grpc_error **error) { - struct epoll_event ev; - int err; - char *err_msg; - const char *err_desc = "polling_island_add_wakeup_fd"; - - ev.events = (uint32_t)(EPOLLIN | EPOLLET); - ev.data.ptr = wakeup_fd; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, - GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); - if (err < 0 && errno != EEXIST) { - gpr_asprintf(&err_msg, - "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " - "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), - errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } -} - -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_remove_all_fds_locked(polling_island *pi, - bool remove_fd_refs, - grpc_error **error) { - int err; - size_t i; - char *err_msg; - const char *err_desc = "polling_island_remove_fds"; - - for (i = 0; i < pi->fd_cnt; i++) { - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, NULL); - if (err < 0 && errno != ENOENT) { - gpr_asprintf(&err_msg, - "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with " - "error: %d (%s)", - pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - if (remove_fd_refs) { - GRPC_FD_UNREF(pi->fds[i], "polling_island"); - } - } - - pi->fd_cnt = 0; -} - -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, - bool is_fd_closed, - grpc_error **error) { - int err; - size_t i; - char *err_msg; - const char *err_desc = "polling_island_remove_fd"; - - /* If fd is already closed, then it would have been automatically been removed - from the epoll set */ - if (!is_fd_closed) { - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); - if (err < 0 && errno != ENOENT) { - gpr_asprintf( - &err_msg, - "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", - pi->epoll_fd, fd->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - } - - for (i = 0; i < pi->fd_cnt; i++) { - if (pi->fds[i] == fd) { - pi->fds[i] = pi->fds[--pi->fd_cnt]; - GRPC_FD_UNREF(fd, "polling_island"); - break; - } - } -} - -/* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, - grpc_fd *initial_fd, - grpc_error **error) { - polling_island *pi = NULL; - const char *err_desc = "polling_island_create"; - - *error = GRPC_ERROR_NONE; - - pi = gpr_malloc(sizeof(*pi)); - pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - gpr_mu_init(&pi->mu); - pi->fd_cnt = 0; - pi->fd_capacity = 0; - pi->fds = NULL; - pi->epoll_fd = -1; - - gpr_mu_init(&pi->workqueue_read_mu); - gpr_mpscq_init(&pi->workqueue_items); - gpr_atm_rel_store(&pi->workqueue_item_count, 0); - - gpr_atm_rel_store(&pi->ref_count, 0); - gpr_atm_rel_store(&pi->poller_count, 0); - gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); - - if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), - err_desc)) { - goto done; - } - - pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - - if (pi->epoll_fd < 0) { - append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); - goto done; - } - - polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); - polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); - - if (initial_fd != NULL) { - polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); - } - -done: - if (*error != GRPC_ERROR_NONE) { - polling_island_delete(exec_ctx, pi); - pi = NULL; - } - return pi; -} - -static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { - GPR_ASSERT(pi->fd_cnt == 0); - - if (pi->epoll_fd >= 0) { - close(pi->epoll_fd); - } - GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); - gpr_mu_destroy(&pi->workqueue_read_mu); - gpr_mpscq_destroy(&pi->workqueue_items); - gpr_mu_destroy(&pi->mu); - grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); - gpr_free(pi->fds); - gpr_free(pi); -} - -/* Attempts to gets the last polling island in the linked list (liked by the - * 'merged_to' field). Since this does not lock the polling island, there are no - * guarantees that the island returned is the last island */ -static polling_island *polling_island_maybe_get_latest(polling_island *pi) { - polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); - while (next != NULL) { - pi = next; - next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); - } - - return pi; -} - -/* Gets the lock on the *latest* polling island i.e the last polling island in - the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the - returned polling island's mu. - Usage: To lock/unlock polling island "pi", do the following: - polling_island *pi_latest = polling_island_lock(pi); - ... - ... critical section .. - ... - gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */ -static polling_island *polling_island_lock(polling_island *pi) { - polling_island *next = NULL; - - while (true) { - next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); - if (next == NULL) { - /* Looks like 'pi' is the last node in the linked list but unless we check - this by holding the pi->mu lock, we cannot be sure (i.e without the - pi->mu lock, we don't prevent island merges). - To be absolutely sure, check once more by holding the pi->mu lock */ - gpr_mu_lock(&pi->mu); - next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); - if (next == NULL) { - /* pi is infact the last node and we have the pi->mu lock. we're done */ - break; - } - - /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu - * isn't the lock we are interested in. Continue traversing the list */ - gpr_mu_unlock(&pi->mu); - } - - pi = next; - } - - return pi; -} - -/* Gets the lock on the *latest* polling islands in the linked lists pointed by - *p and *q (and also updates *p and *q to point to the latest polling islands) - - This function is needed because calling the following block of code to obtain - locks on polling islands (*p and *q) is prone to deadlocks. - { - polling_island_lock(*p, true); - polling_island_lock(*q, true); - } - - Usage/example: - polling_island *p1; - polling_island *p2; - .. - polling_island_lock_pair(&p1, &p2); - .. - .. Critical section with both p1 and p2 locked - .. - // Release locks: Always call polling_island_unlock_pair() to release locks - polling_island_unlock_pair(p1, p2); -*/ -static void polling_island_lock_pair(polling_island **p, polling_island **q) { - polling_island *pi_1 = *p; - polling_island *pi_2 = *q; - polling_island *next_1 = NULL; - polling_island *next_2 = NULL; - - /* The algorithm is simple: - - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and - keep updating pi_1 and pi_2) - - Then obtain locks on the islands by following a lock order rule of - locking polling_island with lower address first - Special case: Before obtaining the locks, check if pi_1 and pi_2 are - pointing to the same island. If that is the case, we can just call - polling_island_lock() - - After obtaining both the locks, double check that the polling islands - are still the last polling islands in their respective linked lists - (this is because there might have been polling island merges before - we got the lock) - - If the polling islands are the last islands, we are done. If not, - release the locks and continue the process from the first step */ - while (true) { - next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); - while (next_1 != NULL) { - pi_1 = next_1; - next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); - } - - next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); - while (next_2 != NULL) { - pi_2 = next_2; - next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); - } - - if (pi_1 == pi_2) { - pi_1 = pi_2 = polling_island_lock(pi_1); - break; - } - - if (pi_1 < pi_2) { - gpr_mu_lock(&pi_1->mu); - gpr_mu_lock(&pi_2->mu); - } else { - gpr_mu_lock(&pi_2->mu); - gpr_mu_lock(&pi_1->mu); - } - - next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); - next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); - if (next_1 == NULL && next_2 == NULL) { - break; - } - - gpr_mu_unlock(&pi_1->mu); - gpr_mu_unlock(&pi_2->mu); - } - - *p = pi_1; - *q = pi_2; -} - -static void polling_island_unlock_pair(polling_island *p, polling_island *q) { - if (p == q) { - gpr_mu_unlock(&p->mu); - } else { - gpr_mu_unlock(&p->mu); - gpr_mu_unlock(&q->mu); - } -} - -static void workqueue_maybe_wakeup(polling_island *pi) { - /* If this thread is the current poller, then it may be that it's about to - decrement the current poller count, so we need to look past this thread */ - bool is_current_poller = (g_current_thread_polling_island == pi); - gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; - gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - /* Only issue a wakeup if it's likely that some poller could come in and take - it right now. Note that since we do an anticipatory mpscq_pop every poll - loop, it's ok if we miss the wakeup here, as we'll get the work item when - the next poller enters anyway. */ - if (current_pollers > min_current_pollers_for_wakeup) { - GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", - grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); - } -} - -static void workqueue_move_items_to_parent(polling_island *q) { - polling_island *p = (polling_island *)gpr_atm_no_barrier_load(&q->merged_to); - if (p == NULL) { - return; - } - gpr_mu_lock(&q->workqueue_read_mu); - int num_added = 0; - while (gpr_atm_no_barrier_load(&q->workqueue_item_count) > 0) { - gpr_mpscq_node *n = gpr_mpscq_pop(&q->workqueue_items); - if (n != NULL) { - gpr_atm_no_barrier_fetch_add(&q->workqueue_item_count, -1); - gpr_atm_no_barrier_fetch_add(&p->workqueue_item_count, 1); - gpr_mpscq_push(&p->workqueue_items, n); - num_added++; - } - } - gpr_mu_unlock(&q->workqueue_read_mu); - if (num_added > 0) { - workqueue_maybe_wakeup(p); - } - workqueue_move_items_to_parent(p); -} - -static polling_island *polling_island_merge(polling_island *p, - polling_island *q, - grpc_error **error) { - /* Get locks on both the polling islands */ - polling_island_lock_pair(&p, &q); - - if (p != q) { - /* Make sure that p points to the polling island with fewer fds than q */ - if (p->fd_cnt > q->fd_cnt) { - GPR_SWAP(polling_island *, p, q); - } - - /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q - Note that the refcounts on the fds being moved will not change here. - This is why the last param in the following two functions is 'false') */ - polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error); - polling_island_remove_all_fds_locked(p, false, error); - - /* Wakeup all the pollers (if any) on p so that they pickup this change */ - polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error); - - /* Add the 'merged_to' link from p --> q */ - gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); - PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ - - workqueue_move_items_to_parent(p); - } - /* else if p == q, nothing needs to be done */ - - polling_island_unlock_pair(p, q); - - /* Return the merged polling island (Note that no merge would have happened - if p == q which is ok) */ - return q; -} - -static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, - grpc_error *error) { - GPR_TIMER_BEGIN("workqueue.enqueue", 0); - grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; - /* take a ref to the workqueue: otherwise it can happen that whatever events - * this kicks off ends up destroying the workqueue before this function - * completes */ - GRPC_WORKQUEUE_REF(workqueue, "enqueue"); - polling_island *pi = (polling_island *)workqueue; - gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); - closure->error_data.error = error; - gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); - if (last == 0) { - workqueue_maybe_wakeup(pi); - } - workqueue_move_items_to_parent(pi); - GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); - GPR_TIMER_END("workqueue.enqueue", 0); -} - -static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - polling_island *pi = (polling_island *)workqueue; - return workqueue == NULL ? grpc_schedule_on_exec_ctx - : &pi->workqueue_scheduler; -} - -static grpc_error *polling_island_global_init() { - grpc_error *error = GRPC_ERROR_NONE; - - error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); - if (error == GRPC_ERROR_NONE) { - error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); - } - - return error; -} - -static void polling_island_global_shutdown() { - grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); -} - -/******************************************************************************* - * Fd Definitions - */ - -/* We need to keep a freelist not because of any concerns of malloc performance - * but instead so that implementations with multiple threads in (for example) - * epoll_wait deal with the race between pollset removal and incoming poll - * notifications. - * - * The problem is that the poller ultimately holds a reference to this - * object, so it is very difficult to know when is safe to free it, at least - * without some expensive synchronization. - * - * If we keep the object freelisted, in the worst case losing this race just - * becomes a spurious read notification on a reused fd. - */ - -/* The alarm system needs to be able to wakeup 'some poller' sometimes - * (specifically when a new alarm needs to be triggered earlier than the next - * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a - * case occurs. */ - -static grpc_fd *fd_freelist = NULL; -static gpr_mu fd_freelist_mu; - -#ifdef GRPC_FD_REF_COUNT_DEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd *fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} - -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - /* Add the fd to the freelist */ - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); - - gpr_mu_unlock(&fd_freelist_mu); - } else { - GPR_ASSERT(old > n); - } -} - -/* Increment refcount by two to avoid changing the orphan bit */ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } -static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } -#endif - -static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } - -static void fd_global_shutdown(void) { - gpr_mu_lock(&fd_freelist_mu); - gpr_mu_unlock(&fd_freelist_mu); - while (fd_freelist != NULL) { - grpc_fd *fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->po.mu); - gpr_free(fd); - } - gpr_mu_destroy(&fd_freelist_mu); -} - -static grpc_fd *fd_create(int fd, const char *name) { - grpc_fd *new_fd = NULL; - - gpr_mu_lock(&fd_freelist_mu); - if (fd_freelist != NULL) { - new_fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - } - gpr_mu_unlock(&fd_freelist_mu); - - if (new_fd == NULL) { - new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->po.mu); - } - - /* Note: It is not really needed to get the new_fd->po.mu lock here. If this - * is a newly created fd (or an fd we got from the freelist), no one else - * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->po.mu); - new_fd->po.pi = NULL; -#ifdef PO_DEBUG - new_fd->po.obj_type = POLL_OBJ_FD; -#endif - - gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); - new_fd->fd = fd; - new_fd->orphaned = false; - grpc_lfev_init(&new_fd->read_closure); - grpc_lfev_init(&new_fd->write_closure); - gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); - - new_fd->freelist_next = NULL; - new_fd->on_done_closure = NULL; - - gpr_mu_unlock(&new_fd->po.mu); - - char *fd_name; - gpr_asprintf(&fd_name, "%s fd=%d", name, fd); - grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); -#ifdef GRPC_FD_REF_COUNT_DEBUG - gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); -#endif - gpr_free(fd_name); - return new_fd; -} - -static int fd_wrapped_fd(grpc_fd *fd) { - int ret_fd = -1; - gpr_mu_lock(&fd->po.mu); - if (!fd->orphaned) { - ret_fd = fd->fd; - } - gpr_mu_unlock(&fd->po.mu); - - return ret_fd; -} - -static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure *on_done, int *release_fd, - const char *reason) { - bool is_fd_closed = false; - grpc_error *error = GRPC_ERROR_NONE; - polling_island *unref_pi = NULL; - - gpr_mu_lock(&fd->po.mu); - fd->on_done_closure = on_done; - - /* If release_fd is not NULL, we should be relinquishing control of the file - descriptor fd->fd (but we still own the grpc_fd structure). */ - if (release_fd != NULL) { - *release_fd = fd->fd; - } else { - close(fd->fd); - is_fd_closed = true; - } - - fd->orphaned = true; - - /* Remove the active status but keep referenced. We want this grpc_fd struct - to be alive (and not added to freelist) until the end of this function */ - REF_BY(fd, 1, reason); - - /* Remove the fd from the polling island: - - Get a lock on the latest polling island (i.e the last island in the - linked list pointed by fd->po.pi). This is the island that - would actually contain the fd - - Remove the fd from the latest polling island - - Unlock the latest polling island - - Set fd->po.pi to NULL (but remove the ref on the polling island - before doing this.) */ - if (fd->po.pi != NULL) { - polling_island *pi_latest = polling_island_lock(fd->po.pi); - polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); - gpr_mu_unlock(&pi_latest->mu); - - unref_pi = fd->po.pi; - fd->po.pi = NULL; - } - - grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); - - gpr_mu_unlock(&fd->po.mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ - if (unref_pi != NULL) { - /* Unref stale polling island here, outside the fd lock above. - The polling island owns a workqueue which owns an fd, and unreffing - inside the lock can cause an eventual lock loop that makes TSAN very - unhappy. */ - PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); - } - GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); - GRPC_ERROR_UNREF(error); -} - -static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, - grpc_fd *fd) { - gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); - return (grpc_pollset *)notifier; -} - -static bool fd_is_shutdown(grpc_fd *fd) { - return grpc_lfev_is_shutdown(&fd->read_closure); -} - -/* Might be called multiple times */ -static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { - if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, - GRPC_ERROR_REF(why))) { - shutdown(fd->fd, SHUT_RDWR); - grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); - } - GRPC_ERROR_UNREF(why); -} - -static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure *closure) { - grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); -} - -static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure *closure) { - grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); -} - -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - gpr_mu_lock(&fd->po.mu); - grpc_workqueue *workqueue = - GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); - gpr_mu_unlock(&fd->po.mu); - return workqueue; -} - -/******************************************************************************* - * Pollset Definitions - */ -GPR_TLS_DECL(g_current_thread_pollset); -GPR_TLS_DECL(g_current_thread_worker); -static __thread bool g_initialized_sigmask; -static __thread sigset_t g_orig_sigmask; - -static void sig_handler(int sig_num) { -#ifdef GRPC_EPOLL_DEBUG - gpr_log(GPR_INFO, "Received signal %d", sig_num); -#endif -} - -static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } - -/* Global state management */ -static grpc_error *pollset_global_init(void) { - gpr_tls_init(&g_current_thread_pollset); - gpr_tls_init(&g_current_thread_worker); - poller_kick_init(); - return grpc_wakeup_fd_init(&global_wakeup_fd); -} - -static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); - gpr_tls_destroy(&g_current_thread_pollset); - gpr_tls_destroy(&g_current_thread_worker); -} - -static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { - grpc_error *err = GRPC_ERROR_NONE; - - /* Kick the worker only if it was not already kicked */ - if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { - GRPC_POLLING_TRACE( - "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", - (void *)worker, (long int)worker->pt_id); - int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); - if (err_num != 0) { - err = GRPC_OS_ERROR(err_num, "pthread_kill"); - } - } - return err; -} - -/* Return 1 if the pollset has active threads in pollset_work (pollset must - * be locked) */ -static int pollset_has_workers(grpc_pollset *p) { - return p->root_worker.next != &p->root_worker; -} - -static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; -} - -static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { - if (pollset_has_workers(p)) { - grpc_pollset_worker *w = p->root_worker.next; - remove_worker(p, w); - return w; - } else { - return NULL; - } -} - -static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->next = &p->root_worker; - worker->prev = worker->next->prev; - worker->prev->next = worker->next->prev = worker; -} - -static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev = &p->root_worker; - worker->next = worker->prev->next; - worker->prev->next = worker->next->prev = worker; -} - -/* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, - grpc_pollset_worker *specific_worker) { - GPR_TIMER_BEGIN("pollset_kick", 0); - grpc_error *error = GRPC_ERROR_NONE; - const char *err_desc = "Kick Failure"; - grpc_pollset_worker *worker = specific_worker; - if (worker != NULL) { - if (worker == GRPC_POLLSET_KICK_BROADCAST) { - if (pollset_has_workers(p)) { - GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); - for (worker = p->root_worker.next; worker != &p->root_worker; - worker = worker->next) { - if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { - append_error(&error, pollset_worker_kick(worker), err_desc); - } - } - GPR_TIMER_END("pollset_kick.broadcast", 0); - } else { - p->kicked_without_pollers = true; - } - } else { - GPR_TIMER_MARK("kicked_specifically", 0); - if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { - append_error(&error, pollset_worker_kick(worker), err_desc); - } - } - } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { - /* Since worker == NULL, it means that we can kick "any" worker on this - pollset 'p'. If 'p' happens to be the same pollset this thread is - currently polling (i.e in pollset_work() function), then there is no need - to kick any other worker since the current thread can just absorb the - kick. This is the reason why we enter this case only when - g_current_thread_pollset is != p */ - - GPR_TIMER_MARK("kick_anonymous", 0); - worker = pop_front_worker(p); - if (worker != NULL) { - GPR_TIMER_MARK("finally_kick", 0); - push_back_worker(p, worker); - append_error(&error, pollset_worker_kick(worker), err_desc); - } else { - GPR_TIMER_MARK("kicked_no_pollers", 0); - p->kicked_without_pollers = true; - } - } - - GPR_TIMER_END("pollset_kick", 0); - GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); - return error; -} - -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - -static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - gpr_mu_init(&pollset->po.mu); - *mu = &pollset->po.mu; - pollset->po.pi = NULL; -#ifdef PO_DEBUG - pollset->po.obj_type = POLL_OBJ_POLLSET; -#endif - - pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; - pollset->kicked_without_pollers = false; - - pollset->shutting_down = false; - pollset->finish_shutdown_called = false; - pollset->shutdown_done = NULL; -} - -/* Convert a timespec to milliseconds: - - Very small or negative poll times are clamped to zero to do a non-blocking - poll (which becomes spin polling) - - Other small values are rounded up to one millisecond - - Longer than a millisecond polls are rounded up to the next nearest - millisecond to avoid spinning - - Infinite timeouts are converted to -1 */ -static int poll_deadline_to_millis_timeout(gpr_timespec deadline, - gpr_timespec now) { - gpr_timespec timeout; - static const int64_t max_spin_polling_us = 10; - if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { - return -1; - } - - if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( - max_spin_polling_us, - GPR_TIMESPAN))) <= 0) { - return 0; - } - timeout = gpr_time_sub(deadline, now); - int millis = gpr_time_to_millis(gpr_time_add( - timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); - return millis >= 1 ? millis : 1; -} - -static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_pollset *notifier) { - grpc_lfev_set_ready(exec_ctx, &fd->read_closure); - - /* Note, it is possible that fd_become_readable might be called twice with - different 'notifier's when an fd becomes readable and it is in two epoll - sets (This can happen briefly during polling island merges). In such cases - it does not really matter which notifer is set as the read_notifier_pollset - (They would both point to the same polling island anyway) */ - /* Use release store to match with acquire load in fd_get_read_notifier */ - gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); -} - -static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - grpc_lfev_set_ready(exec_ctx, &fd->write_closure); -} - -static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, - grpc_pollset *ps, char *reason) { - if (ps->po.pi != NULL) { - PI_UNREF(exec_ctx, ps->po.pi, reason); - } - ps->po.pi = NULL; -} - -static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset) { - /* The pollset cannot have any workers if we are at this stage */ - GPR_ASSERT(!pollset_has_workers(pollset)); - - pollset->finish_shutdown_called = true; - - /* Release the ref and set pollset->po.pi to NULL */ - pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); - grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); -} - -/* pollset->po.mu lock must be held by the caller before calling this */ -static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_closure *closure) { - GPR_TIMER_BEGIN("pollset_shutdown", 0); - GPR_ASSERT(!pollset->shutting_down); - pollset->shutting_down = true; - pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - - /* If the pollset has any workers, we cannot call finish_shutdown_locked() - because it would release the underlying polling island. In such a case, we - let the last worker call finish_shutdown_locked() from pollset_work() */ - if (!pollset_has_workers(pollset)) { - GPR_ASSERT(!pollset->finish_shutdown_called); - GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); - finish_shutdown_locked(exec_ctx, pollset); - } - GPR_TIMER_END("pollset_shutdown", 0); -} - -/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other - * than destroying the mutexes, there is nothing special that needs to be done - * here */ -static void pollset_destroy(grpc_pollset *pollset) { - GPR_ASSERT(!pollset_has_workers(pollset)); - gpr_mu_destroy(&pollset->po.mu); -} - -static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, - polling_island *pi) { - if (gpr_mu_trylock(&pi->workqueue_read_mu)) { - gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); - gpr_mu_unlock(&pi->workqueue_read_mu); - if (n != NULL) { - if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { - workqueue_maybe_wakeup(pi); - } - grpc_closure *c = (grpc_closure *)n; - grpc_error *error = c->error_data.error; -#ifndef NDEBUG - c->scheduled = false; -#endif - c->cb(exec_ctx, c->cb_arg, error); - GRPC_ERROR_UNREF(error); - return true; - } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { - /* n == NULL might mean there's work but it's not available to be popped - * yet - try to ensure another workqueue wakes up to check shortly if so - */ - workqueue_maybe_wakeup(pi); - } - } - return false; -} - -#define GRPC_EPOLL_MAX_EVENTS 100 -/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ -static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, - grpc_pollset_worker *worker, int timeout_ms, - sigset_t *sig_mask, grpc_error **error) { - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; - int epoll_fd = -1; - int ep_rv; - polling_island *pi = NULL; - char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; - GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); - - /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the - latest polling island pointed by pollset->po.pi - - Since epoll_fd is immutable, we can read it without obtaining the polling - island lock. There is however a possibility that the polling island (from - which we got the epoll_fd) got merged with another island while we are - in this function. This is still okay because in such a case, we will wakeup - right-away from epoll_wait() and pick up the latest polling_island the next - this function (i.e pollset_work_and_unlock()) is called */ - - if (pollset->po.pi == NULL) { - pollset->po.pi = polling_island_create(exec_ctx, NULL, error); - if (pollset->po.pi == NULL) { - GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. We cannot continue */ - } - - PI_ADD_REF(pollset->po.pi, "ps"); - GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", - (void *)pollset, (void *)pollset->po.pi); - } - - pi = polling_island_maybe_get_latest(pollset->po.pi); - epoll_fd = pi->epoll_fd; - - /* Update the pollset->po.pi since the island being pointed by - pollset->po.pi maybe older than the one pointed by pi) */ - if (pollset->po.pi != pi) { - /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the - polling island to be deleted */ - PI_ADD_REF(pi, "ps"); - PI_UNREF(exec_ctx, pollset->po.pi, "ps"); - pollset->po.pi = pi; - } - - /* Add an extra ref so that the island does not get destroyed (which means - the epoll_fd won't be closed) while we are are doing an epoll_wait() on the - epoll_fd */ - PI_ADD_REF(pi, "ps_work"); - gpr_mu_unlock(&pollset->po.mu); - - /* If we get some workqueue work to do, it might end up completing an item on - the completion queue, so there's no need to poll... so we skip that and - redo the complete loop to verify */ - if (!maybe_do_workqueue_work(exec_ctx, pi)) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); - g_current_thread_polling_island = pi; - - GRPC_SCHEDULING_START_BLOCKING_REGION; - ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, - sig_mask); - GRPC_SCHEDULING_END_BLOCKING_REGION; - if (ep_rv < 0) { - if (errno != EINTR) { - gpr_asprintf(&err_msg, - "epoll_wait() epoll fd: %d failed with error: %d (%s)", - epoll_fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - } else { - /* We were interrupted. Save an interation by doing a zero timeout - epoll_wait to see if there are any other events of interest */ - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p received kick", - (void *)pollset, (void *)worker); - ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - } - } - -#ifdef GRPC_TSAN - /* See the definition of g_poll_sync for more details */ - gpr_atm_acq_load(&g_epoll_sync); -#endif /* defined(GRPC_TSAN) */ - - for (int i = 0; i < ep_rv; ++i) { - void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { - append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), - err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " - "%d) got merged", - (void *)pollset, (void *)worker, epoll_fd); - /* This means that our polling island is merged with a different - island. We do not have to do anything here since the subsequent call - to the function pollset_work_and_unlock() will pick up the correct - epoll_fd */ - } else { - grpc_fd *fd = data_ptr; - int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write_ev = ep_ev[i].events & EPOLLOUT; - if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); - } - if (write_ev || cancel) { - fd_become_writable(exec_ctx, fd); - } - } - } - - g_current_thread_polling_island = NULL; - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); - } - - GPR_ASSERT(pi != NULL); - - /* Before leaving, release the extra ref we added to the polling island. It - is important to use "pi" here (i.e our old copy of pollset->po.pi - that we got before releasing the polling island lock). This is because - pollset->po.pi pointer might get udpated in other parts of the - code when there is an island merge while we are doing epoll_wait() above */ - PI_UNREF(exec_ctx, pi, "ps_work"); - - GPR_TIMER_END("pollset_work_and_unlock", 0); -} - -/* pollset->po.mu lock must be held by the caller before calling this. - The function pollset_work() may temporarily release the lock (pollset->po.mu) - during the course of its execution but it will always re-acquire the lock and - ensure that it is held by the time the function returns */ -static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker **worker_hdl, - gpr_timespec now, gpr_timespec deadline) { - GPR_TIMER_BEGIN("pollset_work", 0); - grpc_error *error = GRPC_ERROR_NONE; - int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); - - sigset_t new_mask; - - grpc_pollset_worker worker; - worker.next = worker.prev = NULL; - worker.pt_id = pthread_self(); - gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); - - if (worker_hdl) *worker_hdl = &worker; - - gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); - gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); - - if (pollset->kicked_without_pollers) { - /* If the pollset was kicked without pollers, pretend that the current - worker got the kick and skip polling. A kick indicates that there is some - work that needs attention like an event on the completion queue or an - alarm */ - GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); - pollset->kicked_without_pollers = 0; - } else if (!pollset->shutting_down) { - /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up - (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the - worker that there is some pending work that needs immediate attention - (like an event on the completion queue, or a polling island merge that - results in a new epoll-fd to wait on) and that the worker should not - spend time waiting in epoll_pwait(). - - A worker can be kicked anytime from the point it is added to the pollset - via push_front_worker() (or push_back_worker()) to the point it is - removed via remove_worker(). - If the worker is kicked before/during it calls epoll_pwait(), it should - immediately exit from epoll_wait(). If the worker is kicked after it - returns from epoll_wait(), then nothing really needs to be done. - - To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all - times *except* when it is in epoll_pwait(). This way, the worker never - misses acting on a kick */ - - if (!g_initialized_sigmask) { - sigemptyset(&new_mask); - sigaddset(&new_mask, grpc_wakeup_signal); - pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); - sigdelset(&g_orig_sigmask, grpc_wakeup_signal); - g_initialized_sigmask = true; - /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. - This is the mask used at all times *except during - epoll_wait()*" - g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and - this is the mask to use *during epoll_wait()* - - The new_mask is set on the worker before it is added to the pollset - (i.e before it can be kicked) */ - } - - push_front_worker(pollset, &worker); /* Add worker to pollset */ - - pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms, - &g_orig_sigmask, &error); - grpc_exec_ctx_flush(exec_ctx); - - gpr_mu_lock(&pollset->po.mu); - - /* Note: There is no need to reset worker.is_kicked to 0 since we are no - longer going to use this worker */ - remove_worker(pollset, &worker); - } - - /* If we are the last worker on the pollset (i.e pollset_has_workers() is - false at this point) and the pollset is shutting down, we may have to - finish the shutdown process by calling finish_shutdown_locked(). - See pollset_shutdown() for more details. - - Note: Continuing to access pollset here is safe; it is the caller's - responsibility to not destroy a pollset when it has outstanding calls to - pollset_work() */ - if (pollset->shutting_down && !pollset_has_workers(pollset) && - !pollset->finish_shutdown_called) { - GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); - finish_shutdown_locked(exec_ctx, pollset); - - gpr_mu_unlock(&pollset->po.mu); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); - } - - if (worker_hdl) *worker_hdl = NULL; - - gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); - gpr_tls_set(&g_current_thread_worker, (intptr_t)0); - - GPR_TIMER_END("pollset_work", 0); - - GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); - return error; -} - -static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag, - poll_obj_type bag_type, poll_obj *item, - poll_obj_type item_type) { - GPR_TIMER_BEGIN("add_poll_object", 0); - -#ifdef PO_DEBUG - GPR_ASSERT(item->obj_type == item_type); - GPR_ASSERT(bag->obj_type == bag_type); -#endif - - grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi_new = NULL; - - gpr_mu_lock(&bag->mu); - gpr_mu_lock(&item->mu); - -retry: - /* - * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing - * 2) If item->pi and bag->pi are both NULL, create a new polling island (with - * a refcount of 2) and point item->pi and bag->pi to the new island - * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to - * the other's non-NULL pi - * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the - * polling islands and update item->pi and bag->pi to point to the new - * island - */ - - /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already - * orphaned */ - if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) { - gpr_mu_unlock(&item->mu); - gpr_mu_unlock(&bag->mu); - return; - } - - if (item->pi == bag->pi) { - pi_new = item->pi; - if (pi_new == NULL) { - /* GPR_ASSERT(item->pi == bag->pi == NULL) */ - - /* If we are adding an fd to a bag (i.e pollset or pollset_set), then - * we need to do some extra work to make TSAN happy */ - if (item_type == POLL_OBJ_FD) { - /* Unlock before creating a new polling island: the polling island will - create a workqueue which creates a file descriptor, and holding an fd - lock here can eventually cause a loop to appear to TSAN (making it - unhappy). We don't think it's a real loop (there's an epoch point - where that loop possibility disappears), but the advantages of - keeping TSAN happy outweigh any performance advantage we might have - by keeping the lock held. */ - gpr_mu_unlock(&item->mu); - pi_new = polling_island_create(exec_ctx, FD_FROM_PO(item), &error); - gpr_mu_lock(&item->mu); - - /* Need to reverify any assumptions made between the initial lock and - getting to this branch: if they've changed, we need to throw away our - work and figure things out again. */ - if (item->pi != NULL) { - GRPC_POLLING_TRACE( - "add_poll_object: Raced creating new polling island. pi_new: %p " - "(fd: %d, %s: %p)", - (void *)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type), - (void *)bag); - /* No need to lock 'pi_new' here since this is a new polling island - and no one has a reference to it yet */ - polling_island_remove_all_fds_locked(pi_new, true, &error); - - /* Ref and unref so that the polling island gets deleted during unref - */ - PI_ADD_REF(pi_new, "dance_of_destruction"); - PI_UNREF(exec_ctx, pi_new, "dance_of_destruction"); - goto retry; - } - } else { - pi_new = polling_island_create(exec_ctx, NULL, &error); - } - - GRPC_POLLING_TRACE( - "add_poll_object: Created new polling island. pi_new: %p (%s: %p, " - "%s: %p)", - (void *)pi_new, poll_obj_string(item_type), (void *)item, - poll_obj_string(bag_type), (void *)bag); - } else { - GRPC_POLLING_TRACE( - "add_poll_object: Same polling island. pi: %p (%s, %s)", - (void *)pi_new, poll_obj_string(item_type), - poll_obj_string(bag_type)); - } - } else if (item->pi == NULL) { - /* GPR_ASSERT(bag->pi != NULL) */ - /* Make pi_new point to latest pi*/ - pi_new = polling_island_lock(bag->pi); - - if (item_type == POLL_OBJ_FD) { - grpc_fd *fd = FD_FROM_PO(item); - polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); - } - - gpr_mu_unlock(&pi_new->mu); - GRPC_POLLING_TRACE( - "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, " - "bag(%s): %p)", - (void *)pi_new, poll_obj_string(item_type), (void *)item, - poll_obj_string(bag_type), (void *)bag); - } else if (bag->pi == NULL) { - /* GPR_ASSERT(item->pi != NULL) */ - /* Make pi_new to point to latest pi */ - pi_new = polling_island_lock(item->pi); - gpr_mu_unlock(&pi_new->mu); - GRPC_POLLING_TRACE( - "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, " - "bag(%s): %p)", - (void *)pi_new, poll_obj_string(item_type), (void *)item, - poll_obj_string(bag_type), (void *)bag); - } else { - pi_new = polling_island_merge(item->pi, bag->pi, &error); - GRPC_POLLING_TRACE( - "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, " - "bag(%s): %p)", - (void *)pi_new, poll_obj_string(item_type), (void *)item, - poll_obj_string(bag_type), (void *)bag); - } - - /* At this point, pi_new is the polling island that both item->pi and bag->pi - MUST be pointing to */ - - if (item->pi != pi_new) { - PI_ADD_REF(pi_new, poll_obj_string(item_type)); - if (item->pi != NULL) { - PI_UNREF(exec_ctx, item->pi, poll_obj_string(item_type)); - } - item->pi = pi_new; - } - - if (bag->pi != pi_new) { - PI_ADD_REF(pi_new, poll_obj_string(bag_type)); - if (bag->pi != NULL) { - PI_UNREF(exec_ctx, bag->pi, poll_obj_string(bag_type)); - } - bag->pi = pi_new; - } - - gpr_mu_unlock(&item->mu); - gpr_mu_unlock(&bag->mu); - - GRPC_LOG_IF_ERROR("add_poll_object", error); - GPR_TIMER_END("add_poll_object", 0); -} - -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { - add_poll_object(exec_ctx, &pollset->po, POLL_OBJ_POLLSET, &fd->po, - POLL_OBJ_FD); -} - -/******************************************************************************* - * Pollset-set Definitions - */ - -static grpc_pollset_set *pollset_set_create(void) { - grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); - gpr_mu_init(&pss->po.mu); - pss->po.pi = NULL; -#ifdef PO_DEBUG - pss->po.obj_type = POLL_OBJ_POLLSET_SET; -#endif - return pss; -} - -static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss) { - gpr_mu_destroy(&pss->po.mu); - - if (pss->po.pi != NULL) { - PI_UNREF(exec_ctx, pss->po.pi, "pss_destroy"); - } - - gpr_free(pss); -} - -static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - grpc_fd *fd) { - add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &fd->po, - POLL_OBJ_FD); -} - -static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - grpc_fd *fd) { - /* Nothing to do */ -} - -static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss, grpc_pollset *ps) { - add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &ps->po, - POLL_OBJ_POLLSET); -} - -static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss, grpc_pollset *ps) { - /* Nothing to do */ -} - -static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) { - add_poll_object(exec_ctx, &bag->po, POLL_OBJ_POLLSET_SET, &item->po, - POLL_OBJ_POLLSET_SET); -} - -static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) { - /* Nothing to do */ -} - -/* Test helper functions - * */ -void *grpc_fd_get_polling_island(grpc_fd *fd) { - polling_island *pi; - - gpr_mu_lock(&fd->po.mu); - pi = fd->po.pi; - gpr_mu_unlock(&fd->po.mu); - - return pi; -} - -void *grpc_pollset_get_polling_island(grpc_pollset *ps) { - polling_island *pi; - - gpr_mu_lock(&ps->po.mu); - pi = ps->po.pi; - gpr_mu_unlock(&ps->po.mu); - - return pi; -} - -bool grpc_are_polling_islands_equal(void *p, void *q) { - polling_island *p1 = p; - polling_island *p2 = q; - - /* Note: polling_island_lock_pair() may change p1 and p2 to point to the - latest polling islands in their respective linked lists */ - polling_island_lock_pair(&p1, &p2); - polling_island_unlock_pair(p1, p2); - - return p1 == p2; -} - -/******************************************************************************* - * Event engine binding - */ - -static void shutdown_engine(void) { - fd_global_shutdown(); - pollset_global_shutdown(); - polling_island_global_shutdown(); -} - -static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - .fd_get_workqueue = fd_get_workqueue, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .kick_poller = kick_poller, - - .workqueue_ref = workqueue_ref, - .workqueue_unref = workqueue_unref, - .workqueue_scheduler = workqueue_scheduler, - - .shutdown_engine = shutdown_engine, -}; - -/* It is possible that GLIBC has epoll but the underlying kernel doesn't. - * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epoll_available() { - int fd = epoll_create1(EPOLL_CLOEXEC); - if (fd < 0) { - gpr_log( - GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epoll polling engine", - fd); - return false; - } - close(fd); - return true; -} - -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { - /* If use of signals is disabled, we cannot use epoll engine*/ - if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { - return NULL; - } - - if (!grpc_has_wakeup_fd()) { - return NULL; - } - - if (!is_epoll_available()) { - return NULL; - } - - if (!is_grpc_wakeup_signal_initialized) { - grpc_use_signal(SIGRTMIN + 6); - } - - fd_global_init(); - - if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { - return NULL; - } - - if (!GRPC_LOG_IF_ERROR("polling_island_global_init", - polling_island_global_init())) { - return NULL; - } - - return &vtable; -} - -#else /* defined(GRPC_LINUX_EPOLL) */ -#if defined(GRPC_POSIX_SOCKET) -#include "src/core/lib/iomgr/ev_posix.h" -/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return - * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } -#endif /* defined(GRPC_POSIX_SOCKET) */ - -void grpc_use_signal(int signum) {} -#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.h b/src/core/lib/iomgr/ev_epoll_linux.h deleted file mode 100644 index 8fc3ff59a3..0000000000 --- a/src/core/lib/iomgr/ev_epoll_linux.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H -#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H - -#include "src/core/lib/iomgr/ev_posix.h" -#include "src/core/lib/iomgr/port.h" - -const grpc_event_engine_vtable *grpc_init_epoll_linux(void); - -#ifdef GRPC_LINUX_EPOLL -void *grpc_fd_get_polling_island(grpc_fd *fd); -void *grpc_pollset_get_polling_island(grpc_pollset *ps); -bool grpc_are_polling_islands_equal(void *p, void *q); -#endif /* defined(GRPC_LINUX_EPOLL) */ - -#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c new file mode 100644 index 0000000000..ee2af081f0 --- /dev/null +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -0,0 +1,1965 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ +static int grpc_polling_trace = 0; /* Disabled by default */ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (grpc_polling_trace) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +/* Uncomment the following to enable extra checks on poll_object operations */ +/* #define PO_DEBUG */ + +static int grpc_wakeup_signal = -1; +static bool is_grpc_wakeup_signal_initialized = false; + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +/* Implements the function defined in grpc_posix.h. This function might be + * called before even calling grpc_init() to set either a different signal to + * use. If signum == -1, then the use of signals is disabled */ +void grpc_use_signal(int signum) { + grpc_wakeup_signal = signum; + is_grpc_wakeup_signal_initialized = true; + + if (grpc_wakeup_signal < 0) { + gpr_log(GPR_INFO, + "Use of signals is disabled. Epoll engine will not be used"); + } else { + gpr_log(GPR_INFO, "epoll engine will be using signal: %d", + grpc_wakeup_signal); + } +} + +struct polling_island; + +typedef enum { + POLL_OBJ_FD, + POLL_OBJ_POLLSET, + POLL_OBJ_POLLSET_SET +} poll_obj_type; + +typedef struct poll_obj { +#ifdef PO_DEBUG + poll_obj_type obj_type; +#endif + gpr_mu mu; + struct polling_island *pi; +} poll_obj; + +const char *poll_obj_string(poll_obj_type po_type) { + switch (po_type) { + case POLL_OBJ_FD: + return "fd"; + case POLL_OBJ_POLLSET: + return "pollset"; + case POLL_OBJ_POLLSET_SET: + return "pollset_set"; + } + + GPR_UNREACHABLE_CODE(return "UNKNOWN"); +} + +/******************************************************************************* + * Fd Declarations + */ + +#define FD_FROM_PO(po) ((grpc_fd *)(po)) + +struct grpc_fd { + poll_obj po; + + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Polling island Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define PI_UNREF(exec_ctx, p, r) \ + pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define PI_ADD_REF(p, r) pi_add_ref((p)) +#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ + +/* This is also used as grpc_workqueue (by directly casing it) */ +typedef struct polling_island { + grpc_closure_scheduler workqueue_scheduler; + + gpr_mu mu; + /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + the refcount. + Once the ref count becomes zero, this structure is destroyed which means + we should ensure that there is never a scenario where a PI_ADD_REF() is + racing with a PI_UNREF() that just made the ref_count zero. */ + gpr_atm ref_count; + + /* Pointer to the polling_island this merged into. + * merged_to value is only set once in polling_island's lifetime (and that too + * only if the island is merged with another island). Because of this, we can + * use gpr_atm type here so that we can do atomic access on this and reduce + * lock contention on 'mu' mutex. + * + * Note that if this field is not NULL (i.e not 0), all the remaining fields + * (except mu and ref_count) are invalid and must be ignored. */ + gpr_atm merged_to; + + /* Number of threads currently polling on this island */ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* The fd of the underlying epoll set */ + int epoll_fd; + + /* The file descriptors in the epoll set */ + size_t fd_cnt; + size_t fd_capacity; + grpc_fd **fds; +} polling_island; + +/******************************************************************************* + * Pollset Declarations + */ +struct grpc_pollset_worker { + /* Thread id of this worker */ + pthread_t pt_id; + + /* Used to prevent a worker from getting kicked multiple times */ + gpr_atm is_kicked; + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset { + poll_obj po; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + poll_obj po; +}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Polling island Definitions + */ + +/* The wakeup fd that is used to wake up all threads in a Polling island. This + is useful in the polling island merge operation where we need to wakeup all + the threads currently polling the smaller polling island (so that they can + start polling the new/merged polling island) + + NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the + threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd polling_island_wakeup_fd; + +/* The polling island being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread polling_island *g_current_thread_polling_island; + +/* Forward declaration */ +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void pi_add_ref(polling_island *pi); +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void pi_add_ref_dbg(polling_island *pi, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_add_ref(pi); + gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, old_cnt + 1, reason, file, line); +} + +static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_unref(exec_ctx, pi); + gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_add_ref((polling_island *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_unref(exec_ctx, (polling_island *)workqueue); + } +} +#endif + +static void pi_add_ref(polling_island *pi) { + gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +} + +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { + /* If ref count went to zero, delete the polling island. + Note that this deletion not be done under a lock. Once the ref count goes + to zero, we are guaranteed that no one else holds a reference to the + polling island (and that there is no racing pi_add_ref() call either). + + Also, if we are deleting the polling island and the merged_to field is + non-empty, we should remove a ref to the merged_to polling island + */ + if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + polling_island_delete(exec_ctx, pi); + if (next != NULL) { + PI_UNREF(exec_ctx, next, "pi_delete"); /* Recursive call */ + } + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, + size_t fd_count, bool add_fd_refs, + grpc_error **error) { + int err; + size_t i; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "polling_island_add_fds"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + for (i = 0; i < fd_count; i++) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fds[i]; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); + + if (err < 0) { + if (errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + continue; + } + + if (pi->fd_cnt == pi->fd_capacity) { + pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); + pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); + } + + pi->fds[pi->fd_cnt++] = fds[i]; + if (add_fd_refs) { + GRPC_FD_REF(fds[i], "polling_island"); + } + } +} + +/* The caller is expected to hold pi->mu before calling this */ +static void polling_island_add_wakeup_fd_locked(polling_island *pi, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "polling_island_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_all_fds_locked(polling_island *pi, + bool remove_fd_refs, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fds"; + + for (i = 0; i < pi->fd_cnt; i++) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with " + "error: %d (%s)", + pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + if (remove_fd_refs) { + GRPC_FD_UNREF(pi->fds[i], "polling_island"); + } + } + + pi->fd_cnt = 0; +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } + + for (i = 0; i < pi->fd_cnt; i++) { + if (pi->fds[i] == fd) { + pi->fds[i] = pi->fds[--pi->fd_cnt]; + GRPC_FD_UNREF(fd, "polling_island"); + break; + } + } +} + +/* Might return NULL in case of an error */ +static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, + grpc_fd *initial_fd, + grpc_error **error) { + polling_island *pi = NULL; + const char *err_desc = "polling_island_create"; + + *error = GRPC_ERROR_NONE; + + pi = gpr_malloc(sizeof(*pi)); + pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + gpr_mu_init(&pi->mu); + pi->fd_cnt = 0; + pi->fd_capacity = 0; + pi->fds = NULL; + pi->epoll_fd = -1; + + gpr_mu_init(&pi->workqueue_read_mu); + gpr_mpscq_init(&pi->workqueue_items); + gpr_atm_rel_store(&pi->workqueue_item_count, 0); + + gpr_atm_rel_store(&pi->ref_count, 0); + gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); + + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (pi->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); + polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + + if (initial_fd != NULL) { + polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); + } + +done: + if (*error != GRPC_ERROR_NONE) { + polling_island_delete(exec_ctx, pi); + pi = NULL; + } + return pi; +} + +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { + GPR_ASSERT(pi->fd_cnt == 0); + + if (pi->epoll_fd >= 0) { + close(pi->epoll_fd); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); + gpr_mu_destroy(&pi->workqueue_read_mu); + gpr_mpscq_destroy(&pi->workqueue_items); + gpr_mu_destroy(&pi->mu); + grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + gpr_free(pi->fds); + gpr_free(pi); +} + +/* Attempts to gets the last polling island in the linked list (liked by the + * 'merged_to' field). Since this does not lock the polling island, there are no + * guarantees that the island returned is the last island */ +static polling_island *polling_island_maybe_get_latest(polling_island *pi) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + while (next != NULL) { + pi = next; + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + } + + return pi; +} + +/* Gets the lock on the *latest* polling island i.e the last polling island in + the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the + returned polling island's mu. + Usage: To lock/unlock polling island "pi", do the following: + polling_island *pi_latest = polling_island_lock(pi); + ... + ... critical section .. + ... + gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */ +static polling_island *polling_island_lock(polling_island *pi) { + polling_island *next = NULL; + + while (true) { + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* Looks like 'pi' is the last node in the linked list but unless we check + this by holding the pi->mu lock, we cannot be sure (i.e without the + pi->mu lock, we don't prevent island merges). + To be absolutely sure, check once more by holding the pi->mu lock */ + gpr_mu_lock(&pi->mu); + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* pi is infact the last node and we have the pi->mu lock. we're done */ + break; + } + + /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu + * isn't the lock we are interested in. Continue traversing the list */ + gpr_mu_unlock(&pi->mu); + } + + pi = next; + } + + return pi; +} + +/* Gets the lock on the *latest* polling islands in the linked lists pointed by + *p and *q (and also updates *p and *q to point to the latest polling islands) + + This function is needed because calling the following block of code to obtain + locks on polling islands (*p and *q) is prone to deadlocks. + { + polling_island_lock(*p, true); + polling_island_lock(*q, true); + } + + Usage/example: + polling_island *p1; + polling_island *p2; + .. + polling_island_lock_pair(&p1, &p2); + .. + .. Critical section with both p1 and p2 locked + .. + // Release locks: Always call polling_island_unlock_pair() to release locks + polling_island_unlock_pair(p1, p2); +*/ +static void polling_island_lock_pair(polling_island **p, polling_island **q) { + polling_island *pi_1 = *p; + polling_island *pi_2 = *q; + polling_island *next_1 = NULL; + polling_island *next_2 = NULL; + + /* The algorithm is simple: + - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and + keep updating pi_1 and pi_2) + - Then obtain locks on the islands by following a lock order rule of + locking polling_island with lower address first + Special case: Before obtaining the locks, check if pi_1 and pi_2 are + pointing to the same island. If that is the case, we can just call + polling_island_lock() + - After obtaining both the locks, double check that the polling islands + are still the last polling islands in their respective linked lists + (this is because there might have been polling island merges before + we got the lock) + - If the polling islands are the last islands, we are done. If not, + release the locks and continue the process from the first step */ + while (true) { + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + while (next_1 != NULL) { + pi_1 = next_1; + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + } + + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + while (next_2 != NULL) { + pi_2 = next_2; + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + } + + if (pi_1 == pi_2) { + pi_1 = pi_2 = polling_island_lock(pi_1); + break; + } + + if (pi_1 < pi_2) { + gpr_mu_lock(&pi_1->mu); + gpr_mu_lock(&pi_2->mu); + } else { + gpr_mu_lock(&pi_2->mu); + gpr_mu_lock(&pi_1->mu); + } + + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + if (next_1 == NULL && next_2 == NULL) { + break; + } + + gpr_mu_unlock(&pi_1->mu); + gpr_mu_unlock(&pi_2->mu); + } + + *p = pi_1; + *q = pi_2; +} + +static void polling_island_unlock_pair(polling_island *p, polling_island *q) { + if (p == q) { + gpr_mu_unlock(&p->mu); + } else { + gpr_mu_unlock(&p->mu); + gpr_mu_unlock(&q->mu); + } +} + +static void workqueue_maybe_wakeup(polling_island *pi) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_polling_island == pi); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + } +} + +static void workqueue_move_items_to_parent(polling_island *q) { + polling_island *p = (polling_island *)gpr_atm_no_barrier_load(&q->merged_to); + if (p == NULL) { + return; + } + gpr_mu_lock(&q->workqueue_read_mu); + int num_added = 0; + while (gpr_atm_no_barrier_load(&q->workqueue_item_count) > 0) { + gpr_mpscq_node *n = gpr_mpscq_pop(&q->workqueue_items); + if (n != NULL) { + gpr_atm_no_barrier_fetch_add(&q->workqueue_item_count, -1); + gpr_atm_no_barrier_fetch_add(&p->workqueue_item_count, 1); + gpr_mpscq_push(&p->workqueue_items, n); + num_added++; + } + } + gpr_mu_unlock(&q->workqueue_read_mu); + if (num_added > 0) { + workqueue_maybe_wakeup(p); + } + workqueue_move_items_to_parent(p); +} + +static polling_island *polling_island_merge(polling_island *p, + polling_island *q, + grpc_error **error) { + /* Get locks on both the polling islands */ + polling_island_lock_pair(&p, &q); + + if (p != q) { + /* Make sure that p points to the polling island with fewer fds than q */ + if (p->fd_cnt > q->fd_cnt) { + GPR_SWAP(polling_island *, p, q); + } + + /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q + Note that the refcounts on the fds being moved will not change here. + This is why the last param in the following two functions is 'false') */ + polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error); + polling_island_remove_all_fds_locked(p, false, error); + + /* Wakeup all the pollers (if any) on p so that they pickup this change */ + polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error); + + /* Add the 'merged_to' link from p --> q */ + gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); + PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ + + workqueue_move_items_to_parent(p); + } + /* else if p == q, nothing needs to be done */ + + polling_island_unlock_pair(p, q); + + /* Return the merged polling island (Note that no merge would have happened + if p == q which is ok) */ + return q; +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + polling_island *pi = (polling_island *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(pi); + } + workqueue_move_items_to_parent(pi); + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + polling_island *pi = (polling_island *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &pi->workqueue_scheduler; +} + +static grpc_error *polling_island_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + } + + return error; +} + +static void polling_island_global_shutdown() { + grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->po.mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->po.mu); + } + + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->po.mu); + new_fd->po.pi = NULL; +#ifdef PO_DEBUG + new_fd->po.obj_type = POLL_OBJ_FD; +#endif + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->po.mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->po.mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->po.mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->po.mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->po.pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->po.pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->po.pi != NULL) { + polling_island *pi_latest = polling_island_lock(fd->po.pi); + polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi_latest->mu); + + unref_pi = fd->po.pi; + fd->po.pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->po.mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + gpr_mu_lock(&fd->po.mu); + grpc_workqueue *workqueue = + GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); + gpr_mu_unlock(&fd->po.mu); + return workqueue; +} + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static __thread bool g_initialized_sigmask; +static __thread sigset_t g_orig_sigmask; + +static void sig_handler(int sig_num) { +#ifdef GRPC_EPOLL_DEBUG + gpr_log(GPR_INFO, "Received signal %d", sig_num); +#endif +} + +static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + poller_kick_init(); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + grpc_error *err = GRPC_ERROR_NONE; + + /* Kick the worker only if it was not already kicked */ + if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { + GRPC_POLLING_TRACE( + "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", + (void *)worker, (long int)worker->pt_id); + int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); + if (err_num != 0) { + err = GRPC_OS_ERROR(err_num, "pthread_kill"); + } + } + return err; +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->po.mu); + *mu = &pollset->po.mu; + pollset->po.pi = NULL; +#ifdef PO_DEBUG + pollset->po.obj_type = POLL_OBJ_POLLSET; +#endif + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; +} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, + grpc_pollset *ps, char *reason) { + if (ps->po.pi != NULL) { + PI_UNREF(exec_ctx, ps->po.pi, reason); + } + ps->po.pi = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->po.pi to NULL */ + pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying polling island. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->po.mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, + polling_island *pi) { + if (gpr_mu_trylock(&pi->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); + gpr_mu_unlock(&pi->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(pi); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(pi); + } + } + return false; +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ +static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker *worker, int timeout_ms, + sigset_t *sig_mask, grpc_error **error) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int epoll_fd = -1; + int ep_rv; + polling_island *pi = NULL; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + + /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the + latest polling island pointed by pollset->po.pi + + Since epoll_fd is immutable, we can read it without obtaining the polling + island lock. There is however a possibility that the polling island (from + which we got the epoll_fd) got merged with another island while we are + in this function. This is still okay because in such a case, we will wakeup + right-away from epoll_wait() and pick up the latest polling_island the next + this function (i.e pollset_work_and_unlock()) is called */ + + if (pollset->po.pi == NULL) { + pollset->po.pi = polling_island_create(exec_ctx, NULL, error); + if (pollset->po.pi == NULL) { + GPR_TIMER_END("pollset_work_and_unlock", 0); + return; /* Fatal error. We cannot continue */ + } + + PI_ADD_REF(pollset->po.pi, "ps"); + GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", + (void *)pollset, (void *)pollset->po.pi); + } + + pi = polling_island_maybe_get_latest(pollset->po.pi); + epoll_fd = pi->epoll_fd; + + /* Update the pollset->po.pi since the island being pointed by + pollset->po.pi maybe older than the one pointed by pi) */ + if (pollset->po.pi != pi) { + /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the + polling island to be deleted */ + PI_ADD_REF(pi, "ps"); + PI_UNREF(exec_ctx, pollset->po.pi, "ps"); + pollset->po.pi = pi; + } + + /* Add an extra ref so that the island does not get destroyed (which means + the epoll_fd won't be closed) while we are are doing an epoll_wait() on the + epoll_fd */ + PI_ADD_REF(pi, "ps_work"); + gpr_mu_unlock(&pollset->po.mu); + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, pi)) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); + g_current_thread_polling_island = pi; + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, + sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + + g_current_thread_polling_island = NULL; + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + } + + GPR_ASSERT(pi != NULL); + + /* Before leaving, release the extra ref we added to the polling island. It + is important to use "pi" here (i.e our old copy of pollset->po.pi + that we got before releasing the polling island lock). This is because + pollset->po.pi pointer might get udpated in other parts of the + code when there is an island merge while we are doing epoll_wait() above */ + PI_UNREF(exec_ctx, pi, "ps_work"); + + GPR_TIMER_END("pollset_work_and_unlock", 0); +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + sigset_t new_mask; + + grpc_pollset_worker worker; + worker.next = worker.prev = NULL; + worker.pt_id = pthread_self(); + gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up + (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the + worker that there is some pending work that needs immediate attention + (like an event on the completion queue, or a polling island merge that + results in a new epoll-fd to wait on) and that the worker should not + spend time waiting in epoll_pwait(). + + A worker can be kicked anytime from the point it is added to the pollset + via push_front_worker() (or push_back_worker()) to the point it is + removed via remove_worker(). + If the worker is kicked before/during it calls epoll_pwait(), it should + immediately exit from epoll_wait(). If the worker is kicked after it + returns from epoll_wait(), then nothing really needs to be done. + + To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all + times *except* when it is in epoll_pwait(). This way, the worker never + misses acting on a kick */ + + if (!g_initialized_sigmask) { + sigemptyset(&new_mask); + sigaddset(&new_mask, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); + sigdelset(&g_orig_sigmask, grpc_wakeup_signal); + g_initialized_sigmask = true; + /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. + This is the mask used at all times *except during + epoll_wait()*" + g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and + this is the mask to use *during epoll_wait()* + + The new_mask is set on the worker before it is added to the pollset + (i.e before it can be kicked) */ + } + + push_front_worker(pollset, &worker); /* Add worker to pollset */ + + pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms, + &g_orig_sigmask, &error); + grpc_exec_ctx_flush(exec_ctx); + + gpr_mu_lock(&pollset->po.mu); + + /* Note: There is no need to reset worker.is_kicked to 0 since we are no + longer going to use this worker */ + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->po.mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag, + poll_obj_type bag_type, poll_obj *item, + poll_obj_type item_type) { + GPR_TIMER_BEGIN("add_poll_object", 0); + +#ifdef PO_DEBUG + GPR_ASSERT(item->obj_type == item_type); + GPR_ASSERT(bag->obj_type == bag_type); +#endif + + grpc_error *error = GRPC_ERROR_NONE; + polling_island *pi_new = NULL; + + gpr_mu_lock(&bag->mu); + gpr_mu_lock(&item->mu); + +retry: + /* + * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing + * 2) If item->pi and bag->pi are both NULL, create a new polling island (with + * a refcount of 2) and point item->pi and bag->pi to the new island + * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to + * the other's non-NULL pi + * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the + * polling islands and update item->pi and bag->pi to point to the new + * island + */ + + /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already + * orphaned */ + if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) { + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + return; + } + + if (item->pi == bag->pi) { + pi_new = item->pi; + if (pi_new == NULL) { + /* GPR_ASSERT(item->pi == bag->pi == NULL) */ + + /* If we are adding an fd to a bag (i.e pollset or pollset_set), then + * we need to do some extra work to make TSAN happy */ + if (item_type == POLL_OBJ_FD) { + /* Unlock before creating a new polling island: the polling island will + create a workqueue which creates a file descriptor, and holding an fd + lock here can eventually cause a loop to appear to TSAN (making it + unhappy). We don't think it's a real loop (there's an epoch point + where that loop possibility disappears), but the advantages of + keeping TSAN happy outweigh any performance advantage we might have + by keeping the lock held. */ + gpr_mu_unlock(&item->mu); + pi_new = polling_island_create(exec_ctx, FD_FROM_PO(item), &error); + gpr_mu_lock(&item->mu); + + /* Need to reverify any assumptions made between the initial lock and + getting to this branch: if they've changed, we need to throw away our + work and figure things out again. */ + if (item->pi != NULL) { + GRPC_POLLING_TRACE( + "add_poll_object: Raced creating new polling island. pi_new: %p " + "(fd: %d, %s: %p)", + (void *)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type), + (void *)bag); + /* No need to lock 'pi_new' here since this is a new polling island + and no one has a reference to it yet */ + polling_island_remove_all_fds_locked(pi_new, true, &error); + + /* Ref and unref so that the polling island gets deleted during unref + */ + PI_ADD_REF(pi_new, "dance_of_destruction"); + PI_UNREF(exec_ctx, pi_new, "dance_of_destruction"); + goto retry; + } + } else { + pi_new = polling_island_create(exec_ctx, NULL, &error); + } + + GRPC_POLLING_TRACE( + "add_poll_object: Created new polling island. pi_new: %p (%s: %p, " + "%s: %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + GRPC_POLLING_TRACE( + "add_poll_object: Same polling island. pi: %p (%s, %s)", + (void *)pi_new, poll_obj_string(item_type), + poll_obj_string(bag_type)); + } + } else if (item->pi == NULL) { + /* GPR_ASSERT(bag->pi != NULL) */ + /* Make pi_new point to latest pi*/ + pi_new = polling_island_lock(bag->pi); + + if (item_type == POLL_OBJ_FD) { + grpc_fd *fd = FD_FROM_PO(item); + polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); + } + + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else if (bag->pi == NULL) { + /* GPR_ASSERT(item->pi != NULL) */ + /* Make pi_new to point to latest pi */ + pi_new = polling_island_lock(item->pi); + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + pi_new = polling_island_merge(item->pi, bag->pi, &error); + GRPC_POLLING_TRACE( + "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } + + /* At this point, pi_new is the polling island that both item->pi and bag->pi + MUST be pointing to */ + + if (item->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(item_type)); + if (item->pi != NULL) { + PI_UNREF(exec_ctx, item->pi, poll_obj_string(item_type)); + } + item->pi = pi_new; + } + + if (bag->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(bag_type)); + if (bag->pi != NULL) { + PI_UNREF(exec_ctx, bag->pi, poll_obj_string(bag_type)); + } + bag->pi = pi_new; + } + + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + + GRPC_LOG_IF_ERROR("add_poll_object", error); + GPR_TIMER_END("add_poll_object", 0); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pollset->po, POLL_OBJ_POLLSET, &fd->po, + POLL_OBJ_FD); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + gpr_mu_init(&pss->po.mu); + pss->po.pi = NULL; +#ifdef PO_DEBUG + pss->po.obj_type = POLL_OBJ_POLLSET_SET; +#endif + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + gpr_mu_destroy(&pss->po.mu); + + if (pss->po.pi != NULL) { + PI_UNREF(exec_ctx, pss->po.pi, "pss_destroy"); + } + + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &fd->po, + POLL_OBJ_FD); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &ps->po, + POLL_OBJ_POLLSET); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + add_poll_object(exec_ctx, &bag->po, POLL_OBJ_POLLSET_SET, &item->po, + POLL_OBJ_POLLSET_SET); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/* Test helper functions + * */ +void *grpc_fd_get_polling_island(grpc_fd *fd) { + polling_island *pi; + + gpr_mu_lock(&fd->po.mu); + pi = fd->po.pi; + gpr_mu_unlock(&fd->po.mu); + + return pi; +} + +void *grpc_pollset_get_polling_island(grpc_pollset *ps) { + polling_island *pi; + + gpr_mu_lock(&ps->po.mu); + pi = ps->po.pi; + gpr_mu_unlock(&ps->po.mu); + + return pi; +} + +bool grpc_are_polling_islands_equal(void *p, void *q) { + polling_island *p1 = p; + polling_island *p2 = q; + + /* Note: polling_island_lock_pair() may change p1 and p2 to point to the + latest polling islands in their respective linked lists */ + polling_island_lock_pair(&p1, &p2); + polling_island_unlock_pair(p1, p2); + + return p1 == p2; +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { + /* If use of signals is disabled, we cannot use epoll engine*/ + if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { + return NULL; + } + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + if (!is_grpc_wakeup_signal_initialized) { + return NULL; + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ + +void grpc_use_signal(int signum) {} +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h new file mode 100644 index 0000000000..98a7d9ad92 --- /dev/null +++ b/src/core/lib/iomgr/ev_epollsig_linux.h @@ -0,0 +1,48 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epollsig_linux(void); + +#ifdef GRPC_LINUX_EPOLL +void *grpc_fd_get_polling_island(grpc_fd *fd); +void *grpc_pollset_get_polling_island(grpc_pollset *ps); +bool grpc_are_polling_islands_equal(void *p, void *q); +#endif /* defined(GRPC_LINUX_EPOLL) */ + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de8..0e1a93cec4 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,7 +44,8 @@ #include #include -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epoll1_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -65,7 +66,8 @@ typedef struct { } event_engine_factory; static const event_engine_factory g_factories[] = { - {"epoll", grpc_init_epoll_linux}, + {"epollsig", grpc_init_epollsig_linux}, + {"epoll1", grpc_init_epoll1_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 163d90f1a7..1c3b06dc97 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -97,7 +97,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll1_linux.c', + 'src/core/lib/iomgr/ev_epollsig_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f..34fc139229 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -939,8 +939,10 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ -src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll1_linux.c \ +src/core/lib/iomgr/ev_epoll1_linux.h \ +src/core/lib/iomgr/ev_epollsig_linux.c \ +src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a363adc268..f75edda7f9 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1074,8 +1074,10 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ -src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll1_linux.c \ +src/core/lib/iomgr/ev_epoll1_linux.h \ +src/core/lib/iomgr/ev_epollsig_linux.c \ +src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 386cddb583..52efa679f5 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7728,7 +7728,8 @@ "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", - "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll1_linux.h", + "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -7869,8 +7870,10 @@ "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", - "src/core/lib/iomgr/ev_epoll_linux.c", - "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll1_linux.c", + "src/core/lib/iomgr/ev_epoll1_linux.h", + "src/core/lib/iomgr/ev_epollsig_linux.c", + "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a58..2dd59e94d1 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -396,7 +396,8 @@ - + + @@ -610,7 +611,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc297..a77a403636 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -184,7 +184,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -908,7 +911,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651..da16786319 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -390,7 +390,8 @@ - + + @@ -594,7 +595,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab3..8f3c6e0395 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -169,7 +169,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -875,7 +878,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index d8b61b8c9d..285dfe4e94 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -320,7 +320,8 @@ - + + @@ -548,7 +549,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index e1435f4084..e2f94edbec 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -64,7 +64,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -866,7 +869,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97..87aaa3068f 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -215,7 +215,8 @@ - + + @@ -383,7 +384,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4..c0400fbbcb 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -121,7 +121,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -623,7 +626,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0bfda72e81..e0c7057be0 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -310,7 +310,8 @@ - + + @@ -516,7 +517,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 63c8d7f254..0b1bb8ba80 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -67,7 +67,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -779,7 +782,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr -- cgit v1.2.3 From 4509c476148ae2f3ad56561a6c65bd8b10f110a8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 19:05:13 +0000 Subject: Fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 467 +++++++++++++++++++++------------ src/core/lib/iomgr/ev_epollsig_linux.c | 2 +- 2 files changed, 299 insertions(+), 170 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 3c7a16b5ed..669a3df5c5 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -36,7 +36,7 @@ /* This polling engine is only relevant on linux kernels supporting epoll() */ #ifdef GRPC_LINUX_EPOLL -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epoll1_linux.h" #include #include @@ -75,16 +75,10 @@ static int g_epfd; struct grpc_fd { int fd; - /* The fd is either closed or we relinquished control of it. In either - cases, this indicates that the 'fd' on this structure is no longer - valid */ - bool orphaned; - gpr_atm read_closure; gpr_atm write_closure; struct grpc_fd *freelist_next; - grpc_closure *on_done_closure; /* The pollset that last noticed that the fd is readable. The actual type * stored in this is (grpc_pollset *) */ @@ -119,12 +113,12 @@ struct grpc_pollset_worker { }; struct grpc_pollset { - grpc_pollset_worker root_worker; - bool kicked_without_pollers; + grpc_pollset_worker *root_worker; + bool kicked_without_poller; bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ - grpc_closure *shutdown_done; /* Called after after shutdown is complete */ + grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ }; /******************************************************************************* @@ -171,66 +165,6 @@ static bool append_error(grpc_error **composite, grpc_error *error, static grpc_fd *fd_freelist = NULL; static gpr_mu fd_freelist_mu; -#ifdef GRPC_FD_REF_COUNT_DEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd *fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} - -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - /* Add the fd to the freelist */ - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); - - gpr_mu_unlock(&fd_freelist_mu); - } else { - GPR_ASSERT(old > n); - } -} - -/* Increment refcount by two to avoid changing the orphan bit */ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } -static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } -#endif - static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } static void fd_global_shutdown(void) { @@ -239,7 +173,6 @@ static void fd_global_shutdown(void) { while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->po.mu); gpr_free(fd); } gpr_mu_destroy(&fd_freelist_mu); @@ -257,29 +190,20 @@ static grpc_fd *fd_create(int fd, const char *name) { if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->po.mu); } - /* Note: It is not really needed to get the new_fd->po.mu lock here. If this - * is a newly created fd (or an fd we got from the freelist), no one else - * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->po.mu); - new_fd->po.pi = NULL; -#ifdef PO_DEBUG - new_fd->po.obj_type = POLL_OBJ_FD; -#endif + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), + .data.ptr = new_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { + gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); + } - gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; - new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = NULL; - new_fd->on_done_closure = NULL; - - gpr_mu_unlock(&new_fd->po.mu); char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); @@ -291,26 +215,12 @@ static grpc_fd *fd_create(int fd, const char *name) { return new_fd; } -static int fd_wrapped_fd(grpc_fd *fd) { - int ret_fd = -1; - gpr_mu_lock(&fd->po.mu); - if (!fd->orphaned) { - ret_fd = fd->fd; - } - gpr_mu_unlock(&fd->po.mu); - - return ret_fd; -} +static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; } static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, int *release_fd, const char *reason) { - bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - polling_island *unref_pi = NULL; - - gpr_mu_lock(&fd->po.mu); - fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file descriptor fd->fd (but we still own the grpc_fd structure). */ @@ -318,45 +228,18 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, *release_fd = fd->fd; } else { close(fd->fd); - is_fd_closed = true; } - fd->orphaned = true; - - /* Remove the active status but keep referenced. We want this grpc_fd struct - to be alive (and not added to freelist) until the end of this function */ - REF_BY(fd, 1, reason); - - /* Remove the fd from the polling island: - - Get a lock on the latest polling island (i.e the last island in the - linked list pointed by fd->po.pi). This is the island that - would actually contain the fd - - Remove the fd from the latest polling island - - Unlock the latest polling island - - Set fd->po.pi to NULL (but remove the ref on the polling island - before doing this.) */ - if (fd->po.pi != NULL) { - polling_island *pi_latest = polling_island_lock(fd->po.pi); - polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); - gpr_mu_unlock(&pi_latest->mu); - - unref_pi = fd->po.pi; - fd->po.pi = NULL; - } + grpc_closure_sched(exec_ctx, on_done, GRPC_ERROR_REF(error)); - grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + grpc_iomgr_unregister_object(&fd->iomgr_object); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); - gpr_mu_unlock(&fd->po.mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ - if (unref_pi != NULL) { - /* Unref stale polling island here, outside the fd lock above. - The polling island owns a workqueue which owns an fd, and unreffing - inside the lock can cause an eventual lock loop that makes TSAN very - unhappy. */ - PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); - } - GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); - GRPC_ERROR_UNREF(error); + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + gpr_mu_unlock(&fd_freelist_mu); } static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, @@ -390,11 +273,24 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - gpr_mu_lock(&fd->po.mu); - grpc_workqueue *workqueue = - GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); - gpr_mu_unlock(&fd->po.mu); - return workqueue; + return NULL; /* TODO(ctiller): add a global workqueue */ +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } /******************************************************************************* @@ -442,6 +338,263 @@ static worker_remove_result worker_remove(grpc_pollset_worker **root, GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); +static gpr_mu g_pollset_mu; +static grpc_pollset_worker *g_root_worker; + +static grpc_error *pollset_global_init(void) { + gpr_mu_init(&g_pollset_mu); + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { + return GRPC_OS_ERROR(errno, "epoll_ctl"); + } + return GRPC_ERROR_NONE; +} + +static void pollset_global_shutdown(void) { + gpr_mu_destroy(&g_pollset_mu); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + *mu = &g_pollset_mu; +} + +static grpc_error *pollset_kick_all(grpc_pollset *pollset) { + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->root_worker != NULL) { + grpc_pollset_worker *worker = pollset->root_worker; + do { + if (worker->initialized_cv) { + worker->kicked = true; + gpr_cv_signal(&worker->cv); + } else { + append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), + "pollset_shutdown"); + } + + worker = worker->links[PWL_POLLSET].next; + } while (worker != pollset->root_worker); + } + return error; +} + +static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + pollset->shutdown_closure = NULL; + } +} + +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_ASSERT(pollset->shutdown_closure == NULL); + pollset->shutdown_closure = closure; + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + pollset_maybe_finish_shutdown(exec_ctx, pollset); +} + +static void pollset_destroy(grpc_pollset *pollset) {} + +#define MAX_EPOLL_EVENTS 100 + +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, now) <= 0) { + return 0; + } + + static const gpr_timespec round_up = { + .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); + return millis >= 1 ? millis : 1; +} + +static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { + struct epoll_event events[MAX_EPOLL_EVENTS]; + static const char *err_desc = "pollset_poll"; + + int timeout = poll_deadline_to_millis_timeout(deadline, now); + + if (timeout != 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; + } + int r; + do { + r = epoll_wait(g_epfd, events, MAX_EPOLL_EVENTS, timeout); + } while (r < 0 && errno == EINTR); + if (timeout != 0) { + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + + grpc_error *error = GRPC_ERROR_NONE; + for (int i = 0; i < r; i++) { + void *data_ptr = events[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else { + grpc_fd *fd = (grpc_fd *)(data_ptr); + bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; + bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; + bool write_ev = (events[i].events & EPOLLOUT) != 0; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + + return error; +} + +static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl, gpr_timespec *now, + gpr_timespec deadline) { + bool do_poll = true; + if (worker_hdl != NULL) *worker_hdl = worker; + worker->initialized_cv = false; + worker->kicked = false; + + worker_insert(&pollset->root_worker, PWL_POLLSET, worker); + if (!worker_insert(&g_root_worker, PWL_POLLABLE, worker)) { + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (do_poll && g_root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &g_pollset_mu, deadline)) { + do_poll = false; + } else if (worker->kicked) { + do_poll = false; + } + } + *now = gpr_now(now->clock_type); + } + + return do_poll && pollset->shutdown_closure == NULL; +} + +static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl) { + if (NEW_ROOT == worker_remove(&g_root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&g_root_worker->cv); + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_work"; + if (pollset->kicked_without_poller) { + pollset->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } + if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + GPR_ASSERT(!pollset->shutdown_closure); + gpr_mu_unlock(&g_pollset_mu); + append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), + err_desc); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&g_pollset_mu); + gpr_tls_set(&g_current_thread_pollset, 0); + gpr_tls_set(&g_current_thread_worker, 0); + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } + end_worker(exec_ctx, pollset, &worker, worker_hdl); + return error; +} + +static grpc_error *pollset_kick(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { + if (specific_worker == NULL) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { + if (pollset->root_worker == NULL) { + pollset->kicked_without_poller = true; + return GRPC_ERROR_NONE; + } else { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } + } else { + return GRPC_ERROR_NONE; + } + } else if (specific_worker->kicked) { + return GRPC_ERROR_NONE; + } else if (gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + specific_worker->kicked = true; + return GRPC_ERROR_NONE; + } else if (specific_worker == g_root_worker) { + specific_worker->kicked = true; + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } else { + specific_worker->kicked = true; + gpr_cv_signal(&specific_worker->cv); + return GRPC_ERROR_NONE; + } +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) {} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +/******************************************************************************* + * Workqueue Definitions + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) {} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) {} +#endif + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + return grpc_schedule_on_exec_ctx; +} /******************************************************************************* * Pollset-set Definitions @@ -481,7 +634,6 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { fd_global_shutdown(); pollset_global_shutdown(); - polling_island_global_shutdown(); } static const grpc_event_engine_vtable vtable = { @@ -524,45 +676,22 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epoll_available() { - int fd = epoll_create1(EPOLL_CLOEXEC); - if (fd < 0) { - gpr_log( - GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epoll polling engine", - fd); - return false; - } - close(fd); - return true; -} - const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { - /* If use of signals is disabled, we cannot use epoll engine*/ - if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { - return NULL; - } - if (!grpc_has_wakeup_fd()) { return NULL; } - if (!is_epoll_available()) { + g_epfd = epoll_create1(EPOLL_CLOEXEC); + if (g_epfd < 0) { + gpr_log(GPR_ERROR, "epoll unavailable"); return NULL; } - if (!is_grpc_wakeup_signal_initialized) { - grpc_use_signal(SIGRTMIN + 6); - } - fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { - return NULL; - } - - if (!GRPC_LOG_IF_ERROR("polling_island_global_init", - polling_island_global_init())) { + close(g_epfd); + fd_global_shutdown(); return NULL; } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index ee2af081f0..4ce380c7d0 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -36,7 +36,7 @@ /* This polling engine is only relevant on linux kernels supporting epoll() */ #ifdef GRPC_LINUX_EPOLL -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include #include -- cgit v1.2.3 From b72a74ae039d666cf6cfc60477289c03dc5b5933 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 12:07:05 -0700 Subject: Fix wakeup bug --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b7c3e2e959..952f0172cc 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -696,7 +696,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, p->root_worker); } if (specific_worker == NULL) { - if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { if (pollset->root_worker == NULL) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); -- cgit v1.2.3 From d4838a98c5fb8b1227420f7577615ef7f66ac43e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 12:08:18 -0700 Subject: Tweaks --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- src/core/lib/iomgr/ev_epoll1_linux.h | 8 +------- tools/run_tests/run_tests.py | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 3c7a16b5ed..67edae68a6 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h index 5d3651019b..99f4827008 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.h +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,10 +41,4 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(void); -#ifdef GRPC_LINUX_EPOLL -void *grpc_fd_get_polling_island(grpc_fd *fd); -void *grpc_pollset_get_polling_island(grpc_pollset *ps); -bool grpc_are_polling_islands_equal(void *p, void *q); -#endif /* defined(GRPC_LINUX_EPOLL) */ - #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4da2ba4c3b..c1da93f422 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -72,7 +72,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { _POLLING_STRATEGIES = { - 'linux': ['epoll', 'poll', 'poll-cv'] + 'linux': ['epoll1', 'epollsig', 'poll', 'poll-cv'] } -- cgit v1.2.3 From 6f0af49bb2f784a66d5fcc9cf9ba46754451f7e9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 19:26:16 +0000 Subject: Allow specifying a default signal iff epollsig has been explicitly requested --- src/core/lib/iomgr/ev_epoll1_linux.c | 4 ++-- src/core/lib/iomgr/ev_epoll1_linux.h | 2 +- src/core/lib/iomgr/ev_epollsig_linux.c | 7 ++++--- src/core/lib/iomgr/ev_epollsig_linux.h | 2 +- src/core/lib/iomgr/ev_poll_posix.c | 4 ++-- src/core/lib/iomgr/ev_poll_posix.h | 4 ++-- src/core/lib/iomgr/ev_posix.c | 4 ++-- 7 files changed, 14 insertions(+), 13 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index ff4c51f5d8..9909d8b126 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -676,7 +676,7 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ -const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -703,6 +703,6 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h index 99f4827008..bd52478a7c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.h +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -39,6 +39,6 @@ // a polling engine that utilizes a singleton epoll set and turnstile polling -const grpc_event_engine_vtable *grpc_init_epoll1_linux(void); +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 4ce380c7d0..25986598b7 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1921,7 +1921,7 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { return NULL; @@ -1936,7 +1936,8 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { } if (!is_grpc_wakeup_signal_initialized) { - return NULL; + if (explicit_request) grpc_use_signal(SIGRTMIN + 6); + else return NULL; } fd_global_init(); @@ -1958,7 +1959,7 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ void grpc_use_signal(int signum) {} diff --git a/src/core/lib/iomgr/ev_epollsig_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h index 98a7d9ad92..9e4034f2a7 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.h +++ b/src/core/lib/iomgr/ev_epollsig_linux.h @@ -37,7 +37,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epollsig_linux(void); +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request); #ifdef GRPC_LINUX_EPOLL void *grpc_fd_get_polling_island(grpc_fd *fd); diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 9834cdd197..69f322f7df 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -1569,7 +1569,7 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -const grpc_event_engine_vtable *grpc_init_poll_posix(void) { +const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1579,7 +1579,7 @@ const grpc_event_engine_vtable *grpc_init_poll_posix(void) { return &vtable; } -const grpc_event_engine_vtable *grpc_init_poll_cv_posix(void) { +const grpc_event_engine_vtable *grpc_init_poll_cv_posix(bool explicit_request) { global_cv_fd_table_init(); grpc_enable_cv_wakeup_fds(1); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { diff --git a/src/core/lib/iomgr/ev_poll_posix.h b/src/core/lib/iomgr/ev_poll_posix.h index 202ffca14c..2890e93ead 100644 --- a/src/core/lib/iomgr/ev_poll_posix.h +++ b/src/core/lib/iomgr/ev_poll_posix.h @@ -36,7 +36,7 @@ #include "src/core/lib/iomgr/ev_posix.h" -const grpc_event_engine_vtable *grpc_init_poll_posix(void); -const grpc_event_engine_vtable *grpc_init_poll_cv_posix(void); +const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request); +const grpc_event_engine_vtable *grpc_init_poll_cv_posix(bool explicit_request); #endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 0e1a93cec4..442faeac3e 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -58,7 +58,7 @@ grpc_wakeup_fd grpc_global_wakeup_fd; static const grpc_event_engine_vtable *g_event_engine; static const char *g_poll_strategy_name = NULL; -typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(void); +typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(bool explicit_request); typedef struct { const char *name; @@ -104,7 +104,7 @@ static bool is(const char *want, const char *have) { static void try_engine(const char *engine) { for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { if (is(engine, g_factories[i].name)) { - if ((g_event_engine = g_factories[i].factory())) { + if ((g_event_engine = g_factories[i].factory(0 == strcmp(engine, g_factories[i].name)))) { g_poll_strategy_name = g_factories[i].name; gpr_log(GPR_DEBUG, "Using polling engine: %s", g_factories[i].name); return; -- cgit v1.2.3 From 9ddb3151d6ae32027f103322d8ff984ea3eecefa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 21:32:56 +0000 Subject: Bug fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 41 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 9909d8b126..d8d3cb6376 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -192,12 +192,6 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = gpr_malloc(sizeof(grpc_fd)); } - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), - .data.ptr = new_fd}; - if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { - gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); - } - new_fd->fd = fd; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); @@ -212,16 +206,37 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); #endif gpr_free(fd_name); + + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), + .data.ptr = new_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { + gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); + } + return new_fd; } static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; } +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, int *release_fd, const char *reason) { grpc_error *error = GRPC_ERROR_NONE; + if (!grpc_lfev_is_shutdown(&fd->read_closure)) { + fd_shutdown(exec_ctx, fd, GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason)); + } + /* If release_fd is not NULL, we should be relinquishing control of the file descriptor fd->fd (but we still own the grpc_fd structure). */ if (release_fd != NULL) { @@ -252,16 +267,6 @@ static bool fd_is_shutdown(grpc_fd *fd) { return grpc_lfev_is_shutdown(&fd->read_closure); } -/* Might be called multiple times */ -static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { - if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, - GRPC_ERROR_REF(why))) { - shutdown(fd->fd, SHUT_RDWR); - grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); - } - GRPC_ERROR_UNREF(why); -} - static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); @@ -703,6 +708,8 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ -- cgit v1.2.3 From f8382b80caf5f27a69f12e8620cc27255f8afdb8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 15:09:48 -0700 Subject: clang-format --- src/core/lib/iomgr/ev_epollsig_linux.c | 15 +++++++++++---- src/core/lib/iomgr/ev_posix.c | 6 ++++-- 2 files changed, 15 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 25986598b7..65259912a9 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1921,7 +1921,8 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { +const grpc_event_engine_vtable *grpc_init_epollsig_linux( + bool explicit_request) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { return NULL; @@ -1936,8 +1937,11 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) } if (!is_grpc_wakeup_signal_initialized) { - if (explicit_request) grpc_use_signal(SIGRTMIN + 6); - else return NULL; + if (explicit_request) { + grpc_use_signal(SIGRTMIN + 6); + } else { + return NULL; + } } fd_global_init(); @@ -1959,7 +1963,10 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollsig_linux( + bool explicit_request) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ void grpc_use_signal(int signum) {} diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 442faeac3e..a436d649dd 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -58,7 +58,8 @@ grpc_wakeup_fd grpc_global_wakeup_fd; static const grpc_event_engine_vtable *g_event_engine; static const char *g_poll_strategy_name = NULL; -typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(bool explicit_request); +typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)( + bool explicit_request); typedef struct { const char *name; @@ -104,7 +105,8 @@ static bool is(const char *want, const char *have) { static void try_engine(const char *engine) { for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { if (is(engine, g_factories[i].name)) { - if ((g_event_engine = g_factories[i].factory(0 == strcmp(engine, g_factories[i].name)))) { + if ((g_event_engine = g_factories[i].factory( + 0 == strcmp(engine, g_factories[i].name)))) { g_poll_strategy_name = g_factories[i].name; gpr_log(GPR_DEBUG, "Using polling engine: %s", g_factories[i].name); return; -- cgit v1.2.3 From 375eb258d8d66dc3ff1e8a9e82760b8ce6e51954 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 23:29:12 +0000 Subject: Fix kicks --- src/core/lib/iomgr/ev_epoll1_linux.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index d8d3cb6376..195c3bd144 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -67,6 +67,7 @@ * needed) */ static grpc_wakeup_fd global_wakeup_fd; static int g_epfd; +static bool g_timer_kick = false; /******************************************************************************* * Fd Declarations @@ -350,6 +351,9 @@ static grpc_error *pollset_global_init(void) { gpr_mu_init(&g_pollset_mu); gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); + global_wakeup_fd.read_fd = -1; + grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); + if (err != GRPC_ERROR_NONE) return err; struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &global_wakeup_fd}; if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { @@ -362,6 +366,7 @@ static void pollset_global_shutdown(void) { gpr_mu_destroy(&g_pollset_mu); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); + if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -449,7 +454,10 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); + if (g_timer_kick) { + g_timer_kick = false; + grpc_timer_consume_kick(); + } append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else { @@ -543,11 +551,19 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { - if (pollset->root_worker == NULL) { + grpc_pollset_worker *root_worker = pollset->root_worker; + if (root_worker == NULL) { pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; - } else { + } + grpc_pollset_worker *next_worker = root_worker->links[PWL_POLLSET].next; + if (root_worker == next_worker && root_worker == g_root_worker) { + root_worker->kicked = true; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } else { + next_worker->kicked = true; + gpr_cv_signal(&next_worker->cv); + return GRPC_ERROR_NONE; } } else { return GRPC_ERROR_NONE; @@ -572,6 +588,9 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) {} static grpc_error *kick_poller(void) { + gpr_mu_lock(&g_pollset_mu); + g_timer_kick = true; + gpr_mu_unlock(&g_pollset_mu); return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } -- cgit v1.2.3 From 2f128f9a0b304ad740ddf26c7b25494d5a9cb6c6 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Thu, 27 Apr 2017 16:40:43 -0700 Subject: add blocking region annotations around sigtimedwait --- src/core/lib/iomgr/ev_epoll_linux.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 1234828063..64ea7b192a 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1489,7 +1489,9 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, ret = sigwaitinfo(&g_wakeup_sig_set, NULL); } else { struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + GRPC_SCHEDULING_START_BLOCKING_REGION; ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; } if (ret == -1) { -- cgit v1.2.3 From 6de0593e825dc616924847f8a1583b3badaaa3fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 09:17:38 -0700 Subject: Start sketching hierarchical turnstile --- src/core/lib/iomgr/ev_epoll1_linux.c | 53 ++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 11 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 195c3bd144..fdd6384c86 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -113,18 +114,32 @@ struct grpc_pollset_worker { gpr_cv cv; }; +typedef struct pollset_neighbourhood { + gpr_mu mu; + grpc_pollset *active_root; + grpc_pollset *inactive_root; + bool seen_inactive; + char pad[GPR_CACHELINE_SIZE]; +} pollset_neighbourhood; + struct grpc_pollset { + gpr_mu mu; + pollset_neighbourhood *neighbourhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; - + bool seen_inactive; bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ + + grpc_pollset *next; + grpc_pollset *prev; }; /******************************************************************************* * Pollset-set Declarations */ + struct grpc_pollset_set {}; /******************************************************************************* @@ -303,6 +318,11 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { * Pollset Definitions */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static gpr_atm g_active_poller; +static pollset_neighbourhood *g_neighbourhoods; + /* Return true if first in list */ static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, grpc_pollset_worker *worker) { @@ -342,15 +362,10 @@ static worker_remove_result worker_remove(grpc_pollset_worker **root, } } -GPR_TLS_DECL(g_current_thread_pollset); -GPR_TLS_DECL(g_current_thread_worker); -static gpr_mu g_pollset_mu; -static grpc_pollset_worker *g_root_worker; - static grpc_error *pollset_global_init(void) { - gpr_mu_init(&g_pollset_mu); gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); + gpr_atm_no_barrier_store(&g_active_poller, 0); global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); if (err != GRPC_ERROR_NONE) return err; @@ -363,14 +378,32 @@ static grpc_error *pollset_global_init(void) { } static void pollset_global_shutdown(void) { - gpr_mu_destroy(&g_pollset_mu); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - *mu = &g_pollset_mu; + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset->seen_inactive = true; + pollset->next = pollset->prev = pollset; +} + +static void pollset_destroy(grpc_pollset *pollset) { + gpr_mu_destroy(&pollset->mu); + gpr_mu_lock(&pollset->neighbourhood->mu); + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } else if (pollset == pollset->neighbourhood->inactive_root) { + pollset->neighbourhood->inactive_root = + pollset->next == pollset ? NULL : pollset->next; + } + gpr_mu_unlock(&pollset->neighbourhood->mu); } static grpc_error *pollset_kick_all(grpc_pollset *pollset) { @@ -408,8 +441,6 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static void pollset_destroy(grpc_pollset *pollset) {} - #define MAX_EPOLL_EVENTS 100 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, -- cgit v1.2.3 From 32f90eec45cd7e19e7742e8868110d945e0eab73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 12:46:41 -0700 Subject: bug fixes, implementation --- src/core/lib/iomgr/ev_epoll1_linux.c | 245 +++++++++++++++++++++++++---------- 1 file changed, 179 insertions(+), 66 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index fdd6384c86..3355c8344e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -68,7 +68,7 @@ * needed) */ static grpc_wakeup_fd global_wakeup_fd; static int g_epfd; -static bool g_timer_kick = false; +static gpr_atm g_timer_kick; /******************************************************************************* * Fd Declarations @@ -96,21 +96,13 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef struct pollset_worker_link { - grpc_pollset_worker *next; - grpc_pollset_worker *prev; -} pollset_worker_link; - -typedef enum { - PWL_POLLSET, - PWL_POLLABLE, - POLLSET_WORKER_LINK_COUNT -} pollset_worker_links; +typedef enum { UNKICKED, KICKED, KICKED_FOR_POLL } kick_state; struct grpc_pollset_worker { - bool kicked; + kick_state kick_state; bool initialized_cv; - pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; + grpc_pollset_worker *next; + grpc_pollset_worker *prev; gpr_cv cv; }; @@ -322,19 +314,19 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; +static size_t g_num_neighbourhoods; /* Return true if first in list */ -static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, - grpc_pollset_worker *worker) { - if (*root == NULL) { - *root = worker; - worker->links[link].next = worker->links[link].prev = worker; +static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { + if (pollset->root_worker == NULL) { + pollset->root_worker = worker; + worker->next = worker->prev = worker; return true; } else { - worker->links[link].next = *root; - worker->links[link].prev = worker->links[link].next->links[link].prev; - worker->links[link].next->links[link].prev = worker; - worker->links[link].prev->links[link].next = worker; + worker->next = pollset->root_worker; + worker->prev = worker->next->prev; + worker->next->prev = worker; + worker->prev->next = worker; return false; } } @@ -342,22 +334,21 @@ static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, /* Return true if last in list */ typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; -static worker_remove_result worker_remove(grpc_pollset_worker **root, - pollset_worker_links link, +static worker_remove_result worker_remove(grpc_pollset *pollset, grpc_pollset_worker *worker) { - if (worker == *root) { - if (worker == worker->links[link].next) { - *root = NULL; + if (worker == pollset->root_worker) { + if (worker == worker->next) { + pollset->root_worker = NULL; return EMPTIED; } else { - *root = worker->links[link].next; - worker->links[link].prev->links[link].next = worker->links[link].next; - worker->links[link].next->links[link].prev = worker->links[link].prev; + pollset->root_worker = worker->next; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; return NEW_ROOT; } } else { - worker->links[link].prev->links[link].next = worker->links[link].next; - worker->links[link].next->links[link].prev = worker->links[link].prev; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; return REMOVED; } } @@ -374,6 +365,13 @@ static grpc_error *pollset_global_init(void) { if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } + g_num_neighbourhoods = GPR_MAX(1, gpr_cpu_num_cores()); + g_neighbourhoods = + gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); + for (size_t i = 0; i < g_num_neighbourhoods; i++) { + gpr_mu_init(&g_neighbourhoods[i].mu); + g_neighbourhoods[i].seen_inactive = true; + } return GRPC_ERROR_NONE; } @@ -381,6 +379,10 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); + for (size_t i = 0; i < g_num_neighbourhoods; i++) { + gpr_mu_destroy(&g_neighbourhoods[i].mu); + } + gpr_free(g_neighbourhoods); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -392,7 +394,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { } static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->mu); gpr_mu_lock(&pollset->neighbourhood->mu); pollset->prev->next = pollset->next; pollset->next->prev = pollset->prev; @@ -404,6 +405,7 @@ static void pollset_destroy(grpc_pollset *pollset) { pollset->next == pollset ? NULL : pollset->next; } gpr_mu_unlock(&pollset->neighbourhood->mu); + gpr_mu_destroy(&pollset->mu); } static grpc_error *pollset_kick_all(grpc_pollset *pollset) { @@ -412,14 +414,15 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { grpc_pollset_worker *worker = pollset->root_worker; do { if (worker->initialized_cv) { - worker->kicked = true; + worker->kick_state = KICKED; gpr_cv_signal(&worker->cv); } else { + worker->kick_state = KICKED; append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), "pollset_shutdown"); } - worker = worker->links[PWL_POLLSET].next; + worker = worker->next; } while (worker != pollset->root_worker); } return error; @@ -485,8 +488,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - if (g_timer_kick) { - g_timer_kick = false; + if (gpr_atm_no_barrier_cas(&g_timer_kick, 1, 0)) { grpc_timer_consume_kick(); } append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), @@ -508,41 +510,151 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } +#if 0 +static void verify_all_entries_in_neighbourhood_list( + grpc_pollset *root, bool should_be_seen_inactive) { + if (root == NULL) return; + grpc_pollset *p = root; + do { + GPR_ASSERT(p->seen_inactive == should_be_seen_inactive); + p = p->next; + } while (p != root); +} + +static void verify_neighbourhood_lists(pollset_neighbourhood *neighbourhood) { + // assumes neighbourhood->mu locked + verify_all_entries_in_neighbourhood_list(neighbourhood->active_root, false); + verify_all_entries_in_neighbourhood_list(neighbourhood->inactive_root, true); +} +#endif + +static void move_pollset_to_neighbourhood_list(grpc_pollset *pollset, + grpc_pollset **from_root, + grpc_pollset **to_root) { + // remove from old list + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (*from_root == pollset) { + *from_root = pollset->next == pollset ? NULL : pollset->next; + } + // add to new list + if (*to_root == NULL) { + *to_root = pollset->next = pollset->prev = pollset; + } else { + pollset->next = *to_root; + pollset->prev = pollset->next->prev; + pollset->next->prev = pollset->prev->next = pollset; + } +} + static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { - bool do_poll = true; if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; - worker->kicked = false; - - worker_insert(&pollset->root_worker, PWL_POLLSET, worker); - if (!worker_insert(&g_root_worker, PWL_POLLABLE, worker)) { + worker->kick_state = UNKICKED; + + if (pollset->seen_inactive) { + // pollset has been observed to be inactive, we need to move back to the + // active list + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + gpr_mu_lock(&neighbourhood->mu); + gpr_mu_lock(&pollset->mu); + if (pollset->seen_inactive) { + pollset->seen_inactive = false; + move_pollset_to_neighbourhood_list(pollset, &neighbourhood->inactive_root, + &neighbourhood->active_root); + if (neighbourhood->seen_inactive) { + neighbourhood->seen_inactive = false; + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { + worker->kick_state = KICKED_FOR_POLL; + } + } + } + gpr_mu_unlock(&neighbourhood->mu); + } + worker_insert(pollset, worker); + if (worker->kick_state == UNKICKED) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (do_poll && g_root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &g_pollset_mu, deadline)) { - do_poll = false; - } else if (worker->kicked) { - do_poll = false; + do { + if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && + worker->kick_state == UNKICKED) { + worker->kick_state = KICKED; } - } + } while (worker->kick_state == UNKICKED); *now = gpr_now(now->clock_type); } - return do_poll && pollset->shutdown_closure == NULL; + return worker->kick_state == KICKED_FOR_POLL && + pollset->shutdown_closure == NULL; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (NEW_ROOT == worker_remove(&g_root_worker, PWL_POLLABLE, worker)) { - gpr_cv_signal(&g_root_worker->cv); + if (worker->kick_state == KICKED_FOR_POLL) { + GPR_ASSERT(!pollset->seen_inactive); + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker); + if (worker->next != worker) { + assert(worker->next->initialized_cv); + gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); + worker->next->kick_state = KICKED_FOR_POLL; + gpr_cv_signal(&worker->next->cv); + } else { + gpr_atm_no_barrier_store(&g_active_poller, 0); + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + bool found_worker = false; + do { + gpr_mu_lock(&neighbourhood->mu); + do { + grpc_pollset *inspect = neighbourhood->active_root; + if (inspect == NULL) { + break; + } + gpr_mu_lock(&inspect->mu); + GPR_ASSERT(!inspect->seen_inactive); + grpc_pollset_worker *inspect_worker = inspect->root_worker; + if (inspect_worker == worker) inspect_worker = worker->next; + if (inspect_worker == worker) inspect_worker = NULL; + if (inspect_worker != NULL) { + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + GPR_ASSERT(inspect_worker->initialized_cv); + inspect_worker->kick_state = KICKED_FOR_POLL; + gpr_cv_signal(&inspect_worker->cv); + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + } else { + inspect->seen_inactive = true; + move_pollset_to_neighbourhood_list(inspect, + &neighbourhood->active_root, + &neighbourhood->inactive_root); + } + gpr_mu_unlock(&inspect->mu); + } while (!found_worker); + if (!found_worker) { + neighbourhood->seen_inactive = true; + } + gpr_mu_unlock(&neighbourhood->mu); + ssize_t cur_neighbourhood_idx = neighbourhood - g_neighbourhoods; + GPR_ASSERT(cur_neighbourhood_idx >= 0); + GPR_ASSERT(g_num_neighbourhoods < INTPTR_MAX); + GPR_ASSERT(cur_neighbourhood_idx < (ssize_t)g_neighbourhoods); + size_t new_neighbourhood_idx = + ((size_t)cur_neighbourhood_idx + 1) % g_num_neighbourhoods; + neighbourhood = &g_neighbourhoods[new_neighbourhood_idx]; + } while (!found_worker && neighbourhood != pollset->neighbourhood); + gpr_mu_lock(&pollset->mu); + } } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); } - if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + if (EMPTIED == worker_remove(pollset, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); } } @@ -565,11 +677,11 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - gpr_mu_unlock(&g_pollset_mu); + gpr_mu_unlock(&pollset->mu); append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), err_desc); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&g_pollset_mu); + gpr_mu_lock(&pollset->mu); gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); @@ -587,29 +699,32 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; } - grpc_pollset_worker *next_worker = root_worker->links[PWL_POLLSET].next; - if (root_worker == next_worker && root_worker == g_root_worker) { - root_worker->kicked = true; + grpc_pollset_worker *next_worker = root_worker->next; + if (root_worker == next_worker && + root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( + &g_active_poller)) { + root_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } else { - next_worker->kicked = true; + next_worker->kick_state = KICKED; gpr_cv_signal(&next_worker->cv); return GRPC_ERROR_NONE; } } else { return GRPC_ERROR_NONE; } - } else if (specific_worker->kicked) { + } else if (specific_worker->kick_state != KICKED) { return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { - specific_worker->kicked = true; + specific_worker->kick_state = KICKED; return GRPC_ERROR_NONE; - } else if (specific_worker == g_root_worker) { - specific_worker->kicked = true; + } else if (specific_worker == + (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + specific_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } else { - specific_worker->kicked = true; + specific_worker->kick_state = KICKED; gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } @@ -619,9 +734,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) {} static grpc_error *kick_poller(void) { - gpr_mu_lock(&g_pollset_mu); - g_timer_kick = true; - gpr_mu_unlock(&g_pollset_mu); + gpr_atm_no_barrier_store(&g_timer_kick, 1); return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } -- cgit v1.2.3 From c9b09e98fd722cd2e798add0acca0a00ef9e7394 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 13:09:10 -0700 Subject: Fix wakeup condition --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 3355c8344e..27ed978106 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -713,7 +713,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } else { return GRPC_ERROR_NONE; } - } else if (specific_worker->kick_state != KICKED) { + } else if (specific_worker->kick_state != UNKICKED) { return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { -- cgit v1.2.3 From 8502ecbd38457ab1bc6de6f4f6c33e6e42b46239 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 14:22:01 -0700 Subject: Tweak: trigger next poller before exec_ctx flush --- src/core/lib/iomgr/ev_epoll1_linux.c | 26 +++++++++++++++++++------- src/core/lib/iomgr/exec_ctx.c | 5 +++++ src/core/lib/iomgr/exec_ctx.h | 2 ++ 3 files changed, 26 insertions(+), 7 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 27ed978106..b1127d38cb 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -594,14 +594,21 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (worker->kick_state == KICKED_FOR_POLL) { + if (worker_hdl != NULL) *worker_hdl = NULL; + if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { GPR_ASSERT(!pollset->seen_inactive); GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker); if (worker->next != worker) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); + gpr_log(GPR_DEBUG, "Picked sibling worker %p for poller", worker); worker->next->kick_state = KICKED_FOR_POLL; gpr_cv_signal(&worker->next->cv); + if (grpc_exec_ctx_has_work(exec_ctx)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); pollset_neighbourhood *neighbourhood = pollset->neighbourhood; @@ -648,6 +655,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, ((size_t)cur_neighbourhood_idx + 1) % g_num_neighbourhoods; neighbourhood = &g_neighbourhoods[new_neighbourhood_idx]; } while (!found_worker && neighbourhood != pollset->neighbourhood); + grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } } @@ -673,20 +681,18 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { - gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); gpr_mu_unlock(&pollset->mu); append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), err_desc); - grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); - gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); - pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(exec_ctx, pollset, &worker, worker_hdl); + gpr_tls_set(&g_current_thread_pollset, 0); return error; } @@ -705,10 +711,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, &g_active_poller)) { root_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); - } else { + } else if (next_worker->kick_state == UNKICKED) { + GPR_ASSERT(next_worker->initialized_cv); next_worker->kick_state = KICKED; gpr_cv_signal(&next_worker->cv); return GRPC_ERROR_NONE; + } else { + return GRPC_ERROR_NONE; } } else { return GRPC_ERROR_NONE; @@ -723,10 +732,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { specific_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); - } else { + } else if (specific_worker->initialized_cv) { specific_worker->kick_state = KICKED; gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; + } else { + specific_worker->kick_state = KICKED; + return GRPC_ERROR_NONE; } } diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c index 2532a708e7..318bb2b713 100644 --- a/src/core/lib/iomgr/exec_ctx.c +++ b/src/core/lib/iomgr/exec_ctx.c @@ -62,6 +62,11 @@ bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) { return true; } +bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) { + return exec_ctx->active_combiner != NULL || + !grpc_closure_list_empty(exec_ctx->closure_list); +} + bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { bool did_something = 0; GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h index f99a0fee5f..759a3ae2d5 100644 --- a/src/core/lib/iomgr/exec_ctx.h +++ b/src/core/lib/iomgr/exec_ctx.h @@ -93,6 +93,8 @@ struct grpc_exec_ctx { extern grpc_closure_scheduler *grpc_schedule_on_exec_ctx; +bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx); + /** Flush any work that has been enqueued onto this grpc_exec_ctx. * Caller must guarantee that no interfering locks are held. * Returns true if work was performed, false otherwise. */ -- cgit v1.2.3 From f3e40227c39076c7ded7b2217d218f5829830723 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 21:26:35 +0000 Subject: Fix bad assumption --- src/core/lib/iomgr/ev_epoll1_linux.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index b1127d38cb..a25168fdb4 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -601,7 +601,6 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (worker->next != worker) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); - gpr_log(GPR_DEBUG, "Picked sibling worker %p for poller", worker); worker->next->kick_state = KICKED_FOR_POLL; gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { @@ -629,9 +628,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (inspect_worker != NULL) { if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { - GPR_ASSERT(inspect_worker->initialized_cv); inspect_worker->kick_state = KICKED_FOR_POLL; - gpr_cv_signal(&inspect_worker->cv); + if (inspect_worker->initialized_cv) gpr_cv_signal(&inspect_worker->cv); } // even if we didn't win the cas, there's a worker, we can stop found_worker = true; -- cgit v1.2.3 From bbf4c7a00a3b81a600821d5a6be94f2021703e04 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 15:12:10 -0700 Subject: Try to avoid contention --- src/core/lib/iomgr/ev_epoll1_linux.c | 106 ++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 40 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index a25168fdb4..27f448f08e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -314,6 +314,7 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; +static bool *g_neighbour_scan_state; static size_t g_num_neighbourhoods; /* Return true if first in list */ @@ -368,6 +369,8 @@ static grpc_error *pollset_global_init(void) { g_num_neighbourhoods = GPR_MAX(1, gpr_cpu_num_cores()); g_neighbourhoods = gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); + g_neighbour_scan_state = + gpr_malloc(sizeof(*g_neighbour_scan_state) * g_num_neighbourhoods); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_init(&g_neighbourhoods[i].mu); g_neighbourhoods[i].seen_inactive = true; @@ -383,6 +386,7 @@ static void pollset_global_shutdown(void) { gpr_mu_destroy(&g_neighbourhoods[i].mu); } gpr_free(g_neighbourhoods); + gpr_free(g_neighbour_scan_state); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -591,6 +595,42 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, pollset->shutdown_closure == NULL; } +static bool check_neighbourhood_for_available_poller( + pollset_neighbourhood *neighbourhood, grpc_pollset_worker *avoid_worker) { + bool found_worker = false; + do { + grpc_pollset *inspect = neighbourhood->active_root; + if (inspect == NULL) { + break; + } + gpr_mu_lock(&inspect->mu); + GPR_ASSERT(!inspect->seen_inactive); + grpc_pollset_worker *inspect_worker = inspect->root_worker; + if (inspect_worker == avoid_worker) inspect_worker = inspect_worker->next; + if (inspect_worker == avoid_worker) inspect_worker = NULL; + if (inspect_worker != NULL) { + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = KICKED_FOR_POLL; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + } else { + inspect->seen_inactive = true; + move_pollset_to_neighbourhood_list(inspect, &neighbourhood->active_root, + &neighbourhood->inactive_root); + } + gpr_mu_unlock(&inspect->mu); + } while (!found_worker); + if (!found_worker) { + neighbourhood->seen_inactive = true; + } + return found_worker; +} + static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { @@ -610,49 +650,35 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); + size_t poller_neighbourhood_idx = + (size_t)(pollset->neighbourhood - g_neighbourhoods); bool found_worker = false; - do { - gpr_mu_lock(&neighbourhood->mu); - do { - grpc_pollset *inspect = neighbourhood->active_root; - if (inspect == NULL) { - break; - } - gpr_mu_lock(&inspect->mu); - GPR_ASSERT(!inspect->seen_inactive); - grpc_pollset_worker *inspect_worker = inspect->root_worker; - if (inspect_worker == worker) inspect_worker = worker->next; - if (inspect_worker == worker) inspect_worker = NULL; - if (inspect_worker != NULL) { - if (gpr_atm_no_barrier_cas(&g_active_poller, 0, - (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = KICKED_FOR_POLL; - if (inspect_worker->initialized_cv) gpr_cv_signal(&inspect_worker->cv); - } - // even if we didn't win the cas, there's a worker, we can stop - found_worker = true; - } else { - inspect->seen_inactive = true; - move_pollset_to_neighbourhood_list(inspect, - &neighbourhood->active_root, - &neighbourhood->inactive_root); - } - gpr_mu_unlock(&inspect->mu); - } while (!found_worker); - if (!found_worker) { - neighbourhood->seen_inactive = true; + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + if (gpr_mu_trylock(&neighbourhood->mu)) { + found_worker = + check_neighbourhood_for_available_poller(neighbourhood, worker); + gpr_mu_unlock(&neighbourhood->mu); + g_neighbour_scan_state[i] = true; + } else { + g_neighbour_scan_state[i] = false; } - gpr_mu_unlock(&neighbourhood->mu); - ssize_t cur_neighbourhood_idx = neighbourhood - g_neighbourhoods; - GPR_ASSERT(cur_neighbourhood_idx >= 0); - GPR_ASSERT(g_num_neighbourhoods < INTPTR_MAX); - GPR_ASSERT(cur_neighbourhood_idx < (ssize_t)g_neighbourhoods); - size_t new_neighbourhood_idx = - ((size_t)cur_neighbourhood_idx + 1) % g_num_neighbourhoods; - neighbourhood = &g_neighbourhoods[new_neighbourhood_idx]; - } while (!found_worker && neighbourhood != pollset->neighbourhood); + } + if (!found_worker) { + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + if (g_neighbour_scan_state[i]) continue; + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + gpr_mu_lock(&neighbourhood->mu); + found_worker = + check_neighbourhood_for_available_poller(neighbourhood, worker); + gpr_mu_unlock(&neighbourhood->mu); + } + } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } -- cgit v1.2.3 From b9b2cbfb7177a76afe78501da7ccbaea8fcbfd81 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:12:39 +0000 Subject: rr assignments --- include/grpc++/server_builder.h | 2 +- src/core/lib/iomgr/ev_epoll1_linux.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/core') diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 7ac23349c8..d9de83226b 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,7 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(1), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(GPR_MAX(1,gpr_cpu_num_cores())), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index a25168fdb4..6f0416d0d3 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -386,9 +386,12 @@ static void pollset_global_shutdown(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + static gpr_atm next_neighbourhood; + gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset->neighbourhood = &g_neighbourhoods[(size_t)gpr_atm_no_barrier_fetch_add(&next_neighbourhood, 1) % g_num_neighbourhoods]; +gpr_log(GPR_DEBUG, "nh=%d", (int)(pollset->neighbourhood - g_neighbourhoods)); pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } -- cgit v1.2.3 From 8affd20a9986d294296b625b35fd9ef42127d437 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:41:40 +0000 Subject: reduce epoll events to increase parallelism, kill spam --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index f45be3af38..1794af0ac3 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -395,7 +395,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; pollset->neighbourhood = &g_neighbourhoods[(size_t)gpr_atm_no_barrier_fetch_add(&next_neighbourhood, 1) % g_num_neighbourhoods]; -gpr_log(GPR_DEBUG, "nh=%d", (int)(pollset->neighbourhood - g_neighbourhoods)); pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } @@ -451,7 +450,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 100 +#define MAX_EPOLL_EVENTS 10 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { -- cgit v1.2.3 From a346802846567c15e678a55cff1f957a2c284cc2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:48:00 +0000 Subject: hmm --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 1794af0ac3..34d5cd039c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -450,7 +450,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 10 +#define MAX_EPOLL_EVENTS 100 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { -- cgit v1.2.3 From b4bfc4ac5d83f6e99a1d2762a990c6ccc3f1e2f3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:58:54 +0000 Subject: hmmmm --- src/core/lib/iomgr/ev_epoll1_linux.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 34d5cd039c..27f448f08e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -390,11 +390,9 @@ static void pollset_global_shutdown(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - static gpr_atm next_neighbourhood; - gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[(size_t)gpr_atm_no_barrier_fetch_add(&next_neighbourhood, 1) % g_num_neighbourhoods]; + pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } -- cgit v1.2.3 From 7cb2698abdd599adc67389d7da56d84d29d6a1e5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 23:05:37 +0000 Subject: Lower epoll ev count --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 27f448f08e..b9051338b1 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -448,7 +448,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 100 +#define MAX_EPOLL_EVENTS 10 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { -- cgit v1.2.3 From f4b0fcaa8e046d5ee00dd896c9e674fb249792b2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 16:08:42 -0700 Subject: Add comment --- src/core/lib/iomgr/ev_epoll1_linux.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 27f448f08e..612cdc4c85 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -563,6 +563,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, // active list pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); + // pollset unlocked: state may change (even worker->kick_state) gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (pollset->seen_inactive) { -- cgit v1.2.3 From 43bf259f4e03eb228602fb46f7325205c077ac0e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 23:21:01 +0000 Subject: Fix kicking poller --- src/core/lib/iomgr/ev_epoll1_linux.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index b9051338b1..02fa402388 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -96,7 +96,7 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, KICKED_FOR_POLL } kick_state; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; struct grpc_pollset_worker { kick_state kick_state; @@ -572,7 +572,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (neighbourhood->seen_inactive) { neighbourhood->seen_inactive = false; if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { - worker->kick_state = KICKED_FOR_POLL; + worker->kick_state = DESIGNATED_POLLER; } } } @@ -591,7 +591,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, *now = gpr_now(now->clock_type); } - return worker->kick_state == KICKED_FOR_POLL && + return worker->kick_state == DESIGNATED_POLLER && pollset->shutdown_closure == NULL; } @@ -611,7 +611,7 @@ static bool check_neighbourhood_for_available_poller( if (inspect_worker != NULL) { if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = KICKED_FOR_POLL; + inspect_worker->kick_state = DESIGNATED_POLLER; if (inspect_worker->initialized_cv) { gpr_cv_signal(&inspect_worker->cv); } @@ -641,7 +641,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (worker->next != worker) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); - worker->next->kick_state = KICKED_FOR_POLL; + worker->next->kick_state = DESIGNATED_POLLER; gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { gpr_mu_unlock(&pollset->mu); @@ -746,7 +746,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } else { return GRPC_ERROR_NONE; } - } else if (specific_worker->kick_state != UNKICKED) { + } else if (specific_worker->kick_state == KICKED) { return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { -- cgit v1.2.3 From a4b8eb003ecd69a77f82e6db2b2d825def890de9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 29 Apr 2017 00:13:52 +0000 Subject: Fix loss of poller bug --- src/core/lib/iomgr/ev_epoll1_linux.c | 39 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 9fb640af6b..5c158baa77 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -581,6 +581,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } worker_insert(pollset, worker); if (worker->kick_state == UNKICKED) { + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); do { @@ -597,7 +598,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood, grpc_pollset_worker *avoid_worker) { + pollset_neighbourhood *neighbourhood) { bool found_worker = false; do { grpc_pollset *inspect = neighbourhood->active_root; @@ -607,19 +608,24 @@ static bool check_neighbourhood_for_available_poller( gpr_mu_lock(&inspect->mu); GPR_ASSERT(!inspect->seen_inactive); grpc_pollset_worker *inspect_worker = inspect->root_worker; - if (inspect_worker == avoid_worker) inspect_worker = inspect_worker->next; - if (inspect_worker == avoid_worker) inspect_worker = NULL; if (inspect_worker != NULL) { - if (gpr_atm_no_barrier_cas(&g_active_poller, 0, - (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = DESIGNATED_POLLER; - if (inspect_worker->initialized_cv) { - gpr_cv_signal(&inspect_worker->cv); + do { + if (inspect_worker->kick_state == UNKICKED) { + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = DESIGNATED_POLLER; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + break; } - } - // even if we didn't win the cas, there's a worker, we can stop - found_worker = true; - } else { + inspect_worker = inspect_worker->next; + } while (inspect_worker != inspect->root_worker); + } + if (!found_worker) { inspect->seen_inactive = true; move_pollset_to_neighbourhood_list(inspect, &neighbourhood->active_root, &neighbourhood->inactive_root); @@ -636,10 +642,10 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { if (worker_hdl != NULL) *worker_hdl = NULL; + worker->kick_state = KICKED; if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { GPR_ASSERT(!pollset->seen_inactive); - GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker); - if (worker->next != worker) { + if (worker->next != worker && worker->next->kick_state == UNKICKED) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); worker->next->kick_state = DESIGNATED_POLLER; @@ -661,7 +667,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; if (gpr_mu_trylock(&neighbourhood->mu)) { found_worker = - check_neighbourhood_for_available_poller(neighbourhood, worker); + check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); g_neighbour_scan_state[i] = true; } else { @@ -676,7 +682,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); found_worker = - check_neighbourhood_for_available_poller(neighbourhood, worker); + check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } } @@ -690,6 +696,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (EMPTIED == worker_remove(pollset, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); } + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); } /* pollset->po.mu lock must be held by the caller before calling this. -- cgit v1.2.3 From 2acab6e53a9b2704070d8da3451b7b7b89ad090c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 30 Apr 2017 23:06:33 +0000 Subject: Allow neighbourhood reassignment, add debug --- src/core/lib/iomgr/ev_epoll1_linux.c | 81 +++++++++++++++--------------------- 1 file changed, 34 insertions(+), 47 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5c158baa77..1284891ded 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -109,8 +109,6 @@ struct grpc_pollset_worker { typedef struct pollset_neighbourhood { gpr_mu mu; grpc_pollset *active_root; - grpc_pollset *inactive_root; - bool seen_inactive; char pad[GPR_CACHELINE_SIZE]; } pollset_neighbourhood; @@ -373,7 +371,6 @@ static grpc_error *pollset_global_init(void) { gpr_malloc(sizeof(*g_neighbour_scan_state) * g_num_neighbourhoods); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_init(&g_neighbourhoods[i].mu); - g_neighbourhoods[i].seen_inactive = true; } return GRPC_ERROR_NONE; } @@ -404,9 +401,6 @@ static void pollset_destroy(grpc_pollset *pollset) { if (pollset == pollset->neighbourhood->active_root) { pollset->neighbourhood->active_root = pollset->next == pollset ? NULL : pollset->next; - } else if (pollset == pollset->neighbourhood->inactive_root) { - pollset->neighbourhood->inactive_root = - pollset->next == pollset ? NULL : pollset->next; } gpr_mu_unlock(&pollset->neighbourhood->mu); gpr_mu_destroy(&pollset->mu); @@ -532,25 +526,6 @@ static void verify_neighbourhood_lists(pollset_neighbourhood *neighbourhood) { } #endif -static void move_pollset_to_neighbourhood_list(grpc_pollset *pollset, - grpc_pollset **from_root, - grpc_pollset **to_root) { - // remove from old list - pollset->prev->next = pollset->next; - pollset->next->prev = pollset->prev; - if (*from_root == pollset) { - *from_root = pollset->next == pollset ? NULL : pollset->next; - } - // add to new list - if (*to_root == NULL) { - *to_root = pollset->next = pollset->prev = pollset; - } else { - pollset->next = *to_root; - pollset->prev = pollset->next->prev; - pollset->next->prev = pollset->prev->next = pollset; - } -} - static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { @@ -561,20 +536,29 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (pollset->seen_inactive) { // pollset has been observed to be inactive, we need to move back to the // active list - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighbourhood *neighbourhood = pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) +retry_lock_neighbourhood: gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (pollset->seen_inactive) { + if (neighbourhood != pollset->neighbourhood) { + gpr_mu_unlock(&neighbourhood->mu); + neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + goto retry_lock_neighbourhood; + } pollset->seen_inactive = false; - move_pollset_to_neighbourhood_list(pollset, &neighbourhood->inactive_root, - &neighbourhood->active_root); - if (neighbourhood->seen_inactive) { - neighbourhood->seen_inactive = false; + if (neighbourhood->active_root == NULL) { + neighbourhood->active_root = pollset->next = pollset->prev = pollset; if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { worker->kick_state = DESIGNATED_POLLER; } + } else { + pollset->next = neighbourhood->active_root; + pollset->prev = pollset->next->prev; + pollset->next->prev = pollset->prev->next = pollset; } } gpr_mu_unlock(&neighbourhood->mu); @@ -627,14 +611,18 @@ static bool check_neighbourhood_for_available_poller( } if (!found_worker) { inspect->seen_inactive = true; - move_pollset_to_neighbourhood_list(inspect, &neighbourhood->active_root, - &neighbourhood->inactive_root); + if (inspect == neighbourhood->active_root) { + if (inspect->next == neighbourhood->active_root) { + neighbourhood->active_root = NULL; + } else { + neighbourhood->active_root = inspect->next; + } + } + inspect->next->prev = inspect->prev; + inspect->prev->next = inspect->next; } gpr_mu_unlock(&inspect->mu); } while (!found_worker); - if (!found_worker) { - neighbourhood->seen_inactive = true; - } return found_worker; } @@ -646,7 +634,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { GPR_ASSERT(!pollset->seen_inactive); if (worker->next != worker && worker->next->kick_state == UNKICKED) { - assert(worker->next->initialized_cv); + GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); worker->next->kick_state = DESIGNATED_POLLER; gpr_cv_signal(&worker->next->cv); @@ -674,17 +662,15 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_neighbour_scan_state[i] = false; } } - if (!found_worker) { - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - if (g_neighbour_scan_state[i]) continue; - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - gpr_mu_lock(&neighbourhood->mu); - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); - } + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + if (g_neighbour_scan_state[i]) continue; + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + gpr_mu_lock(&neighbourhood->mu); + found_worker = + check_neighbourhood_for_available_poller(neighbourhood); + gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); @@ -717,6 +703,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); + GPR_ASSERT(!pollset->seen_inactive); gpr_mu_unlock(&pollset->mu); append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), err_desc); -- cgit v1.2.3 From ba550da853b9e036ff29c5260c8f17ec4c03fc02 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 14:26:31 +0000 Subject: Fix several races --- src/core/lib/iomgr/ev_epoll1_linux.c | 87 +++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 35 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 1284891ded..fccccccd09 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -106,6 +106,8 @@ struct grpc_pollset_worker { gpr_cv cv; }; +#define MAX_NEIGHBOURHOODS 1024 + typedef struct pollset_neighbourhood { gpr_mu mu; grpc_pollset *active_root; @@ -121,6 +123,7 @@ struct grpc_pollset { bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ + int begin_refs; grpc_pollset *next; grpc_pollset *prev; @@ -312,7 +315,6 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; -static bool *g_neighbour_scan_state; static size_t g_num_neighbourhoods; /* Return true if first in list */ @@ -352,6 +354,10 @@ static worker_remove_result worker_remove(grpc_pollset *pollset, } } +static size_t choose_neighbourhood(void) { + return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods; +} + static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); @@ -364,11 +370,9 @@ static grpc_error *pollset_global_init(void) { if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } - g_num_neighbourhoods = GPR_MAX(1, gpr_cpu_num_cores()); + g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS); g_neighbourhoods = gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); - g_neighbour_scan_state = - gpr_malloc(sizeof(*g_neighbour_scan_state) * g_num_neighbourhoods); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_init(&g_neighbourhoods[i].mu); } @@ -383,26 +387,27 @@ static void pollset_global_shutdown(void) { gpr_mu_destroy(&g_neighbourhoods[i].mu); } gpr_free(g_neighbourhoods); - gpr_free(g_neighbour_scan_state); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_lock(&pollset->neighbourhood->mu); - pollset->prev->next = pollset->next; - pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = - pollset->next == pollset ? NULL : pollset->next; - } - gpr_mu_unlock(&pollset->neighbourhood->mu); + if (!pollset->seen_inactive) { + gpr_mu_lock(&pollset->neighbourhood->mu); + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } + gpr_mu_unlock(&pollset->neighbourhood->mu); + } gpr_mu_destroy(&pollset->mu); } @@ -428,7 +433,8 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL && + pollset->begin_refs == 0) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); pollset->shutdown_closure = NULL; } @@ -532,14 +538,16 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; worker->kick_state = UNKICKED; + pollset->begin_refs++; if (pollset->seen_inactive) { // pollset has been observed to be inactive, we need to move back to the // active list - pollset_neighbourhood *neighbourhood = pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset_neighbourhood *neighbourhood = pollset->neighbourhood = + &g_neighbourhoods[choose_neighbourhood()]; gpr_mu_unlock(&pollset->mu); - // pollset unlocked: state may change (even worker->kick_state) -retry_lock_neighbourhood: + // pollset unlocked: state may change (even worker->kick_state) + retry_lock_neighbourhood: gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (pollset->seen_inactive) { @@ -564,16 +572,18 @@ retry_lock_neighbourhood: gpr_mu_unlock(&neighbourhood->mu); } worker_insert(pollset, worker); + pollset->begin_refs--; if (worker->kick_state == UNKICKED) { GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); - do { + while (worker->kick_state == UNKICKED && + pollset->shutdown_closure == NULL) { if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && worker->kick_state == UNKICKED) { worker->kick_state = KICKED; } - } while (worker->kick_state == UNKICKED); + } *now = gpr_now(now->clock_type); } @@ -594,17 +604,24 @@ static bool check_neighbourhood_for_available_poller( grpc_pollset_worker *inspect_worker = inspect->root_worker; if (inspect_worker != NULL) { do { - if (inspect_worker->kick_state == UNKICKED) { - if (gpr_atm_no_barrier_cas(&g_active_poller, 0, - (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = DESIGNATED_POLLER; - if (inspect_worker->initialized_cv) { - gpr_cv_signal(&inspect_worker->cv); + switch (inspect_worker->kick_state) { + case UNKICKED: + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = DESIGNATED_POLLER; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } } - } - // even if we didn't win the cas, there's a worker, we can stop - found_worker = true; - break; + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + break; + case KICKED: + break; + case DESIGNATED_POLLER: + found_worker = true; // ok, so someone else found the worker, but + // we'll accept that + break; } inspect_worker = inspect_worker->next; } while (inspect_worker != inspect->root_worker); @@ -649,6 +666,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, size_t poller_neighbourhood_idx = (size_t)(pollset->neighbourhood - g_neighbourhoods); bool found_worker = false; + bool scan_state[MAX_NEIGHBOURHOODS]; for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { pollset_neighbourhood *neighbourhood = &g_neighbourhoods[(poller_neighbourhood_idx + i) % @@ -657,19 +675,18 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, found_worker = check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); - g_neighbour_scan_state[i] = true; + scan_state[i] = true; } else { - g_neighbour_scan_state[i] = false; + scan_state[i] = false; } } for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - if (g_neighbour_scan_state[i]) continue; + if (scan_state[i]) continue; pollset_neighbourhood *neighbourhood = &g_neighbourhoods[(poller_neighbourhood_idx + i) % g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); + found_worker = check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); -- cgit v1.2.3 From e00d7335de79fb52ecba1756a991829feb6c1c7d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 15:43:51 +0000 Subject: Refine neighbourhood reassignment --- src/core/lib/iomgr/ev_epoll1_linux.c | 48 +++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 14 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index fccccccd09..99e4441a6c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -117,6 +117,7 @@ typedef struct pollset_neighbourhood { struct grpc_pollset { gpr_mu mu; pollset_neighbourhood *neighbourhood; + bool reassigning_neighbourhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; bool seen_inactive; @@ -394,20 +395,33 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { *mu = &pollset->mu; pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; pollset->seen_inactive = true; - pollset->next = pollset->prev = pollset; } static void pollset_destroy(grpc_pollset *pollset) { + gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - gpr_mu_lock(&pollset->neighbourhood->mu); - pollset->prev->next = pollset->next; - pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = - pollset->next == pollset ? NULL : pollset->next; + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); +retry_lock_neighbourhood: + gpr_mu_lock(&neighbourhood->mu); + gpr_mu_lock(&pollset->mu); + if (!pollset->seen_inactive) { + if (pollset->neighbourhood != neighbourhood) { + gpr_mu_unlock(&neighbourhood->mu); + neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + goto retry_lock_neighbourhood; + } + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } } gpr_mu_unlock(&pollset->neighbourhood->mu); } + gpr_mu_unlock(&pollset->mu); gpr_mu_destroy(&pollset->mu); } @@ -543,8 +557,13 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (pollset->seen_inactive) { // pollset has been observed to be inactive, we need to move back to the // active list - pollset_neighbourhood *neighbourhood = pollset->neighbourhood = - &g_neighbourhoods[choose_neighbourhood()]; + bool is_reassigning = false; + if (!pollset->reassigning_neighbourhood) { + is_reassigning = true; + pollset->reassigning_neighbourhood = true; + pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + } + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) retry_lock_neighbourhood: @@ -569,6 +588,10 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, pollset->next->prev = pollset->prev->next = pollset; } } + if (is_reassigning) { + GPR_ASSERT(pollset->reassigning_neighbourhood); + pollset->reassigning_neighbourhood = false; + } gpr_mu_unlock(&neighbourhood->mu); } worker_insert(pollset, worker); @@ -629,14 +652,11 @@ static bool check_neighbourhood_for_available_poller( if (!found_worker) { inspect->seen_inactive = true; if (inspect == neighbourhood->active_root) { - if (inspect->next == neighbourhood->active_root) { - neighbourhood->active_root = NULL; - } else { - neighbourhood->active_root = inspect->next; - } + neighbourhood->active_root = inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; inspect->prev->next = inspect->next; + inspect->next = inspect->prev = NULL; } gpr_mu_unlock(&inspect->mu); } while (!found_worker); -- cgit v1.2.3 From e45a277b56273ae63dca8a0bb65e0337c15f7e37 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 15:56:20 +0000 Subject: Remove bogus assert --- src/core/lib/iomgr/ev_epoll1_linux.c | 1 - 1 file changed, 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 99e4441a6c..8946d32697 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -669,7 +669,6 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (worker_hdl != NULL) *worker_hdl = NULL; worker->kick_state = KICKED; if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { - GPR_ASSERT(!pollset->seen_inactive); if (worker->next != worker && worker->next->kick_state == UNKICKED) { GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); -- cgit v1.2.3 From a95bacf7dba3d5ffc8968729525a4cd5ff0ff88c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 12:51:24 -0700 Subject: clang-format, revert parameter --- include/grpc++/server_builder.h | 5 ++++- src/core/lib/iomgr/ev_epoll1_linux.c | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'src/core') diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index d9de83226b..5a8577659a 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,10 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(GPR_MAX(1,gpr_cpu_num_cores())), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(GPR_MAX(1, gpr_cpu_num_cores())), + min_pollers(1), + max_pollers(2), + cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8946d32697..29254e6324 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -402,7 +402,7 @@ static void pollset_destroy(grpc_pollset *pollset) { if (!pollset->seen_inactive) { pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); -retry_lock_neighbourhood: + retry_lock_neighbourhood: gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { @@ -462,7 +462,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 10 +#define MAX_EPOLL_EVENTS 100 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { @@ -652,7 +652,8 @@ static bool check_neighbourhood_for_available_poller( if (!found_worker) { inspect->seen_inactive = true; if (inspect == neighbourhood->active_root) { - neighbourhood->active_root = inspect->next == inspect ? NULL : inspect->next; + neighbourhood->active_root = + inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; inspect->prev->next = inspect->next; -- cgit v1.2.3 From 50da5ec21d3d8be5e76b9809242821f9e5badba1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 13:51:14 -0700 Subject: Add workqueue --- src/core/lib/iomgr/ev_epoll1_linux.c | 84 +++++++++++++++++++++++++++--------- src/core/lib/support/mpscq.c | 13 +++++- src/core/lib/support/mpscq.h | 4 ++ 3 files changed, 80 insertions(+), 21 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 29254e6324..cce52b2d94 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -104,6 +104,7 @@ struct grpc_pollset_worker { grpc_pollset_worker *next; grpc_pollset_worker *prev; gpr_cv cv; + grpc_closure_list schedule_on_end_work; }; #define MAX_NEIGHBOURHOODS 1024 @@ -288,7 +289,7 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - return NULL; /* TODO(ctiller): add a global workqueue */ + return (grpc_workqueue *)0xb0b51ed; } static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, @@ -317,6 +318,7 @@ GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; static size_t g_num_neighbourhoods; +static gpr_mpscq g_workqueue_items; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -365,6 +367,7 @@ static grpc_error *pollset_global_init(void) { gpr_atm_no_barrier_store(&g_active_poller, 0); global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); + gpr_mpscq_init(&g_workqueue_items); if (err != GRPC_ERROR_NONE) return err; struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &global_wakeup_fd}; @@ -383,6 +386,7 @@ static grpc_error *pollset_global_init(void) { static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); + gpr_mpscq_destroy(&g_workqueue_items); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_destroy(&g_neighbourhoods[i].mu); @@ -528,30 +532,13 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } -#if 0 -static void verify_all_entries_in_neighbourhood_list( - grpc_pollset *root, bool should_be_seen_inactive) { - if (root == NULL) return; - grpc_pollset *p = root; - do { - GPR_ASSERT(p->seen_inactive == should_be_seen_inactive); - p = p->next; - } while (p != root); -} - -static void verify_neighbourhood_lists(pollset_neighbourhood *neighbourhood) { - // assumes neighbourhood->mu locked - verify_all_entries_in_neighbourhood_list(neighbourhood->active_root, false); - verify_all_entries_in_neighbourhood_list(neighbourhood->inactive_root, true); -} -#endif - static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; worker->kick_state = UNKICKED; + worker->schedule_on_end_work = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; pollset->begin_refs++; if (pollset->seen_inactive) { @@ -669,6 +656,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl) { if (worker_hdl != NULL) *worker_hdl = NULL; worker->kick_state = KICKED; + grpc_closure_list_move(&worker->schedule_on_end_work, + &exec_ctx->closure_list); if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { if (worker->next != worker && worker->next->kick_state == UNKICKED) { GPR_ASSERT(worker->next->initialized_cv); @@ -712,6 +701,10 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } + } else if (grpc_exec_ctx_has_work(exec_ctx)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); @@ -828,8 +821,59 @@ static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {} #endif +static void wq_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + // find a neighbourhood to wakeup + bool scheduled = false; + size_t initial_neighbourhood = choose_neighbourhood(); + for (size_t i = 0; !scheduled && i < g_num_neighbourhoods; i++) { + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(initial_neighbourhood + i) % g_num_neighbourhoods]; + if (gpr_mu_trylock(&neighbourhood->mu)) { + if (neighbourhood->active_root != NULL) { + grpc_pollset *inspect = neighbourhood->active_root; + do { + if (gpr_mu_trylock(&inspect->mu)) { + if (inspect->root_worker != NULL) { + grpc_pollset_worker *inspect_worker = inspect->root_worker; + do { + if (inspect_worker->kick_state == UNKICKED) { + inspect_worker->kick_state = KICKED; + grpc_closure_list_append( + &inspect_worker->schedule_on_end_work, closure, error); + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + scheduled = true; + } + inspect_worker = inspect_worker->next; + } while (!scheduled && inspect_worker != inspect->root_worker); + } + gpr_mu_unlock(&inspect->mu); + } + inspect = inspect->next; + } while (!scheduled && inspect != neighbourhood->active_root); + } + gpr_mu_unlock(&neighbourhood->mu); + } + } + if (!scheduled) { + closure->error_data.error = error; + gpr_mpscq_push(&g_workqueue_items, &closure->next_data.atm_next); + GRPC_LOG_IF_ERROR("workqueue_scheduler", + grpc_wakeup_fd_wakeup(&global_wakeup_fd)); + } +} + +static const grpc_closure_scheduler_vtable + singleton_workqueue_scheduler_vtable = {wq_sched, wq_sched, + "epoll1_workqueue"}; + +static grpc_closure_scheduler singleton_workqueue_scheduler = { + &singleton_workqueue_scheduler_vtable}; + static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - return grpc_schedule_on_exec_ctx; + return &singleton_workqueue_scheduler; } /******************************************************************************* diff --git a/src/core/lib/support/mpscq.c b/src/core/lib/support/mpscq.c index 5b9323275a..1015cc6776 100644 --- a/src/core/lib/support/mpscq.c +++ b/src/core/lib/support/mpscq.c @@ -54,21 +54,31 @@ void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) { } gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) { + bool empty; + return gpr_mpscq_pop_and_check_end(q, &empty); +} + +gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty) { gpr_mpscq_node *tail = q->tail; gpr_mpscq_node *next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next); if (tail == &q->stub) { // indicates the list is actually (ephemerally) empty - if (next == NULL) return NULL; + if (next == NULL) { + *empty = true; + return NULL; + } q->tail = next; tail = next; next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next); } if (next != NULL) { + *empty = false; q->tail = next; return tail; } gpr_mpscq_node *head = (gpr_mpscq_node *)gpr_atm_acq_load(&q->head); if (tail != head) { + *empty = false; // indicates a retry is in order: we're still adding return NULL; } @@ -79,5 +89,6 @@ gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) { return tail; } // indicates a retry is in order: we're still adding + *empty = false; return NULL; } diff --git a/src/core/lib/support/mpscq.h b/src/core/lib/support/mpscq.h index 977a117952..24c89f90c9 100644 --- a/src/core/lib/support/mpscq.h +++ b/src/core/lib/support/mpscq.h @@ -35,6 +35,7 @@ #define GRPC_CORE_LIB_SUPPORT_MPSCQ_H #include +#include #include // Multiple-producer single-consumer lock free queue, based upon the @@ -62,4 +63,7 @@ void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n); // the queue is empty!!) gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q); +// Pop a node; sets *empty to true if the queue is empty, or false if it is not +gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty); + #endif /* GRPC_CORE_LIB_SUPPORT_MPSCQ_H */ -- cgit v1.2.3 From 67e229e5c124efa826ef15e8a7e16634786fabeb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 20:57:59 +0000 Subject: Fix wakeup path --- src/core/lib/iomgr/ev_epoll1_linux.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index cce52b2d94..8766e4a1aa 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -318,7 +318,8 @@ GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; static size_t g_num_neighbourhoods; -static gpr_mpscq g_workqueue_items; +static gpr_mu g_wq_mu; +static grpc_closure_list g_wq_items; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -367,7 +368,8 @@ static grpc_error *pollset_global_init(void) { gpr_atm_no_barrier_store(&g_active_poller, 0); global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); - gpr_mpscq_init(&g_workqueue_items); + gpr_mu_init(&g_wq_mu); + g_wq_items = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; if (err != GRPC_ERROR_NONE) return err; struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &global_wakeup_fd}; @@ -386,7 +388,7 @@ static grpc_error *pollset_global_init(void) { static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); - gpr_mpscq_destroy(&g_workqueue_items); + gpr_mu_destroy(&g_wq_mu); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_destroy(&g_neighbourhoods[i].mu); @@ -513,6 +515,9 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (gpr_atm_no_barrier_cas(&g_timer_kick, 1, 0)) { grpc_timer_consume_kick(); } + gpr_mu_lock(&g_wq_mu); + grpc_closure_list_move(&g_wq_items, &exec_ctx->closure_list); + gpr_mu_unlock(&g_wq_mu); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else { @@ -858,8 +863,9 @@ static void wq_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, } } if (!scheduled) { - closure->error_data.error = error; - gpr_mpscq_push(&g_workqueue_items, &closure->next_data.atm_next); + gpr_mu_lock(&g_wq_mu); + grpc_closure_list_append(&g_wq_items, closure, error); + gpr_mu_unlock(&g_wq_mu); GRPC_LOG_IF_ERROR("workqueue_scheduler", grpc_wakeup_fd_wakeup(&global_wakeup_fd)); } -- cgit v1.2.3 From c3571791a5e20ed82cc2efebd21d48f898f13eba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 2 May 2017 12:33:38 -0700 Subject: Isolate timer checking in its own thread --- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + src/core/lib/iomgr/ev_epoll_linux.c | 25 +--- src/core/lib/iomgr/ev_poll_posix.c | 31 +--- src/core/lib/iomgr/ev_posix.c | 2 - src/core/lib/iomgr/ev_posix.h | 2 - src/core/lib/iomgr/iomgr.c | 3 + src/core/lib/iomgr/pollset_windows.c | 2 - src/core/lib/iomgr/timer_manager.c | 166 +++++++++++++++++++++ src/core/lib/iomgr/timer_manager.h | 43 ++++++ src/core/lib/surface/completion_queue.c | 67 +++------ src/python/grpcio/grpc_core_dependencies.py | 1 + test/cpp/qps/client_async.cc | 48 +++--- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + tools/run_tests/generated/sources_and_headers.json | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util/grpc_test_util.vcxproj.filters | 6 + .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 6 + 32 files changed, 340 insertions(+), 127 deletions(-) create mode 100644 src/core/lib/iomgr/timer_manager.c create mode 100644 src/core/lib/iomgr/timer_manager.h (limited to 'src/core') diff --git a/CMakeLists.txt b/CMakeLists.txt index 325344a81c..b52a42f7e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -984,6 +984,7 @@ add_library(grpc src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -1310,6 +1311,7 @@ add_library(grpc_cronet src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -1621,6 +1623,7 @@ add_library(grpc_test_util src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -1877,6 +1880,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -2296,6 +2300,7 @@ add_library(grpc++ src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -2621,6 +2626,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -3390,6 +3396,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c diff --git a/Makefile b/Makefile index 4597b6fe97..d9de26f546 100644 --- a/Makefile +++ b/Makefile @@ -2967,6 +2967,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -3291,6 +3292,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -3601,6 +3603,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -3829,6 +3832,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -4225,6 +4229,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -4558,6 +4563,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -5317,6 +5323,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ diff --git a/binding.gyp b/binding.gyp index 582c61282f..9434ab6d32 100644 --- a/binding.gyp +++ b/binding.gyp @@ -718,6 +718,7 @@ 'src/core/lib/iomgr/time_averaged_stats.c', 'src/core/lib/iomgr/timer_generic.c', 'src/core/lib/iomgr/timer_heap.c', + 'src/core/lib/iomgr/timer_manager.c', 'src/core/lib/iomgr/timer_uv.c', 'src/core/lib/iomgr/udp_server.c', 'src/core/lib/iomgr/unix_sockets_posix.c', diff --git a/build.yaml b/build.yaml index 4cfa75cae1..9453c1a2b3 100644 --- a/build.yaml +++ b/build.yaml @@ -238,6 +238,7 @@ filegroups: - src/core/lib/iomgr/timer.h - src/core/lib/iomgr/timer_generic.h - src/core/lib/iomgr/timer_heap.h + - src/core/lib/iomgr/timer_manager.h - src/core/lib/iomgr/timer_uv.h - src/core/lib/iomgr/udp_server.h - src/core/lib/iomgr/unix_sockets_posix.h @@ -350,6 +351,7 @@ filegroups: - src/core/lib/iomgr/time_averaged_stats.c - src/core/lib/iomgr/timer_generic.c - src/core/lib/iomgr/timer_heap.c + - src/core/lib/iomgr/timer_manager.c - src/core/lib/iomgr/timer_uv.c - src/core/lib/iomgr/udp_server.c - src/core/lib/iomgr/unix_sockets_posix.c diff --git a/config.m4 b/config.m4 index bbd667c9ec..36c04cf07b 100644 --- a/config.m4 +++ b/config.m4 @@ -154,6 +154,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 07755ac727..120463f40c 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -320,6 +320,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/timer.h', 'src/core/lib/iomgr/timer_generic.h', 'src/core/lib/iomgr/timer_heap.h', + 'src/core/lib/iomgr/timer_manager.h', 'src/core/lib/iomgr/timer_uv.h', 'src/core/lib/iomgr/udp_server.h', 'src/core/lib/iomgr/unix_sockets_posix.h', @@ -531,6 +532,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/time_averaged_stats.c', 'src/core/lib/iomgr/timer_generic.c', 'src/core/lib/iomgr/timer_heap.c', + 'src/core/lib/iomgr/timer_manager.c', 'src/core/lib/iomgr/timer_uv.c', 'src/core/lib/iomgr/udp_server.c', 'src/core/lib/iomgr/unix_sockets_posix.c', @@ -781,6 +783,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/timer.h', 'src/core/lib/iomgr/timer_generic.h', 'src/core/lib/iomgr/timer_heap.h', + 'src/core/lib/iomgr/timer_manager.h', 'src/core/lib/iomgr/timer_uv.h', 'src/core/lib/iomgr/udp_server.h', 'src/core/lib/iomgr/unix_sockets_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index 1cd6d66335..7841f27102 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -236,6 +236,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/timer.h ) s.files += %w( src/core/lib/iomgr/timer_generic.h ) s.files += %w( src/core/lib/iomgr/timer_heap.h ) + s.files += %w( src/core/lib/iomgr/timer_manager.h ) s.files += %w( src/core/lib/iomgr/timer_uv.h ) s.files += %w( src/core/lib/iomgr/udp_server.h ) s.files += %w( src/core/lib/iomgr/unix_sockets_posix.h ) @@ -447,6 +448,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/time_averaged_stats.c ) s.files += %w( src/core/lib/iomgr/timer_generic.c ) s.files += %w( src/core/lib/iomgr/timer_heap.c ) + s.files += %w( src/core/lib/iomgr/timer_manager.c ) s.files += %w( src/core/lib/iomgr/timer_uv.c ) s.files += %w( src/core/lib/iomgr/udp_server.c ) s.files += %w( src/core/lib/iomgr/unix_sockets_posix.c ) diff --git a/package.xml b/package.xml index e7d67eca18..3f7ddb9e49 100644 --- a/package.xml +++ b/package.xml @@ -245,6 +245,7 @@ + @@ -456,6 +457,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index e603a75593..735c37be95 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -76,11 +76,6 @@ static int grpc_polling_trace = 0; /* Disabled by default */ static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - /* Implements the function defined in grpc_posix.h. This function might be * called before even calling grpc_init() to set either a different signal to * use. If signum == -1, then the use of signals is disabled */ @@ -454,8 +449,8 @@ static void polling_island_add_wakeup_fd_locked(polling_island *pi, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), - errno, strerror(errno)); + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -558,7 +553,6 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, goto done; } - polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); if (initial_fd != NULL) { @@ -1116,11 +1110,10 @@ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); poller_kick_init(); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } @@ -1226,10 +1219,6 @@ static grpc_error *pollset_kick(grpc_pollset *p, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->po.mu); *mu = &pollset->po.mu; @@ -1453,11 +1442,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, for (int i = 0; i < ep_rv; ++i) { void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { + if (data_ptr == &pi->workqueue_wakeup_fd) { append_error(error, grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), err_desc); @@ -1897,8 +1882,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 9834cdd197..3ef24be569 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -122,8 +122,6 @@ struct grpc_fd { grpc_pollset *read_notifier_pollset; }; -static grpc_wakeup_fd global_wakeup_fd; - /* Begin polling on an fd. Registers that the given pollset is interested in this fd - so that if read or writability interest changes, the pollset can be kicked to pick up that @@ -784,19 +782,14 @@ static grpc_error *pollset_kick(grpc_pollset *p, static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_poller); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_poller); gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - /* main interface */ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -952,13 +945,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } fd_count = 0; - pfd_count = 2; - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd); + pfd_count = 1; + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd); pfds[0].events = POLLIN; pfds[0].revents = 0; - pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd); - pfds[1].events = POLLIN; - pfds[1].revents = 0; for (i = 0; i < pollset->fd_count; i++) { if (fd_is_orphaned(pollset->fds[i])) { GRPC_FD_UNREF(pollset->fds[i], "multipoller"); @@ -974,7 +964,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->fd_count = fd_count; gpr_mu_unlock(&pollset->mu); - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { grpc_fd *fd = watchers[i].fd; pfds[i].events = (short)fd_begin_poll(fd, pollset, &worker, POLLIN, POLLOUT, &watchers[i]); @@ -992,7 +982,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); } - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == NULL) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } else { @@ -1002,20 +992,15 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } } else if (r == 0) { - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } } else { if (pfds[0].revents & POLLIN_CHECK) { - grpc_timer_consume_kick(); - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd)); - } - if (pfds[1].revents & POLLIN_CHECK) { work_combine_error( &error, grpc_wakeup_fd_consume_wakeup(&worker.wakeup_fd->fd)); } - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == NULL) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } else { @@ -1560,8 +1545,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de8..41464c137a 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -260,8 +260,6 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, g_event_engine->pollset_set_del_fd(exec_ctx, pollset_set, fd); } -grpc_error *grpc_kick_poller(void) { return g_event_engine->kick_poller(); } - #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG grpc_workqueue *grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, const char *reason) { diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index becc4d359e..a77720e61f 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -93,8 +93,6 @@ typedef struct grpc_event_engine_vtable { void (*pollset_set_del_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_fd *fd); - grpc_error *(*kick_poller)(void); - void (*shutdown_engine)(void); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 001e528409..445f7aa422 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -47,6 +47,7 @@ #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/network_status_tracker.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" @@ -65,6 +66,7 @@ void grpc_iomgr_init(void) { g_root_object.name = "root"; grpc_network_status_init(); grpc_iomgr_platform_init(); + grpc_timer_manager_init(); } static size_t count_objects(void) { @@ -88,6 +90,7 @@ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx) { gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN)); gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME); + grpc_timer_manager_shutdown(); grpc_iomgr_platform_flush(); gpr_mu_lock(&g_mu); diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 04c6b71747..bea4232273 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -227,6 +227,4 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p, return GRPC_ERROR_NONE; } -void grpc_kick_poller(void) { grpc_iocp_kick(); } - #endif /* GRPC_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c new file mode 100644 index 0000000000..00e868de01 --- /dev/null +++ b/src/core/lib/iomgr/timer_manager.c @@ -0,0 +1,166 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/timer_manager.h" + +#include +#include +#include + +#include "src/core/lib/iomgr/timer.h" + +typedef struct completed_thread { + gpr_thd_id t; + struct completed_thread *next; +} completed_thread; + +static gpr_mu g_mu; +static gpr_cv g_cv_wait; +static gpr_cv g_cv_shutdown; +static int g_thread_count; +static int g_waiter_count; +static bool g_shutdown; +static completed_thread *g_completed_threads; +static bool g_kicked; + +#define MAX_WAITERS 3 + +static void timer_thread(void *unused); + +static void gc_completed_threads(void) { + if (g_completed_threads != NULL) { + completed_thread *to_gc = g_completed_threads; + g_completed_threads = NULL; + gpr_mu_unlock(&g_mu); + while (to_gc != NULL) { + gpr_thd_join(to_gc->t); + completed_thread *next = to_gc->next; + gpr_free(to_gc); + to_gc = next; + } + gpr_mu_lock(&g_mu); + } +} + +static void start_timer_thread_and_unlock(void) { + ++g_waiter_count; + ++g_thread_count; + gpr_mu_unlock(&g_mu); + gpr_log(GPR_DEBUG, "Spawn timer thread"); + gpr_thd_id thd; + gpr_thd_options opt = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&opt); + gpr_thd_new(&thd, timer_thread, NULL, &opt); +} + +static void timer_thread(void *unused) { + grpc_exec_ctx exec_ctx = + GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + for (;;) { + gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + if (grpc_timer_check(&exec_ctx, now, &next)) { + gpr_mu_lock(&g_mu); + --g_waiter_count; + bool start_thread = g_waiter_count == 0; + if (start_thread && !g_shutdown) { + start_timer_thread_and_unlock(); + } else { + gpr_mu_unlock(&g_mu); + } + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(&g_mu); + gc_completed_threads(); + ++g_waiter_count; + gpr_mu_unlock(&g_mu); + } else { + gpr_mu_lock(&g_mu); + if (g_shutdown) break; + if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) { + if (g_kicked) { + grpc_timer_consume_kick(); + g_kicked = false; + } else if (g_waiter_count > MAX_WAITERS) { + break; + } + } + gpr_mu_unlock(&g_mu); + } + } + --g_waiter_count; + --g_thread_count; + if (0 == g_thread_count) { + gpr_cv_signal(&g_cv_shutdown); + } + completed_thread *ct = gpr_malloc(sizeof(*ct)); + ct->t = gpr_thd_currentid(); + ct->next = g_completed_threads; + g_completed_threads = ct; + gpr_mu_unlock(&g_mu); + gpr_log(GPR_DEBUG, "End timer thread"); +} + +void grpc_timer_manager_init(void) { + gpr_mu_init(&g_mu); + gpr_cv_init(&g_cv_wait); + gpr_cv_init(&g_cv_shutdown); + g_thread_count = 0; + g_waiter_count = 0; + g_shutdown = false; + g_completed_threads = NULL; + + gpr_mu_lock(&g_mu); + start_timer_thread_and_unlock(); +} + +void grpc_timer_manager_shutdown(void) { + gpr_mu_lock(&g_mu); + g_shutdown = true; + gpr_cv_broadcast(&g_cv_wait); + while (g_thread_count > 0) { + gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + gc_completed_threads(); + } + gpr_mu_unlock(&g_mu); + + gpr_mu_destroy(&g_mu); + gpr_cv_destroy(&g_cv_wait); + gpr_cv_destroy(&g_cv_shutdown); +} + +void grpc_kick_poller(void) { + gpr_mu_lock(&g_mu); + g_kicked = true; + gpr_cv_signal(&g_cv_wait); + gpr_mu_unlock(&g_mu); +} diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h new file mode 100644 index 0000000000..a24c9da328 --- /dev/null +++ b/src/core/lib/iomgr/timer_manager.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H +#define GRPC_CORE_IOMGR_TIMER_MANAGER_H + +/* Timer Manager tries to keep one thread waiting for the next timeout at all + times */ + +void grpc_timer_manager_init(void); +void grpc_timer_manager_shutdown(void); + +#endif diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index eae3f103b1..048564e32f 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -580,31 +580,18 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, dump_pending_tags(cc); break; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - gpr_timespec iteration_deadline = deadline; - if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { - GPR_TIMER_MARK("alarm_triggered", 0); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + NULL, now, deadline); + if (err != GRPC_ERROR_NONE) { gpr_mu_unlock(cc->mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - continue; - } else { - grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), - NULL, now, iteration_deadline); - if (err != GRPC_ERROR_NONE) { - gpr_mu_unlock(cc->mu); - const char *msg = grpc_error_string(err); - gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); + const char *msg = grpc_error_string(err); + gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); - GRPC_ERROR_UNREF(err); - memset(&ret, 0, sizeof(ret)); - ret.type = GRPC_QUEUE_TIMEOUT; - dump_pending_tags(cc); - break; - } + GRPC_ERROR_UNREF(err); + memset(&ret, 0, sizeof(ret)); + ret.type = GRPC_QUEUE_TIMEOUT; + dump_pending_tags(cc); + break; } is_finished_arg.first_loop = false; } @@ -773,31 +760,19 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, dump_pending_tags(cc); break; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - gpr_timespec iteration_deadline = deadline; - if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { - GPR_TIMER_MARK("alarm_triggered", 0); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + &worker, now, deadline); + if (err != GRPC_ERROR_NONE) { + del_plucker(cc, tag, &worker); gpr_mu_unlock(cc->mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - } else { - grpc_error *err = cc->poller_vtable->work( - &exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); - if (err != GRPC_ERROR_NONE) { - del_plucker(cc, tag, &worker); - gpr_mu_unlock(cc->mu); - const char *msg = grpc_error_string(err); - gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); + const char *msg = grpc_error_string(err); + gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); - GRPC_ERROR_UNREF(err); - memset(&ret, 0, sizeof(ret)); - ret.type = GRPC_QUEUE_TIMEOUT; - dump_pending_tags(cc); - break; - } + GRPC_ERROR_UNREF(err); + memset(&ret, 0, sizeof(ret)); + ret.type = GRPC_QUEUE_TIMEOUT; + dump_pending_tags(cc); + break; } is_finished_arg.first_loop = false; del_plucker(cc, tag, &worker); diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d2a570cc87..02328867ae 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -143,6 +143,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/time_averaged_stats.c', 'src/core/lib/iomgr/timer_generic.c', 'src/core/lib/iomgr/timer_heap.c', + 'src/core/lib/iomgr/timer_manager.c', 'src/core/lib/iomgr/timer_uv.c', 'src/core/lib/iomgr/udp_server.c', 'src/core/lib/iomgr/unix_sockets_posix.c', diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 29a79e7343..751986d7ac 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -238,39 +238,27 @@ class AsyncClient : public ClientImpl { void* got_tag; bool ok; - switch (cli_cqs_[thread_idx]->AsyncNext( - &got_tag, &ok, - std::chrono::system_clock::now() + std::chrono::milliseconds(10))) { - case CompletionQueue::GOT_EVENT: { - // Got a regular event, so process it - ClientRpcContext* ctx = ClientRpcContext::detag(got_tag); - // Proceed while holding a lock to make sure that - // this thread isn't supposed to shut down - std::lock_guard l(shutdown_state_[thread_idx]->mutex); - if (shutdown_state_[thread_idx]->shutdown) { - delete ctx; - return true; - } else if (!ctx->RunNextState(ok, entry)) { - // The RPC and callback are done, so clone the ctx - // and kickstart the new one - ctx->StartNewClone(cli_cqs_[thread_idx].get()); - // delete the old version - delete ctx; - } + if (cli_cqs_[thread_idx]->Next(&got_tag, &ok)) { + // Got a regular event, so process it + ClientRpcContext* ctx = ClientRpcContext::detag(got_tag); + // Proceed while holding a lock to make sure that + // this thread isn't supposed to shut down + std::lock_guard l(shutdown_state_[thread_idx]->mutex); + if (shutdown_state_[thread_idx]->shutdown) { + delete ctx; return true; + } else if (!ctx->RunNextState(ok, entry)) { + // The RPC and callback are done, so clone the ctx + // and kickstart the new one + ctx->StartNewClone(cli_cqs_[thread_idx].get()); + // delete the old version + delete ctx; } - case CompletionQueue::TIMEOUT: { - std::lock_guard l(shutdown_state_[thread_idx]->mutex); - if (shutdown_state_[thread_idx]->shutdown) { - return true; - } - return true; - } - case CompletionQueue::SHUTDOWN: // queue is shutting down, so we must be - // done - return true; + return true; + } else { + // queue is shutting down, so we must be done + return true; } - GPR_UNREACHABLE_CODE(return true); } std::vector> cli_cqs_; diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f..e50d90648a 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1026,6 +1026,8 @@ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_generic.h \ src/core/lib/iomgr/timer_heap.c \ src/core/lib/iomgr/timer_heap.h \ +src/core/lib/iomgr/timer_manager.c \ +src/core/lib/iomgr/timer_manager.h \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/timer_uv.h \ src/core/lib/iomgr/udp_server.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index f49c2de76c..24737de859 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1161,6 +1161,8 @@ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_generic.h \ src/core/lib/iomgr/timer_heap.c \ src/core/lib/iomgr/timer_heap.h \ +src/core/lib/iomgr/timer_manager.c \ +src/core/lib/iomgr/timer_manager.h \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/timer_uv.h \ src/core/lib/iomgr/udp_server.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index adaf5481b2..a9f6ac5cbd 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7791,6 +7791,7 @@ "src/core/lib/iomgr/timer.h", "src/core/lib/iomgr/timer_generic.h", "src/core/lib/iomgr/timer_heap.h", + "src/core/lib/iomgr/timer_manager.h", "src/core/lib/iomgr/timer_uv.h", "src/core/lib/iomgr/udp_server.h", "src/core/lib/iomgr/unix_sockets_posix.h", @@ -7978,6 +7979,8 @@ "src/core/lib/iomgr/timer_generic.h", "src/core/lib/iomgr/timer_heap.c", "src/core/lib/iomgr/timer_heap.h", + "src/core/lib/iomgr/timer_manager.c", + "src/core/lib/iomgr/timer_manager.h", "src/core/lib/iomgr/timer_uv.c", "src/core/lib/iomgr/timer_uv.h", "src/core/lib/iomgr/udp_server.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a58..da9816f9a3 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -437,6 +437,7 @@ + @@ -702,6 +703,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc297..924576d35b 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -322,6 +322,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -1031,6 +1034,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651..3e4265bda2 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -431,6 +431,7 @@ + @@ -686,6 +687,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab3..c17d4deca4 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -307,6 +307,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -998,6 +1001,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 71520098a6..bb44a0772c 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -361,6 +361,7 @@ + @@ -641,6 +642,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index de2dfe67e6..35002b1756 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -202,6 +202,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -992,6 +995,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97..b5605a9381 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -256,6 +256,7 @@ + @@ -475,6 +476,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4..95e22922f4 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -259,6 +259,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -746,6 +749,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0bfda72e81..8b59aa82dd 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -351,6 +351,7 @@ + @@ -608,6 +609,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 63c8d7f254..2cec671ebe 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -205,6 +205,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -902,6 +905,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr -- cgit v1.2.3 From d4fc32eacd07a5b9ae9573d25dbee819266d761a Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 2 May 2017 14:15:12 -0700 Subject: New epoll-based polling engine with dedicated poller threads --- BUILD | 82 +- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 1499 ++++++++++++++++++++ src/core/lib/iomgr/ev_epoll_thread_pool_linux.h | 42 + src/core/lib/iomgr/ev_posix.c | 2 + src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/iomgr/pollset_set_test.c | 4 +- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + tools/run_tests/generated/sources_and_headers.json | 3 + tools/run_tests/run_tests.py | 2 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util/grpc_test_util.vcxproj.filters | 6 + .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 6 + 28 files changed, 1672 insertions(+), 37 deletions(-) create mode 100644 src/core/lib/iomgr/ev_epoll_thread_pool_linux.c create mode 100644 src/core/lib/iomgr/ev_epoll_thread_pool_linux.h (limited to 'src/core') diff --git a/BUILD b/BUILD index 7b9e92e29f..2a042fb116 100644 --- a/BUILD +++ b/BUILD @@ -35,8 +35,12 @@ exports_files(["LICENSE"]) package(default_visibility = ["//visibility:public"]) -load("//bazel:grpc_build_system.bzl", "grpc_cc_library", - "grpc_proto_plugin", "grpc_cc_libraries") +load( + "//bazel:grpc_build_system.bzl", + "grpc_cc_library", + "grpc_proto_plugin", + "grpc_cc_libraries", +) # This should be updated along with build.yaml g_stands_for = "gregarious" @@ -55,10 +59,19 @@ grpc_cc_library( ) grpc_cc_libraries( - name_list = ["grpc", "grpc_unsecure",], srcs = [ "src/core/lib/surface/init.c", ], + additional_dep_list = [ + [ + "grpc_secure", + "grpc_resolver_dns_ares", + "grpc_lb_policy_grpclb_secure", + "grpc_transport_chttp2_client_secure", + "grpc_transport_chttp2_server_secure", + ], + [], + ], additional_src_list = [ [ "src/core/plugin_registry/grpc_plugin_registry.c", @@ -69,30 +82,24 @@ grpc_cc_libraries( ], ], language = "c", + name_list = [ + "grpc", + "grpc_unsecure", + ], standalone = True, deps = [ "census", "grpc_base", + "grpc_deadline_filter", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", "grpc_max_age_filter", + "grpc_message_size_filter", "grpc_resolver_dns_native", "grpc_resolver_sockaddr", "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_server_insecure", - "grpc_message_size_filter", - "grpc_deadline_filter", - ], - additional_dep_list = [ - [ - "grpc_secure", - "grpc_resolver_dns_ares", - "grpc_lb_policy_grpclb_secure", - "grpc_transport_chttp2_client_secure", - "grpc_transport_chttp2_server_secure", - ], - [], ], ) @@ -105,9 +112,9 @@ grpc_cc_library( language = "c", deps = [ "grpc_base", + "grpc_http_filters", "grpc_transport_chttp2_client_secure", "grpc_transport_cronet_client_secure", - "grpc_http_filters", ], ) @@ -373,13 +380,13 @@ grpc_cc_library( hdrs = [ "src/core/lib/profiling/timers.h", "src/core/lib/support/arena.h", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", "src/core/lib/support/memory.h", - "src/core/lib/support/atomic.h", - "src/core/lib/support/atomic_with_atm.h", - "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.h", "src/core/lib/support/spinlock.h", @@ -466,6 +473,7 @@ grpc_cc_library( "src/core/lib/iomgr/endpoint_pair_windows.c", "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll_linux.c", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_posix.c", "src/core/lib/iomgr/exec_ctx.c", @@ -588,6 +596,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -791,16 +800,16 @@ grpc_cc_library( grpc_cc_library( name = "grpc_http_filters", - hdrs = [ - "src/core/ext/filters/http/message_compress/message_compress_filter.h", - "src/core/ext/filters/http/client/http_client_filter.h", - "src/core/ext/filters/http/server/http_server_filter.h", - ], srcs = [ - "src/core/ext/filters/http/message_compress/message_compress_filter.c", "src/core/ext/filters/http/client/http_client_filter.c", + "src/core/ext/filters/http/http_filters_plugin.c", + "src/core/ext/filters/http/message_compress/message_compress_filter.c", "src/core/ext/filters/http/server/http_server_filter.c", - "src/core/ext/filters/http/http_filters_plugin.c" + ], + hdrs = [ + "src/core/ext/filters/http/client/http_client_filter.h", + "src/core/ext/filters/http/message_compress/message_compress_filter.h", + "src/core/ext/filters/http/server/http_server_filter.h", ], language = "c", deps = [ @@ -1069,8 +1078,8 @@ grpc_cc_library( language = "c", deps = [ "grpc_base", - "grpc_transport_chttp2_alpn", "grpc_http_filters", + "grpc_transport_chttp2_alpn", ], ) @@ -1226,11 +1235,6 @@ grpc_cc_library( ) grpc_cc_libraries( - name_list = ["grpc++_base", "grpc++_base_unsecure"], - additional_dep_list = [ - ["grpc", ], - ["grpc_unsecure", ], - ], srcs = [ "src/cpp/client/channel_cc.cc", "src/cpp/client/client_context.cc", @@ -1265,7 +1269,7 @@ grpc_cc_libraries( "src/cpp/util/status.cc", "src/cpp/util/string_ref.cc", "src/cpp/util/time_cc.cc", - ], + ], hdrs = [ "src/cpp/client/create_channel_internal.h", "src/cpp/common/channel_filter.h", @@ -1274,8 +1278,16 @@ grpc_cc_libraries( "src/cpp/server/health/health.pb.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/thread_manager/thread_manager.h", - ], + ], + additional_dep_list = [ + ["grpc"], + ["grpc_unsecure"], + ], language = "c++", + name_list = [ + "grpc++_base", + "grpc++_base_unsecure", + ], public_hdrs = [ "include/grpc++/alarm.h", "include/grpc++/channel.h", @@ -1324,7 +1336,7 @@ grpc_cc_libraries( "include/grpc++/support/stub_options.h", "include/grpc++/support/sync_stream.h", "include/grpc++/support/time.h", - ], + ], deps = [ "grpc++_codegen_base", ], diff --git a/CMakeLists.txt b/CMakeLists.txt index 325344a81c..c78dcce13a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -939,6 +939,7 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1265,6 +1266,7 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1576,6 +1578,7 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1832,6 +1835,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2251,6 +2255,7 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2576,6 +2581,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -3345,6 +3351,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c diff --git a/Makefile b/Makefile index 4597b6fe97..e93ac94db7 100644 --- a/Makefile +++ b/Makefile @@ -2922,6 +2922,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3246,6 +3247,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3556,6 +3558,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3784,6 +3787,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4180,6 +4184,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4513,6 +4518,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -5272,6 +5278,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/binding.gyp b/binding.gyp index 582c61282f..2584c6af6b 100644 --- a/binding.gyp +++ b/binding.gyp @@ -673,6 +673,7 @@ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/build.yaml b/build.yaml index 4cfa75cae1..887ca7646f 100644 --- a/build.yaml +++ b/build.yaml @@ -198,6 +198,7 @@ filegroups: - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h - src/core/lib/iomgr/ev_epoll_linux.h + - src/core/lib/iomgr/ev_epoll_thread_pool_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -305,6 +306,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + - src/core/lib/iomgr/ev_epoll_thread_pool_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c - src/core/lib/iomgr/exec_ctx.c diff --git a/config.m4 b/config.m4 index bbd667c9ec..c478f77ba3 100644 --- a/config.m4 +++ b/config.m4 @@ -109,6 +109,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 07755ac727..b233df2feb 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -280,6 +280,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -486,6 +487,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', @@ -741,6 +743,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/grpc.gemspec b/grpc.gemspec index 1cd6d66335..783af00bb6 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -196,6 +196,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -402,6 +403,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) diff --git a/package.xml b/package.xml index e7d67eca18..00dc5aeb26 100644 --- a/package.xml +++ b/package.xml @@ -205,6 +205,7 @@ + @@ -411,6 +412,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c new file mode 100644 index 0000000000..7a9e20b1ea --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -0,0 +1,1499 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ +static int grpc_polling_trace = 0; /* Disabled by default */ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (grpc_polling_trace) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +struct polling_island; + +/******************************************************************************* + * Fd Declarations + */ +struct grpc_fd { + gpr_mu mu; + struct polling_island *pi; + + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Polling island Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define PI_UNREF(exec_ctx, p, r) \ + pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define PI_ADD_REF(p, r) pi_add_ref((p)) +#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ + +/* This is also used as grpc_workqueue (by directly casting it) */ +typedef struct polling_island { + grpc_closure_scheduler workqueue_scheduler; + + gpr_mu mu; + /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + the refcount. + Once the ref count becomes zero, this structure is destroyed which means + we should ensure that there is never a scenario where a PI_ADD_REF() is + racing with a PI_UNREF() that just made the ref_count zero. */ + gpr_atm ref_count; + + /* Number of threads currently polling on this island */ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* The fd of the underlying epoll set */ + int epoll_fd; + + /* The file descriptors in the epoll set */ + /* TODO: sreek - We no longer need this (and since no other structure in this + * polling engine keeps a reference to grpc_fd, we actually no longer need a + * ref count field in FD. Just a flag to say wheter it is orphaned or not */ + size_t fd_cnt; + size_t fd_capacity; + grpc_fd **fds; +} polling_island; + +/******************************************************************************* + * Pollset Declarations + */ +struct grpc_pollset_worker { + gpr_cv kick_cv; + + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset { + gpr_mu mu; + struct polling_island *pi; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ + gpr_atm is_shutdown; +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + void *no_op; +}; + +/***************************************************************************** + * Dedicated polling threads and pollsets - Declarations + */ + +size_t g_num_pollsets = 0; +struct grpc_pollset *g_pollsets = NULL; +gpr_thd_id *g_poller_threads = NULL; + +static void add_fd_to_global_pollset(grpc_fd *fd); +static void init_dedicated_pollsets(); +static void poller_thread_loop(void *arg); +static void start_dedicated_poller_threads(); +static void shutdown_dedicated_poller_threads(); + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Polling island Definitions + */ + +/* The wakeup fd that is used to wake up all threads in a Polling island. This + is useful in the polling island merge operation where we need to wakeup all + the threads currently polling the smaller polling island (so that they can + start polling the new/merged polling island) + + NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the + threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd polling_island_wakeup_fd; + +/* The polling island being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread polling_island *g_current_thread_polling_island; + +/* Forward declaration */ +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void pi_add_ref(polling_island *pi); +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void pi_add_ref_dbg(polling_island *pi, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_add_ref(pi); + gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, old_cnt + 1, reason, file, line); +} + +static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_unref(exec_ctx, pi); + gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_add_ref((polling_island *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_unref(exec_ctx, (polling_island *)workqueue); + } +} +#endif + +static void pi_add_ref(polling_island *pi) { + gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +} + +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { + /* If ref count went to zero, delete the polling island. This deletion is + not done under a lock since once the ref count goes to zero, we are + guaranteed that no one else holds a reference to the polling island (and + that there is no racing pi_add_ref() call either).*/ + if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { + polling_island_delete(exec_ctx, pi); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, + size_t fd_count, bool add_fd_refs, + grpc_error **error) { + int err; + size_t i; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "polling_island_add_fds"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + for (i = 0; i < fd_count; i++) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fds[i]; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); + if (err < 0) { + if (errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + continue; + } + + if (pi->fd_cnt == pi->fd_capacity) { + pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); + pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); + } + + pi->fds[pi->fd_cnt++] = fds[i]; + if (add_fd_refs) { + GRPC_FD_REF(fds[i], "polling_island"); + } + } +} + +/* The caller is expected to hold pi->mu before calling this */ +static void polling_island_add_wakeup_fd_locked(polling_island *pi, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "polling_island_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } + + for (i = 0; i < pi->fd_cnt; i++) { + if (pi->fds[i] == fd) { + pi->fds[i] = pi->fds[--pi->fd_cnt]; + GRPC_FD_UNREF(fd, "polling_island"); + break; + } + } +} + +/* Might return NULL in case of an error */ +static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, + grpc_fd *initial_fd, + grpc_error **error) { + polling_island *pi = NULL; + const char *err_desc = "polling_island_create"; + + *error = GRPC_ERROR_NONE; + + pi = gpr_malloc(sizeof(*pi)); + pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + gpr_mu_init(&pi->mu); + pi->fd_cnt = 0; + pi->fd_capacity = 0; + pi->fds = NULL; + pi->epoll_fd = -1; + + gpr_mu_init(&pi->workqueue_read_mu); + gpr_mpscq_init(&pi->workqueue_items); + gpr_atm_rel_store(&pi->workqueue_item_count, 0); + + gpr_atm_rel_store(&pi->ref_count, 0); + gpr_atm_rel_store(&pi->poller_count, 0); + + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (pi->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); + polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + + if (initial_fd != NULL) { + polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); + } + +done: + if (*error != GRPC_ERROR_NONE) { + polling_island_delete(exec_ctx, pi); + pi = NULL; + } + return pi; +} + +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { + GPR_ASSERT(pi->fd_cnt == 0); + + if (pi->epoll_fd >= 0) { + close(pi->epoll_fd); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); + gpr_mu_destroy(&pi->workqueue_read_mu); + gpr_mpscq_destroy(&pi->workqueue_items); + gpr_mu_destroy(&pi->mu); + grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + + gpr_free(pi->fds); + gpr_free(pi); +} + +static void workqueue_maybe_wakeup(polling_island *pi) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_polling_island == pi); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + } +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + polling_island *pi = (polling_island *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(pi); + } + + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + polling_island *pi = (polling_island *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &pi->workqueue_scheduler; +} + +static grpc_error *polling_island_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + } + + return error; +} + +static void polling_island_global_shutdown() { + grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->mu); + } + + /* Note: It is not really needed to get the new_fd->mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->mu); + new_fd->pi = NULL; + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + add_fd_to_global_pollset(new_fd); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->pi != NULL) { + polling_island *pi = fd->pi; + gpr_mu_lock(&pi->mu); + polling_island_remove_fd_locked(pi, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi->mu); + + unref_pi = fd->pi; + fd->pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + return NULL; +} + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->next = worker->prev = NULL; + gpr_cv_init(&worker->kick_cv); +} + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + gpr_cv_signal(&worker->kick_cv); + return GRPC_ERROR_NONE; +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->pi = NULL; + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; + gpr_atm_no_barrier_store(&pollset->is_shutdown, 0); +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, + grpc_pollset *ps, char *reason) { + if (ps->pi != NULL) { + PI_UNREF(exec_ctx, ps->pi, reason); + } + ps->pi = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->pi to NULL */ + pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying polling island. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, + polling_island *pi) { + if (gpr_mu_trylock(&pi->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); + gpr_mu_unlock(&pi->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(pi); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(pi); + } + } + return false; +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_error **error) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + int timeout_ms = -1; + + GRPC_SCHEDULING_START_BLOCKING_REGION; + // gpr_log(GPR_ERROR, "epoll_wait(%d)..", epoll_fd); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); + /* gpr_log(GPR_ERROR, "epoll_wait(%d) returned: %d (errno: %d - %s)", + epoll_fd, ep_rv, errno, strerror(errno)); */ + + GRPC_SCHEDULING_END_BLOCKING_REGION; + + if (ep_rv < 0) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + gpr_atm_rel_store(&pollset->is_shutdown, 1); + gpr_log(GPR_INFO, "pollset poller: shutdown set"); + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + +static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_error **error) { + int epoll_fd = -1; + polling_island *pi = NULL; + GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + + /* Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. */ + + if (pollset->pi == NULL) { + pollset->pi = polling_island_create(exec_ctx, NULL, error); + if (pollset->pi == NULL) { + GPR_TIMER_END("pollset_work_and_unlock", 0); + return; /* Fatal error. Cannot continue */ + } + + PI_ADD_REF(pollset->pi, "ps"); + GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", + (void *)pollset, (void *)pollset->pi); + } + + pi = pollset->pi; + epoll_fd = pi->epoll_fd; + + /* Add an extra ref so that the island does not get destroyed (which means + the epoll_fd won't be closed) while we are are doing an epoll_wait() on the + epoll_fd */ + PI_ADD_REF(pi, "ps_work"); + gpr_mu_unlock(&pollset->mu); + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, pi)) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); + g_current_thread_polling_island = pi; + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, error); + g_current_thread_polling_island = NULL; + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + } + + /* Before leaving, release the extra ref we added to the polling island. It + is important to use "pi" here (i.e our old copy of pollset->pi + that we got before releasing the polling island lock). This is because + pollset->pi pointer might get udpated in other parts of the + code when there is an island merge while we are doing epoll_wait() above */ + PI_UNREF(exec_ctx, pi, "ps_work"); + + GPR_TIMER_END("pollset_work_and_unlock", 0); +} + +/* pollset->mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + + grpc_pollset_worker worker; + pollset_worker_init(&worker); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + push_front_worker(pollset, &worker); + + gpr_cv_wait(&worker.kick_cv, &pollset->mu, + gpr_convert_clock_type(deadline, GPR_CLOCK_REALTIME)); + /* pollset->mu locked here */ + + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, + grpc_fd *fd) { + GPR_TIMER_BEGIN("pollset_add_fd", 0); + + grpc_error *error = GRPC_ERROR_NONE; + polling_island *pi_new = NULL; + + gpr_mu_lock(&ps->mu); + gpr_mu_lock(&fd->mu); + + /* fd MUST have a NULL polling island */ + GPR_ASSERT(fd->pi == NULL); + + /* Early out if we are trying to add an 'fd' to a 'pollset' but the fd is + * already orphaned */ + if (fd->orphaned) { + gpr_mu_unlock(&ps->mu); + gpr_mu_unlock(&fd->mu); + return; + } + + pi_new = ps->pi; + if (pi_new == NULL) { + /* Unlock before creating a new polling island: the polling island will + create a workqueue which creates a file descriptor, and holding an fd + lock here can eventually cause a loop to appear to TSAN (making it + unhappy). We don't think it's a real loop (there's an epoch point + where that loop possibility disappears), but the advantages of + keeping TSAN happy outweigh any performance advantage we might have + by keeping the lock held. */ + gpr_mu_unlock(&fd->mu); + pi_new = polling_island_create(exec_ctx, fd, &error); + gpr_mu_lock(&fd->mu); + + GRPC_POLLING_TRACE( + "pollset_add_fd: Created new polling island: %p (ps: %p, fd: %d", + (void *)pi_new, (void *)ps, fd->fd); + } else { + gpr_mu_lock(&pi_new->mu); + polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); + gpr_mu_unlock(&pi_new->mu); + + GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", + (void *)pi_new, fd->fd); + } + + PI_ADD_REF(pi_new, "fd"); + fd->pi = pi_new; + + GPR_ASSERT((ps->pi == NULL) || (ps->pi == pi_new)); + if (ps->pi == NULL) { + PI_ADD_REF(pi_new, "pollset"); + ps->pi = pi_new; + } + + gpr_mu_unlock(&ps->mu); + gpr_mu_unlock(&fd->mu); + + GRPC_LOG_IF_ERROR("pollset_add_fd", error); + GPR_TIMER_END("pollset_add_fd", 0); +} + +static void pollset_add_fd_no_op(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + /* Nothing to do */ +} + +/******************************************************************************* + * Pollset-set Definitions + */ +grpc_pollset_set g_dummy_pollset_set; +static grpc_pollset_set *pollset_set_create(void) { + return &g_dummy_pollset_set; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + /* Nothing to do */ +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + shutdown_dedicated_poller_threads(); + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd_no_op, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/***************************************************************************** + * Dedicated polling threads and pollsets - Definitions + */ +static void add_fd_to_global_pollset(grpc_fd *fd) { + size_t idx = ((size_t)rand()) % g_num_pollsets; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + pollset_add_fd(&exec_ctx, &g_pollsets[idx], fd); + grpc_exec_ctx_finish(&exec_ctx); +} + +static void init_dedicated_pollsets() { + gpr_mu *temp_mu; + + g_num_pollsets = (size_t)gpr_cpu_num_cores(); + g_pollsets = (grpc_pollset *)malloc(g_num_pollsets * sizeof(grpc_pollset)); + for (size_t i = 0; i < g_num_pollsets; i++) { + pollset_init(&g_pollsets[i], &temp_mu); + } + + gpr_log(GPR_INFO, "Created %ld pollsets", g_num_pollsets); +} + +static void poller_thread_loop(void *arg) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_error *error = GRPC_ERROR_NONE; + grpc_pollset *ps = (grpc_pollset *)arg; + + while (!gpr_atm_acq_load(&ps->is_shutdown)) { + gpr_mu_lock(&ps->mu); + pollset_work_and_unlock(&exec_ctx, ps, &error); + grpc_exec_ctx_flush(&exec_ctx); + } + + grpc_exec_ctx_finish(&exec_ctx); +} + +/* g_pollsets MUST be initialized before calling this */ +static void start_dedicated_poller_threads() { + GPR_ASSERT(g_pollsets); + gpr_log(GPR_ERROR, "Starting poller threads"); + + /* One thread per pollset */ + g_poller_threads = (gpr_thd_id *)malloc(g_num_pollsets * sizeof(gpr_thd_id)); + gpr_thd_options options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&options); + + for (size_t i = 0; i < g_num_pollsets; i++) { + gpr_thd_new(&g_poller_threads[i], poller_thread_loop, + (void *)&g_pollsets[i], &options); + } +} + +static void shutdown_dedicated_poller_threads() { + GPR_ASSERT(g_poller_threads); + grpc_error *error = GRPC_ERROR_NONE; + + gpr_log(GPR_INFO, "Shutting down pollers"); + + for (size_t i = 0; i < g_num_pollsets; i++) { + gpr_mu_lock(&g_pollsets[i].mu); + polling_island *pi = g_pollsets[i].pi; + GPR_ASSERT(pi); + gpr_mu_lock(&pi->mu); + polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); + gpr_mu_unlock(&pi->mu); + } + + for (size_t i = 0; i < g_num_pollsets; i++) { + gpr_thd_join(g_poller_threads[i]); + } +} + +/****************************************************************************/ + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + init_dedicated_pollsets(); + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + /* TODO (sreek): Maynot be a good idea to start threads here (especially if + * this engine doesn't get picked. Consider introducing an engine_init + * function in the vtable */ + start_dedicated_poller_threads(); + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h new file mode 100644 index 0000000000..6743dcfe3f --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de8..4742b2e571 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,6 +45,7 @@ #include #include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -66,6 +67,7 @@ typedef struct { static const event_engine_factory g_factories[] = { {"epoll", grpc_init_epoll_linux}, + {"epoll-threadpool", grpc_init_epoll_thread_pool_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d2a570cc87..b249d2ee11 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -98,6 +98,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 3a9d459579..89854407b1 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -449,7 +449,9 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_iomgr_init(); - if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { + if (poll_strategy != NULL && + (strcmp(poll_strategy, "epoll") == 0 || + strcmp(poll_strategy, "epoll-threadpool") == 0)) { pollset_set_test_basic(); pollset_set_test_dup_fds(); pollset_set_test_empty_pollset(); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f..13543edb71 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -941,6 +941,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index f49c2de76c..de7a7e9c82 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1076,6 +1076,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index adaf5481b2..0bc4bd627e 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7751,6 +7751,7 @@ "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -7893,6 +7894,8 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.c", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4da2ba4c3b..2cadb7f811 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -72,7 +72,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { _POLLING_STRATEGIES = { - 'linux': ['epoll', 'poll', 'poll-cv'] + 'linux': ['epoll', 'epoll-threadpool', 'poll', 'poll-cv'] } diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a58..37d151b78a 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -397,6 +397,7 @@ + @@ -612,6 +613,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc297..16b3326d81 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -187,6 +187,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -911,6 +914,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651..596893ed45 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -391,6 +391,7 @@ + @@ -596,6 +597,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab3..6e8e91fb55 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -172,6 +172,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -878,6 +881,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 71520098a6..da34b36b6f 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -321,6 +321,7 @@ + @@ -551,6 +552,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index de2dfe67e6..d113dc9840 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -67,6 +67,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -872,6 +875,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97..2c8cfa7cad 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -216,6 +216,7 @@ + @@ -385,6 +386,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4..3e68ddc388 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -124,6 +124,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -626,6 +629,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0bfda72e81..d67c6c3d57 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -311,6 +311,7 @@ + @@ -518,6 +519,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 63c8d7f254..9526515a44 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -70,6 +70,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -782,6 +785,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr -- cgit v1.2.3 From a03edfd2855779a672c396efe3d6c39edd390cf3 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 2 May 2017 16:26:24 -0700 Subject: Make sure dedicated pollsets have polling islands. Simplies a lot of code. Fix init/shutdown --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 135 +++++++++++++----------- 1 file changed, 72 insertions(+), 63 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 7a9e20b1ea..c91164a629 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -221,7 +221,8 @@ struct grpc_pollset *g_pollsets = NULL; gpr_thd_id *g_poller_threads = NULL; static void add_fd_to_global_pollset(grpc_fd *fd); -static void init_dedicated_pollsets(); +static bool init_dedicated_pollsets(); +static void shutdown_dedicated_pollsets(); static void poller_thread_loop(void *arg); static void start_dedicated_poller_threads(); static void shutdown_dedicated_poller_threads(); @@ -258,7 +259,7 @@ static grpc_wakeup_fd polling_island_wakeup_fd; static __thread polling_island *g_current_thread_polling_island; /* Forward declaration */ -static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void polling_island_delete(polling_island *pi); static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error); @@ -337,7 +338,7 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { guaranteed that no one else holds a reference to the polling island (and that there is no racing pi_add_ref() call either).*/ if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { - polling_island_delete(exec_ctx, pi); + polling_island_delete(pi); } } @@ -442,8 +443,7 @@ static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, } /* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, - grpc_fd *initial_fd, +static polling_island *polling_island_create(grpc_fd *initial_fd, grpc_error **error) { polling_island *pi = NULL; const char *err_desc = "polling_island_create"; @@ -486,13 +486,13 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, done: if (*error != GRPC_ERROR_NONE) { - polling_island_delete(exec_ctx, pi); + polling_island_delete(pi); pi = NULL; } return pi; } -static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { +static void polling_island_delete(polling_island *pi) { GPR_ASSERT(pi->fd_cnt == 0); if (pi->epoll_fd >= 0) { @@ -807,9 +807,7 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - return NULL; -} +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; } /******************************************************************************* * Pollset Definitions @@ -1042,7 +1040,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; + const char *err_desc = "pollset_do_epoll_pwait"; int timeout_ms = -1; @@ -1095,27 +1093,29 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, } } +static void pollset_add_polling_island(grpc_pollset *ps, grpc_error **error) { + GPR_ASSERT(ps->pi == NULL); + ps->pi = polling_island_create(NULL, error); + if (ps->pi) { + PI_ADD_REF(ps->pi, "ps"); + GRPC_POLLING_TRACE( + "pollset_add_polling_island: pollset: %p created new pi: %p", + (void *)ps, (void *)ps->pi); + } +} + +/* Note: Make sure the pollset has a polling island (i.e pollset->pi != NULL) + * before calling this */ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_error **error) { + GPR_ASSERT(pollset->pi); + int epoll_fd = -1; polling_island *pi = NULL; GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the polling island. */ - - if (pollset->pi == NULL) { - pollset->pi = polling_island_create(exec_ctx, NULL, error); - if (pollset->pi == NULL) { - GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. Cannot continue */ - } - - PI_ADD_REF(pollset->pi, "ps"); - GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", - (void *)pollset, (void *)pollset->pi); - } - pi = pollset->pi; epoll_fd = pi->epoll_fd; @@ -1212,17 +1212,19 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, grpc_fd *fd) { + /* fd MUST have a NULL polling island and pollset MUST have a non-NULL polling + * island*/ + GPR_ASSERT(fd->pi == NULL); + GPR_ASSERT(ps->pi); + GPR_TIMER_BEGIN("pollset_add_fd", 0); grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi_new = NULL; + polling_island *pi = NULL; gpr_mu_lock(&ps->mu); gpr_mu_lock(&fd->mu); - /* fd MUST have a NULL polling island */ - GPR_ASSERT(fd->pi == NULL); - /* Early out if we are trying to add an 'fd' to a 'pollset' but the fd is * already orphaned */ if (fd->orphaned) { @@ -1231,39 +1233,16 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, return; } - pi_new = ps->pi; - if (pi_new == NULL) { - /* Unlock before creating a new polling island: the polling island will - create a workqueue which creates a file descriptor, and holding an fd - lock here can eventually cause a loop to appear to TSAN (making it - unhappy). We don't think it's a real loop (there's an epoch point - where that loop possibility disappears), but the advantages of - keeping TSAN happy outweigh any performance advantage we might have - by keeping the lock held. */ - gpr_mu_unlock(&fd->mu); - pi_new = polling_island_create(exec_ctx, fd, &error); - gpr_mu_lock(&fd->mu); + pi = ps->pi; + gpr_mu_lock(&pi->mu); + polling_island_add_fds_locked(pi, &fd, 1, true, &error); + gpr_mu_unlock(&pi->mu); - GRPC_POLLING_TRACE( - "pollset_add_fd: Created new polling island: %p (ps: %p, fd: %d", - (void *)pi_new, (void *)ps, fd->fd); - } else { - gpr_mu_lock(&pi_new->mu); - polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); - gpr_mu_unlock(&pi_new->mu); + PI_ADD_REF(pi, "fd"); + fd->pi = pi; - GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", - (void *)pi_new, fd->fd); - } - - PI_ADD_REF(pi_new, "fd"); - fd->pi = pi_new; - - GPR_ASSERT((ps->pi == NULL) || (ps->pi == pi_new)); - if (ps->pi == NULL) { - PI_ADD_REF(pi_new, "pollset"); - ps->pi = pi_new; - } + GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", (void *)pi, + fd->fd); gpr_mu_unlock(&ps->mu); gpr_mu_unlock(&fd->mu); @@ -1328,6 +1307,7 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { shutdown_dedicated_poller_threads(); + shutdown_dedicated_pollsets(); fd_global_shutdown(); pollset_global_shutdown(); polling_island_global_shutdown(); @@ -1381,16 +1361,38 @@ static void add_fd_to_global_pollset(grpc_fd *fd) { grpc_exec_ctx_finish(&exec_ctx); } -static void init_dedicated_pollsets() { +static bool init_dedicated_pollsets() { gpr_mu *temp_mu; + grpc_error *error = GRPC_ERROR_NONE; + bool is_success = true; g_num_pollsets = (size_t)gpr_cpu_num_cores(); g_pollsets = (grpc_pollset *)malloc(g_num_pollsets * sizeof(grpc_pollset)); + for (size_t i = 0; i < g_num_pollsets; i++) { pollset_init(&g_pollsets[i], &temp_mu); + pollset_add_polling_island(&g_pollsets[i], &error); + if (g_pollsets[i].pi == NULL) { + is_success = false; + break; + } + } + + if (is_success) { + gpr_log(GPR_INFO, "Created %ld dedicated pollsets", g_num_pollsets); + } else { + shutdown_dedicated_pollsets(); } - gpr_log(GPR_INFO, "Created %ld pollsets", g_num_pollsets); + GRPC_LOG_IF_ERROR("init_dedicated_pollsets", error); + return is_success; +} + +static void shutdown_dedicated_pollsets() { + if (g_pollsets) { + gpr_free(g_pollsets); + g_pollsets = NULL; + } } static void poller_thread_loop(void *arg) { @@ -1405,6 +1407,7 @@ static void poller_thread_loop(void *arg) { } grpc_exec_ctx_finish(&exec_ctx); + GRPC_LOG_IF_ERROR("poller_thread_loop", error); } /* g_pollsets MUST be initialized before calling this */ @@ -1425,6 +1428,7 @@ static void start_dedicated_poller_threads() { static void shutdown_dedicated_poller_threads() { GPR_ASSERT(g_poller_threads); + GPR_ASSERT(g_pollsets); grpc_error *error = GRPC_ERROR_NONE; gpr_log(GPR_INFO, "Shutting down pollers"); @@ -1432,7 +1436,6 @@ static void shutdown_dedicated_poller_threads() { for (size_t i = 0; i < g_num_pollsets; i++) { gpr_mu_lock(&g_pollsets[i].mu); polling_island *pi = g_pollsets[i].pi; - GPR_ASSERT(pi); gpr_mu_lock(&pi->mu); polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); gpr_mu_unlock(&pi->mu); @@ -1441,6 +1444,10 @@ static void shutdown_dedicated_poller_threads() { for (size_t i = 0; i < g_num_pollsets; i++) { gpr_thd_join(g_poller_threads[i]); } + + GRPC_LOG_IF_ERROR("shutdown_dedicated_poller_threads", error); + gpr_free(g_poller_threads); + g_poller_threads = NULL; } /****************************************************************************/ @@ -1469,7 +1476,9 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - init_dedicated_pollsets(); + if (!init_dedicated_pollsets()) { + return NULL; + } fd_global_init(); -- cgit v1.2.3 From 50f85f726b4f011f491e90f428f2d116947224e3 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 2 May 2017 19:44:28 -0700 Subject: More simplifications --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 402 ++++++++---------------- src/core/lib/iomgr/ev_epoll_thread_pool_linux.h | 2 +- 2 files changed, 140 insertions(+), 264 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index c91164a629..055b31331d 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -104,28 +104,9 @@ struct grpc_fd { struct grpc_fd *freelist_next; grpc_closure *on_done_closure; - /* The pollset that last noticed that the fd is readable. The actual type - * stored in this is (grpc_pollset *) */ - gpr_atm read_notifier_pollset; - grpc_iomgr_object iomgr_object; }; -/* Reference counting for fds */ -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd *fd); -static void fd_unref(grpc_fd *fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - static void fd_global_init(void); static void fd_global_shutdown(void); @@ -150,12 +131,11 @@ static void fd_global_shutdown(void); typedef struct polling_island { grpc_closure_scheduler workqueue_scheduler; - gpr_mu mu; /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement - the refcount. - Once the ref count becomes zero, this structure is destroyed which means - we should ensure that there is never a scenario where a PI_ADD_REF() is - racing with a PI_UNREF() that just made the ref_count zero. */ + the refcount. Once the ref count becomes zero, this structure is destroyed + which means we should ensure that there is never a scenario where a + PI_ADD_REF() is racing with a PI_UNREF() that just made the ref_count + zero. */ gpr_atm ref_count; /* Number of threads currently polling on this island */ @@ -170,16 +150,11 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; + /* Is the polling island shutdown */ + gpr_atm is_shutdown; + /* The fd of the underlying epoll set */ int epoll_fd; - - /* The file descriptors in the epoll set */ - /* TODO: sreek - We no longer need this (and since no other structure in this - * polling engine keeps a reference to grpc_fd, we actually no longer need a - * ref count field in FD. Just a flag to say wheter it is orphaned or not */ - size_t fd_cnt; - size_t fd_capacity; - grpc_fd **fds; } polling_island; /******************************************************************************* @@ -202,7 +177,6 @@ struct grpc_pollset { bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ grpc_closure *shutdown_done; /* Called after after shutdown is complete */ - gpr_atm is_shutdown; }; /******************************************************************************* @@ -216,13 +190,19 @@ struct grpc_pollset_set { * Dedicated polling threads and pollsets - Declarations */ -size_t g_num_pollsets = 0; -struct grpc_pollset *g_pollsets = NULL; +size_t g_num_pi = 1; +struct polling_island **g_polling_islands = NULL; +size_t g_num_threads_per_pi = 1; gpr_thd_id *g_poller_threads = NULL; -static void add_fd_to_global_pollset(grpc_fd *fd); -static bool init_dedicated_pollsets(); -static void shutdown_dedicated_pollsets(); +/* Used as read-notifier pollsets for fds. We won't be using read notifier + * pollsets with this polling engine. So it does not matter what pollset we + * return */ +grpc_pollset g_read_notifier; + +static void add_fd_to_dedicated_pi(grpc_fd *fd); +static bool init_dedicated_polling_islands(); +static void shutdown_dedicated_polling_islands(); static void poller_thread_loop(void *arg); static void start_dedicated_poller_threads(); static void shutdown_dedicated_poller_threads(); @@ -342,51 +322,31 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { } } -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, - size_t fd_count, bool add_fd_refs, - grpc_error **error) { +static void polling_island_add_fd_locked(polling_island *pi, grpc_fd *fd, + grpc_error **error) { int err; - size_t i; struct epoll_event ev; char *err_msg; - const char *err_desc = "polling_island_add_fds"; + const char *err_desc = "polling_island_add_fd_locked"; #ifdef GRPC_TSAN /* See the definition of g_epoll_sync for more context */ gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); #endif /* defined(GRPC_TSAN) */ - for (i = 0; i < fd_count; i++) { - ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); - ev.data.ptr = fds[i]; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); - if (err < 0) { - if (errno != EEXIST) { - gpr_asprintf( - &err_msg, - "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", - pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - continue; - } - - if (pi->fd_cnt == pi->fd_capacity) { - pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); - pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); - } - - pi->fds[pi->fd_cnt++] = fds[i]; - if (add_fd_refs) { - GRPC_FD_REF(fds[i], "polling_island"); - } + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); } } -/* The caller is expected to hold pi->mu before calling this */ static void polling_island_add_wakeup_fd_locked(polling_island *pi, grpc_wakeup_fd *wakeup_fd, grpc_error **error) { @@ -410,12 +370,9 @@ static void polling_island_add_wakeup_fd_locked(polling_island *pi, } } -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, - bool is_fd_closed, - grpc_error **error) { +static void polling_island_remove_fd(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, grpc_error **error) { int err; - size_t i; char *err_msg; const char *err_desc = "polling_island_remove_fd"; @@ -432,19 +389,10 @@ static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, gpr_free(err_msg); } } - - for (i = 0; i < pi->fd_cnt; i++) { - if (pi->fds[i] == fd) { - pi->fds[i] = pi->fds[--pi->fd_cnt]; - GRPC_FD_UNREF(fd, "polling_island"); - break; - } - } } /* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_fd *initial_fd, - grpc_error **error) { +static polling_island *polling_island_create(grpc_error **error) { polling_island *pi = NULL; const char *err_desc = "polling_island_create"; @@ -452,10 +400,6 @@ static polling_island *polling_island_create(grpc_fd *initial_fd, pi = gpr_malloc(sizeof(*pi)); pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - gpr_mu_init(&pi->mu); - pi->fd_cnt = 0; - pi->fd_capacity = 0; - pi->fds = NULL; pi->epoll_fd = -1; gpr_mu_init(&pi->workqueue_read_mu); @@ -465,6 +409,8 @@ static polling_island *polling_island_create(grpc_fd *initial_fd, gpr_atm_rel_store(&pi->ref_count, 0); gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&pi->is_shutdown, false); + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), err_desc)) { goto done; @@ -480,10 +426,6 @@ static polling_island *polling_island_create(grpc_fd *initial_fd, polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); - if (initial_fd != NULL) { - polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); - } - done: if (*error != GRPC_ERROR_NONE) { polling_island_delete(pi); @@ -493,18 +435,15 @@ done: } static void polling_island_delete(polling_island *pi) { - GPR_ASSERT(pi->fd_cnt == 0); - if (pi->epoll_fd >= 0) { close(pi->epoll_fd); } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); gpr_mu_destroy(&pi->workqueue_read_mu); gpr_mpscq_destroy(&pi->workqueue_items); - gpr_mu_destroy(&pi->mu); grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); - gpr_free(pi->fds); gpr_free(pi); } @@ -590,6 +529,7 @@ static void polling_island_global_shutdown() { static grpc_fd *fd_freelist = NULL; static gpr_mu fd_freelist_mu; +// #define GRPC_FD_REF_COUNT_DEBUG #ifdef GRPC_FD_REF_COUNT_DEBUG #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) #define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) @@ -634,22 +574,6 @@ static void unref_by(grpc_fd *fd, int n) { } } -/* Increment refcount by two to avoid changing the orphan bit */ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } -static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } -#endif - static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } static void fd_global_shutdown(void) { @@ -690,7 +614,6 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); - gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -704,7 +627,9 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); #endif gpr_free(fd_name); - add_fd_to_global_pollset(new_fd); + + /* Associate the fd with one of the dedicated pi */ + add_fd_to_dedicated_pi(new_fd); return new_fd; } @@ -744,20 +669,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, to be alive (and not added to freelist) until the end of this function */ REF_BY(fd, 1, reason); - /* Remove the fd from the polling island: - - Get a lock on the latest polling island (i.e the last island in the - linked list pointed by fd->pi). This is the island that - would actually contain the fd - - Remove the fd from the latest polling island - - Unlock the latest polling island - - Set fd->pi to NULL (but remove the ref on the polling island - before doing this.) */ + /* Remove the fd from the polling island */ if (fd->pi != NULL) { polling_island *pi = fd->pi; - gpr_mu_lock(&pi->mu); - polling_island_remove_fd_locked(pi, fd, is_fd_closed, &error); - gpr_mu_unlock(&pi->mu); - + polling_island_remove_fd(pi, fd, is_fd_closed, &error); unref_pi = fd->pi; fd->pi = NULL; } @@ -777,10 +692,11 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_ERROR_UNREF(error); } +/* This polling engine doesn't really need the read notifier functionality. So + * it just returns a dummy read notifier pollset */ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); - return (grpc_pollset *)notifier; + return &g_read_notifier; } static bool fd_is_shutdown(grpc_fd *fd) { @@ -812,6 +728,7 @@ static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; } /******************************************************************************* * Pollset Definitions */ +/* TODO: sreek - Not needed anymore */ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); @@ -938,20 +855,10 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->shutting_down = false; pollset->finish_shutdown_called = false; pollset->shutdown_done = NULL; - gpr_atm_no_barrier_store(&pollset->is_shutdown, 0); } -static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_pollset *notifier) { +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->read_closure); - - /* Note, it is possible that fd_become_readable might be called twice with - different 'notifier's when an fd becomes readable and it is in two epoll - sets (This can happen briefly during polling island merges). In such cases - it does not really matter which notifer is set as the read_notifier_pollset - (They would both point to the same polling island anyway) */ - /* Use release store to match with acquire load in fd_get_read_notifier */ - gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); } static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { @@ -1034,22 +941,17 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, } #define GRPC_EPOLL_MAX_EVENTS 100 -static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, - grpc_pollset *pollset, polling_island *pi, - grpc_error **error) { +static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, + polling_island *pi, grpc_error **error) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; - const char *err_desc = "pollset_do_epoll_pwait"; + const char *err_desc = "do_epoll_wait"; int timeout_ms = -1; GRPC_SCHEDULING_START_BLOCKING_REGION; - // gpr_log(GPR_ERROR, "epoll_wait(%d)..", epoll_fd); ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); - /* gpr_log(GPR_ERROR, "epoll_wait(%d) returned: %d (errno: %d - %s)", - epoll_fd, ep_rv, errno, strerror(errno)); */ - GRPC_SCHEDULING_END_BLOCKING_REGION; if (ep_rv < 0) { @@ -1076,7 +978,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, err_desc); maybe_do_workqueue_work(exec_ctx, pi); } else if (data_ptr == &polling_island_wakeup_fd) { - gpr_atm_rel_store(&pollset->is_shutdown, 1); + gpr_atm_rel_store(&pi->is_shutdown, 1); gpr_log(GPR_INFO, "pollset poller: shutdown set"); } else { grpc_fd *fd = data_ptr; @@ -1084,7 +986,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); int write_ev = ep_ev[i].events & EPOLLOUT; if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); + fd_become_readable(exec_ctx, fd); } if (write_ev || cancel) { fd_become_writable(exec_ctx, fd); @@ -1093,37 +995,19 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, } } -static void pollset_add_polling_island(grpc_pollset *ps, grpc_error **error) { - GPR_ASSERT(ps->pi == NULL); - ps->pi = polling_island_create(NULL, error); - if (ps->pi) { - PI_ADD_REF(ps->pi, "ps"); - GRPC_POLLING_TRACE( - "pollset_add_polling_island: pollset: %p created new pi: %p", - (void *)ps, (void *)ps->pi); - } -} - -/* Note: Make sure the pollset has a polling island (i.e pollset->pi != NULL) - * before calling this */ -static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, grpc_error **error) { - GPR_ASSERT(pollset->pi); - +static void polling_island_work(grpc_exec_ctx *exec_ctx, polling_island *pi, + grpc_error **error) { int epoll_fd = -1; - polling_island *pi = NULL; - GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + GPR_TIMER_BEGIN("polling_island_work", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the polling island. */ - pi = pollset->pi; epoll_fd = pi->epoll_fd; /* Add an extra ref so that the island does not get destroyed (which means the epoll_fd won't be closed) while we are are doing an epoll_wait() on the epoll_fd */ PI_ADD_REF(pi, "ps_work"); - gpr_mu_unlock(&pollset->mu); /* If we get some workqueue work to do, it might end up completing an item on the completion queue, so there's no need to poll... so we skip that and @@ -1131,7 +1015,9 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, if (!maybe_do_workqueue_work(exec_ctx, pi)) { gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); g_current_thread_polling_island = pi; - pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, error); + + do_epoll_wait(exec_ctx, epoll_fd, pi, error); + g_current_thread_polling_island = NULL; gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); } @@ -1143,7 +1029,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, code when there is an island merge while we are doing epoll_wait() above */ PI_UNREF(exec_ctx, pi, "ps_work"); - GPR_TIMER_END("pollset_work_and_unlock", 0); + GPR_TIMER_END("polling_island_work", 0); } /* pollset->mu lock must be held by the caller before calling this. @@ -1210,49 +1096,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - /* fd MUST have a NULL polling island and pollset MUST have a non-NULL polling - * island*/ - GPR_ASSERT(fd->pi == NULL); - GPR_ASSERT(ps->pi); - - GPR_TIMER_BEGIN("pollset_add_fd", 0); - - grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi = NULL; - - gpr_mu_lock(&ps->mu); - gpr_mu_lock(&fd->mu); - - /* Early out if we are trying to add an 'fd' to a 'pollset' but the fd is - * already orphaned */ - if (fd->orphaned) { - gpr_mu_unlock(&ps->mu); - gpr_mu_unlock(&fd->mu); - return; - } - - pi = ps->pi; - gpr_mu_lock(&pi->mu); - polling_island_add_fds_locked(pi, &fd, 1, true, &error); - gpr_mu_unlock(&pi->mu); - - PI_ADD_REF(pi, "fd"); - fd->pi = pi; - - GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", (void *)pi, - fd->fd); - - gpr_mu_unlock(&ps->mu); - gpr_mu_unlock(&fd->mu); - - GRPC_LOG_IF_ERROR("pollset_add_fd", error); - GPR_TIMER_END("pollset_add_fd", 0); -} - -static void pollset_add_fd_no_op(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { /* Nothing to do */ } @@ -1307,10 +1152,11 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { shutdown_dedicated_poller_threads(); - shutdown_dedicated_pollsets(); + shutdown_dedicated_polling_islands(); fd_global_shutdown(); pollset_global_shutdown(); polling_island_global_shutdown(); + gpr_log(GPR_INFO, "ev-epoll-threadpool engine shutdown complete"); } static const grpc_event_engine_vtable vtable = { @@ -1331,7 +1177,7 @@ static const grpc_event_engine_vtable vtable = { .pollset_destroy = pollset_destroy, .pollset_work = pollset_work, .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd_no_op, + .pollset_add_fd = pollset_add_fd, .pollset_set_create = pollset_set_create, .pollset_set_destroy = pollset_set_destroy, @@ -1354,55 +1200,85 @@ static const grpc_event_engine_vtable vtable = { /***************************************************************************** * Dedicated polling threads and pollsets - Definitions */ -static void add_fd_to_global_pollset(grpc_fd *fd) { - size_t idx = ((size_t)rand()) % g_num_pollsets; - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - pollset_add_fd(&exec_ctx, &g_pollsets[idx], fd); - grpc_exec_ctx_finish(&exec_ctx); +static void add_fd_to_dedicated_pi(grpc_fd *fd) { + GPR_ASSERT(fd->pi == NULL); + GPR_TIMER_BEGIN("add_fd_to_dedicated_pi", 0); + + grpc_error *error = GRPC_ERROR_NONE; + size_t idx = ((size_t)rand()) % g_num_pi; + polling_island *pi = g_polling_islands[idx]; + + gpr_mu_lock(&fd->mu); + + if (fd->orphaned) { + gpr_mu_unlock(&fd->mu); + return; /* Early out */ + } + + polling_island_add_fd_locked(pi, fd, &error); + PI_ADD_REF(pi, "fd"); + fd->pi = pi; + + GRPC_POLLING_TRACE("add_fd_to_dedicated_pi (fd: %d, pi idx = %ld)", fd->fd, + idx); + gpr_mu_unlock(&fd->mu); + + GRPC_LOG_IF_ERROR("add_fd_to_dedicated_pi", error); + GPR_TIMER_END("add_fd_to_dedicated_pi", 0); } -static bool init_dedicated_pollsets() { - gpr_mu *temp_mu; +static bool init_dedicated_polling_islands() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; - g_num_pollsets = (size_t)gpr_cpu_num_cores(); - g_pollsets = (grpc_pollset *)malloc(g_num_pollsets * sizeof(grpc_pollset)); + g_polling_islands = + (polling_island **)malloc(g_num_pi * sizeof(polling_island *)); - for (size_t i = 0; i < g_num_pollsets; i++) { - pollset_init(&g_pollsets[i], &temp_mu); - pollset_add_polling_island(&g_pollsets[i], &error); - if (g_pollsets[i].pi == NULL) { + for (size_t i = 0; i < g_num_pi; i++) { + g_polling_islands[i] = polling_island_create(&error); + if (g_polling_islands[i] == NULL) { + gpr_log(GPR_ERROR, "Error in creating a dedicated polling island"); + g_num_pi = i; /* Helps cleanup */ + shutdown_dedicated_polling_islands(); is_success = false; - break; + goto done; } - } - if (is_success) { - gpr_log(GPR_INFO, "Created %ld dedicated pollsets", g_num_pollsets); - } else { - shutdown_dedicated_pollsets(); + PI_ADD_REF(g_polling_islands[i], "init_dedicated_polling_islands"); } - GRPC_LOG_IF_ERROR("init_dedicated_pollsets", error); + gpr_mu *mu; + pollset_init(&g_read_notifier, &mu); + +done: + GRPC_LOG_IF_ERROR("init_dedicated_polling_islands", error); return is_success; } -static void shutdown_dedicated_pollsets() { - if (g_pollsets) { - gpr_free(g_pollsets); - g_pollsets = NULL; +static void shutdown_dedicated_polling_islands() { + if (!g_polling_islands) { + return; } + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + for (size_t i = 0; i < g_num_pi; i++) { + PI_UNREF(&exec_ctx, g_polling_islands[i], + "shutdown_dedicated_polling_islands"); + } + grpc_exec_ctx_finish(&exec_ctx); + + gpr_free(g_polling_islands); + g_polling_islands = NULL; + pollset_destroy(&g_read_notifier); } static void poller_thread_loop(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_error *error = GRPC_ERROR_NONE; - grpc_pollset *ps = (grpc_pollset *)arg; + polling_island *pi = (polling_island *)arg; - while (!gpr_atm_acq_load(&ps->is_shutdown)) { - gpr_mu_lock(&ps->mu); - pollset_work_and_unlock(&exec_ctx, ps, &error); + while (!gpr_atm_acq_load(&pi->is_shutdown)) { + polling_island_work(&exec_ctx, pi, &error); grpc_exec_ctx_flush(&exec_ctx); } @@ -1410,41 +1286,41 @@ static void poller_thread_loop(void *arg) { GRPC_LOG_IF_ERROR("poller_thread_loop", error); } -/* g_pollsets MUST be initialized before calling this */ +/* g_polling_islands MUST be initialized before calling this */ static void start_dedicated_poller_threads() { - GPR_ASSERT(g_pollsets); - gpr_log(GPR_ERROR, "Starting poller threads"); + GPR_ASSERT(g_polling_islands); + + gpr_log(GPR_INFO, "Starting poller threads"); /* One thread per pollset */ - g_poller_threads = (gpr_thd_id *)malloc(g_num_pollsets * sizeof(gpr_thd_id)); + g_poller_threads = (gpr_thd_id *)malloc(g_num_pi * sizeof(gpr_thd_id)); gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); - for (size_t i = 0; i < g_num_pollsets; i++) { + for (size_t i = 0; i < g_num_pi; i++) { gpr_thd_new(&g_poller_threads[i], poller_thread_loop, - (void *)&g_pollsets[i], &options); + (void *)g_polling_islands[i], &options); } } static void shutdown_dedicated_poller_threads() { GPR_ASSERT(g_poller_threads); - GPR_ASSERT(g_pollsets); + GPR_ASSERT(g_polling_islands); grpc_error *error = GRPC_ERROR_NONE; gpr_log(GPR_INFO, "Shutting down pollers"); - for (size_t i = 0; i < g_num_pollsets; i++) { - gpr_mu_lock(&g_pollsets[i].mu); - polling_island *pi = g_pollsets[i].pi; - gpr_mu_lock(&pi->mu); + polling_island *pi = NULL; + for (size_t i = 0; i < g_num_pi; i++) { + pi = g_polling_islands[i]; polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); - gpr_mu_unlock(&pi->mu); } - for (size_t i = 0; i < g_num_pollsets; i++) { + for (size_t i = 0; i < g_num_pi; i++) { gpr_thd_join(g_poller_threads[i]); } + gpr_log(GPR_ERROR, "polling island delete called"); GRPC_LOG_IF_ERROR("shutdown_dedicated_poller_threads", error); gpr_free(g_poller_threads); g_poller_threads = NULL; @@ -1476,10 +1352,6 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!init_dedicated_pollsets()) { - return NULL; - } - fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { @@ -1491,6 +1363,10 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } + if (!init_dedicated_polling_islands()) { + return NULL; + } + /* TODO (sreek): Maynot be a good idea to start threads here (especially if * this engine doesn't get picked. Consider introducing an engine_init * function in the vtable */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h index 6743dcfe3f..f4959e3fee 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without -- cgit v1.2.3 From 8ed56f5a4b63157c98d579bca114ac355357fc82 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 09:59:25 -0700 Subject: Remove refcnt from fd --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 112 ++++++++---------------- 1 file changed, 35 insertions(+), 77 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 055b31331d..96e8000da4 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -72,6 +72,11 @@ static int grpc_polling_trace = 0; /* Disabled by default */ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + /* TODO: sreek: Right now, this wakes up all pollers. In future we should make * sure to wake up one polling thread (which can wake up other threads if * needed) */ @@ -87,15 +92,9 @@ struct grpc_fd { struct polling_island *pi; int fd; - /* refst format: - bit 0 : 1=Active / 0=Orphaned - bits 1-n : refcount - Ref/Unref by two to avoid altering the orphaned bit */ - gpr_atm refst; - - /* The fd is either closed or we relinquished control of it. In either - cases, this indicates that the 'fd' on this structure is no longer - valid */ + + /* The fd is either closed or we relinquished control of it. In either cases, + this indicates that the 'fd' on this structure is no longer valid */ bool orphaned; gpr_atm read_closure; @@ -182,9 +181,7 @@ struct grpc_pollset { /******************************************************************************* * Pollset-set Declarations */ -struct grpc_pollset_set { - void *no_op; -}; +struct grpc_pollset_set {}; /***************************************************************************** * Dedicated polling threads and pollsets - Declarations @@ -521,57 +518,31 @@ static void polling_island_global_shutdown() { * becomes a spurious read notification on a reused fd. */ -/* The alarm system needs to be able to wakeup 'some poller' sometimes - * (specifically when a new alarm needs to be triggered earlier than the next - * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a - * case occurs. */ - static grpc_fd *fd_freelist = NULL; static gpr_mu fd_freelist_mu; -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd *fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} +static grpc_fd *get_fd_from_freelist() { + grpc_fd *new_fd = NULL; -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - /* Add the fd to the freelist */ - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); - - gpr_mu_unlock(&fd_freelist_mu); - } else { - GPR_ASSERT(old > n); + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; } + gpr_mu_unlock(&fd_freelist_mu); + return new_fd; +} + +static void add_fd_to_freelist(grpc_fd *fd) { + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); } static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } @@ -589,15 +560,7 @@ static void fd_global_shutdown(void) { } static grpc_fd *fd_create(int fd, const char *name) { - grpc_fd *new_fd = NULL; - - gpr_mu_lock(&fd_freelist_mu); - if (fd_freelist != NULL) { - new_fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - } - gpr_mu_unlock(&fd_freelist_mu); - + grpc_fd *new_fd = get_fd_from_freelist(); if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); gpr_mu_init(&new_fd->mu); @@ -609,7 +572,6 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_mu_lock(&new_fd->mu); new_fd->pi = NULL; - gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); @@ -623,9 +585,7 @@ static grpc_fd *fd_create(int fd, const char *name) { char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); -#ifdef GRPC_FD_REF_COUNT_DEBUG gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); -#endif gpr_free(fd_name); /* Associate the fd with one of the dedicated pi */ @@ -665,14 +625,9 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; - /* Remove the active status but keep referenced. We want this grpc_fd struct - to be alive (and not added to freelist) until the end of this function */ - REF_BY(fd, 1, reason); - /* Remove the fd from the polling island */ if (fd->pi != NULL) { - polling_island *pi = fd->pi; - polling_island_remove_fd(pi, fd, is_fd_closed, &error); + polling_island_remove_fd(fd->pi, fd, is_fd_closed, &error); unref_pi = fd->pi; fd->pi = NULL; } @@ -680,7 +635,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ + + /* We are done with this fd. Release it (i.e add back to freelist) */ + add_fd_to_freelist(fd); + if (unref_pi != NULL) { /* Unref stale polling island here, outside the fd lock above. The polling island owns a workqueue which owns an fd, and unreffing -- cgit v1.2.3 From aa033db15e3cc3bb43b8cca0df9edccbf24690fd Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 11:00:27 -0700 Subject: Rename polling_island to epoll_set --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 382 ++++++++++++------------ 1 file changed, 191 insertions(+), 191 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 96e8000da4..448f2685fe 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -82,14 +82,14 @@ static int grpc_polling_trace = 0; /* Disabled by default */ * needed) */ static grpc_wakeup_fd global_wakeup_fd; -struct polling_island; +struct epoll_set; /******************************************************************************* * Fd Declarations */ struct grpc_fd { gpr_mu mu; - struct polling_island *pi; + struct epoll_set *eps; int fd; @@ -115,25 +115,25 @@ static void fd_global_shutdown(void); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) -#define PI_UNREF(exec_ctx, p, r) \ - pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) +#define EPS_ADD_REF(p, r) eps_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define EPS_UNREF(exec_ctx, p, r) \ + eps_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) #else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ -#define PI_ADD_REF(p, r) pi_add_ref((p)) -#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) +#define EPS_ADD_REF(p, r) eps_add_ref((p)) +#define EPS_UNREF(exec_ctx, p, r) eps_unref((exec_ctx), (p)) -#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ +#endif /* !defined(GRPC_EPS_REF_COUNT_DEBUG) */ /* This is also used as grpc_workqueue (by directly casting it) */ -typedef struct polling_island { +typedef struct epoll_set { grpc_closure_scheduler workqueue_scheduler; - /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + /* Ref count. Use EPS_ADD_REF() and EPS_UNREF() macros to increment/decrement the refcount. Once the ref count becomes zero, this structure is destroyed which means we should ensure that there is never a scenario where a - PI_ADD_REF() is racing with a PI_UNREF() that just made the ref_count + EPS_ADD_REF() is racing with a EPS_UNREF() that just made the ref_count zero. */ gpr_atm ref_count; @@ -154,7 +154,7 @@ typedef struct polling_island { /* The fd of the underlying epoll set */ int epoll_fd; -} polling_island; +} epoll_set; /******************************************************************************* * Pollset Declarations @@ -168,7 +168,7 @@ struct grpc_pollset_worker { struct grpc_pollset { gpr_mu mu; - struct polling_island *pi; + struct epoll_set *eps; grpc_pollset_worker root_worker; bool kicked_without_pollers; @@ -187,9 +187,9 @@ struct grpc_pollset_set {}; * Dedicated polling threads and pollsets - Declarations */ -size_t g_num_pi = 1; -struct polling_island **g_polling_islands = NULL; -size_t g_num_threads_per_pi = 1; +size_t g_num_eps = 1; +struct epoll_set **g_epoll_sets = NULL; +size_t g_num_threads_per_eps = 1; gpr_thd_id *g_poller_threads = NULL; /* Used as read-notifier pollsets for fds. We won't be using read notifier @@ -197,9 +197,9 @@ gpr_thd_id *g_poller_threads = NULL; * return */ grpc_pollset g_read_notifier; -static void add_fd_to_dedicated_pi(grpc_fd *fd); -static bool init_dedicated_polling_islands(); -static void shutdown_dedicated_polling_islands(); +static void add_fd_to_dedicated_eps(grpc_fd *fd); +static bool init_dedicated_epoll_sets(); +static void shutdown_dedicated_epoll_sets(); static void poller_thread_loop(void *arg); static void start_dedicated_poller_threads(); static void shutdown_dedicated_poller_threads(); @@ -229,14 +229,14 @@ static bool append_error(grpc_error **composite, grpc_error *error, NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ -static grpc_wakeup_fd polling_island_wakeup_fd; +static grpc_wakeup_fd epoll_set_wakeup_fd; /* The polling island being polled right now. See comments in workqueue_maybe_wakeup for why this is tracked. */ -static __thread polling_island *g_current_thread_polling_island; +static __thread epoll_set *g_current_thread_epoll_set; /* Forward declaration */ -static void polling_island_delete(polling_island *pi); +static void epoll_set_delete(epoll_set *eps); static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error); @@ -254,31 +254,31 @@ gpr_atm g_epoll_sync; static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { workqueue_enqueue, workqueue_enqueue, "workqueue"}; -static void pi_add_ref(polling_island *pi); -static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void eps_add_ref(epoll_set *eps); +static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -static void pi_add_ref_dbg(polling_island *pi, const char *reason, +static void eps_add_ref_dbg(epoll_set *eps, const char *reason, const char *file, int line) { - long old_cnt = gpr_atm_acq_load(&pi->ref_count); - pi_add_ref(pi); - gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", - (void *)pi, old_cnt, old_cnt + 1, reason, file, line); + long old_cnt = gpr_atm_acq_load(&eps->ref_count); + eps_add_ref(eps); + gpr_log(GPR_DEBUG, "Add ref eps: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)eps, old_cnt, old_cnt + 1, reason, file, line); } -static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, +static void eps_unref_dbg(grpc_exec_ctx *exec_ctx, epoll_set *eps, const char *reason, const char *file, int line) { - long old_cnt = gpr_atm_acq_load(&pi->ref_count); - pi_unref(exec_ctx, pi); - gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", - (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); + long old_cnt = gpr_atm_acq_load(&eps->ref_count); + eps_unref(exec_ctx, eps); + gpr_log(GPR_DEBUG, "Unref eps: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)eps, old_cnt, (old_cnt - 1), reason, file, line); } static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, const char *reason) { if (workqueue != NULL) { - pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + eps_add_ref_dbg((epoll_set *)workqueue, reason, file, line); } return workqueue; } @@ -286,13 +286,13 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, const char *file, int line, const char *reason) { if (workqueue != NULL) { - pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + eps_unref_dbg(exec_ctx, (epoll_set *)workqueue, reason, file, line); } } #else static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { if (workqueue != NULL) { - pi_add_ref((polling_island *)workqueue); + eps_add_ref((epoll_set *)workqueue); } return workqueue; } @@ -300,31 +300,31 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) { if (workqueue != NULL) { - pi_unref(exec_ctx, (polling_island *)workqueue); + eps_unref(exec_ctx, (epoll_set *)workqueue); } } #endif -static void pi_add_ref(polling_island *pi) { - gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +static void eps_add_ref(epoll_set *eps) { + gpr_atm_no_barrier_fetch_add(&eps->ref_count, 1); } -static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { +static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { /* If ref count went to zero, delete the polling island. This deletion is not done under a lock since once the ref count goes to zero, we are guaranteed that no one else holds a reference to the polling island (and - that there is no racing pi_add_ref() call either).*/ - if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { - polling_island_delete(pi); + that there is no racing eps_add_ref() call either).*/ + if (1 == gpr_atm_full_fetch_add(&eps->ref_count, -1)) { + epoll_set_delete(eps); } } -static void polling_island_add_fd_locked(polling_island *pi, grpc_fd *fd, +static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, grpc_error **error) { int err; struct epoll_event ev; char *err_msg; - const char *err_desc = "polling_island_add_fd_locked"; + const char *err_desc = "epoll_set_add_fd_locked"; #ifdef GRPC_TSAN /* See the definition of g_epoll_sync for more context */ @@ -333,55 +333,55 @@ static void polling_island_add_fd_locked(polling_island *pi, grpc_fd *fd, ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); ev.data.ptr = fd; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); if (err < 0 && errno != EEXIST) { gpr_asprintf( &err_msg, "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", - pi->epoll_fd, fd->fd, errno, strerror(errno)); + eps->epoll_fd, fd->fd, errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } } -static void polling_island_add_wakeup_fd_locked(polling_island *pi, +static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, grpc_wakeup_fd *wakeup_fd, grpc_error **error) { struct epoll_event ev; int err; char *err_msg; - const char *err_desc = "polling_island_add_wakeup_fd"; + const char *err_desc = "epoll_set_add_wakeup_fd"; ev.events = (uint32_t)(EPOLLIN | EPOLLET); ev.data.ptr = wakeup_fd; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_ADD, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); if (err < 0 && errno != EEXIST) { gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } } -static void polling_island_remove_fd(polling_island *pi, grpc_fd *fd, +static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, bool is_fd_closed, grpc_error **error) { int err; char *err_msg; - const char *err_desc = "polling_island_remove_fd"; + const char *err_desc = "epoll_set_remove_fd"; /* If fd is already closed, then it would have been automatically been removed from the epoll set */ if (!is_fd_closed) { - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); if (err < 0 && errno != ENOENT) { gpr_asprintf( &err_msg, "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", - pi->epoll_fd, fd->fd, errno, strerror(errno)); + eps->epoll_fd, fd->fd, errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -389,74 +389,74 @@ static void polling_island_remove_fd(polling_island *pi, grpc_fd *fd, } /* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_error **error) { - polling_island *pi = NULL; - const char *err_desc = "polling_island_create"; +static epoll_set *epoll_set_create(grpc_error **error) { + epoll_set *eps = NULL; + const char *err_desc = "epoll_set_create"; *error = GRPC_ERROR_NONE; - pi = gpr_malloc(sizeof(*pi)); - pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - pi->epoll_fd = -1; + eps = gpr_malloc(sizeof(*eps)); + eps->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + eps->epoll_fd = -1; - gpr_mu_init(&pi->workqueue_read_mu); - gpr_mpscq_init(&pi->workqueue_items); - gpr_atm_rel_store(&pi->workqueue_item_count, 0); + gpr_mu_init(&eps->workqueue_read_mu); + gpr_mpscq_init(&eps->workqueue_items); + gpr_atm_rel_store(&eps->workqueue_item_count, 0); - gpr_atm_rel_store(&pi->ref_count, 0); - gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&eps->ref_count, 0); + gpr_atm_rel_store(&eps->poller_count, 0); - gpr_atm_rel_store(&pi->is_shutdown, false); + gpr_atm_rel_store(&eps->is_shutdown, false); - if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + if (!append_error(error, grpc_wakeup_fd_init(&eps->workqueue_wakeup_fd), err_desc)) { goto done; } - pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + eps->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - if (pi->epoll_fd < 0) { + if (eps->epoll_fd < 0) { append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); goto done; } - polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); - polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + epoll_set_add_wakeup_fd_locked(eps, &global_wakeup_fd, error); + epoll_set_add_wakeup_fd_locked(eps, &eps->workqueue_wakeup_fd, error); done: if (*error != GRPC_ERROR_NONE) { - polling_island_delete(pi); - pi = NULL; + epoll_set_delete(eps); + eps = NULL; } - return pi; + return eps; } -static void polling_island_delete(polling_island *pi) { - if (pi->epoll_fd >= 0) { - close(pi->epoll_fd); +static void epoll_set_delete(epoll_set *eps) { + if (eps->epoll_fd >= 0) { + close(eps->epoll_fd); } - GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); - gpr_mu_destroy(&pi->workqueue_read_mu); - gpr_mpscq_destroy(&pi->workqueue_items); - grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + GPR_ASSERT(gpr_atm_no_barrier_load(&eps->workqueue_item_count) == 0); + gpr_mu_destroy(&eps->workqueue_read_mu); + gpr_mpscq_destroy(&eps->workqueue_items); + grpc_wakeup_fd_destroy(&eps->workqueue_wakeup_fd); - gpr_free(pi); + gpr_free(eps); } -static void workqueue_maybe_wakeup(polling_island *pi) { +static void workqueue_maybe_wakeup(epoll_set *eps) { /* If this thread is the current poller, then it may be that it's about to decrement the current poller count, so we need to look past this thread */ - bool is_current_poller = (g_current_thread_polling_island == pi); + bool is_current_poller = (g_current_thread_epoll_set == eps); gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; - gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + gpr_atm current_pollers = gpr_atm_no_barrier_load(&eps->poller_count); /* Only issue a wakeup if it's likely that some poller could come in and take it right now. Note that since we do an anticipatory mpscq_pop every poll loop, it's ok if we miss the wakeup here, as we'll get the work item when the next poller enters anyway. */ if (current_pollers > min_current_pollers_for_wakeup) { GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", - grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + grpc_wakeup_fd_wakeup(&eps->workqueue_wakeup_fd)); } } @@ -468,12 +468,12 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, * this kicks off ends up destroying the workqueue before this function * completes */ GRPC_WORKQUEUE_REF(workqueue, "enqueue"); - polling_island *pi = (polling_island *)workqueue; - gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + epoll_set *eps = (epoll_set *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&eps->workqueue_item_count, 1); closure->error_data.error = error; - gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + gpr_mpscq_push(&eps->workqueue_items, &closure->next_data.atm_next); if (last == 0) { - workqueue_maybe_wakeup(pi); + workqueue_maybe_wakeup(eps); } GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); @@ -481,24 +481,24 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, } static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - polling_island *pi = (polling_island *)workqueue; + epoll_set *eps = (epoll_set *)workqueue; return workqueue == NULL ? grpc_schedule_on_exec_ctx - : &pi->workqueue_scheduler; + : &eps->workqueue_scheduler; } -static grpc_error *polling_island_global_init() { +static grpc_error *epoll_set_global_init() { grpc_error *error = GRPC_ERROR_NONE; - error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + error = grpc_wakeup_fd_init(&epoll_set_wakeup_fd); if (error == GRPC_ERROR_NONE) { - error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + error = grpc_wakeup_fd_wakeup(&epoll_set_wakeup_fd); } return error; } -static void polling_island_global_shutdown() { - grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +static void epoll_set_global_shutdown() { + grpc_wakeup_fd_destroy(&epoll_set_wakeup_fd); } /******************************************************************************* @@ -570,7 +570,7 @@ static grpc_fd *fd_create(int fd, const char *name) { * is a newly created fd (or an fd we got from the freelist), no one else * would be holding a lock to it anyway. */ gpr_mu_lock(&new_fd->mu); - new_fd->pi = NULL; + new_fd->eps = NULL; new_fd->fd = fd; new_fd->orphaned = false; @@ -588,8 +588,8 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); gpr_free(fd_name); - /* Associate the fd with one of the dedicated pi */ - add_fd_to_dedicated_pi(new_fd); + /* Associate the fd with one of the dedicated eps */ + add_fd_to_dedicated_eps(new_fd); return new_fd; } @@ -609,7 +609,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, const char *reason) { bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - polling_island *unref_pi = NULL; + epoll_set *unref_eps = NULL; gpr_mu_lock(&fd->mu); fd->on_done_closure = on_done; @@ -626,10 +626,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; /* Remove the fd from the polling island */ - if (fd->pi != NULL) { - polling_island_remove_fd(fd->pi, fd, is_fd_closed, &error); - unref_pi = fd->pi; - fd->pi = NULL; + if (fd->eps != NULL) { + epoll_set_remove_fd(fd->eps, fd, is_fd_closed, &error); + unref_eps = fd->eps; + fd->eps = NULL; } grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); @@ -639,12 +639,12 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, /* We are done with this fd. Release it (i.e add back to freelist) */ add_fd_to_freelist(fd); - if (unref_pi != NULL) { + if (unref_eps != NULL) { /* Unref stale polling island here, outside the fd lock above. The polling island owns a workqueue which owns an fd, and unreffing inside the lock can cause an eventual lock loop that makes TSAN very unhappy. */ - PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + EPS_UNREF(exec_ctx, unref_eps, "fd_orphan"); } GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -805,7 +805,7 @@ static grpc_error *kick_poller(void) { static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->pi = NULL; + pollset->eps = NULL; pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; pollset->kicked_without_pollers = false; @@ -823,12 +823,12 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, +static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, char *reason) { - if (ps->pi != NULL) { - PI_UNREF(exec_ctx, ps->pi, reason); + if (ps->eps != NULL) { + EPS_UNREF(exec_ctx, ps->eps, reason); } - ps->pi = NULL; + ps->eps = NULL; } static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, @@ -838,8 +838,8 @@ static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, pollset->finish_shutdown_called = true; - /* Release the ref and set pollset->pi to NULL */ - pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + /* Release the ref and set pollset->eps to NULL */ + pollset_release_epoll_set(exec_ctx, pollset, "ps_shutdown"); grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); } @@ -872,13 +872,13 @@ static void pollset_destroy(grpc_pollset *pollset) { } static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, - polling_island *pi) { - if (gpr_mu_trylock(&pi->workqueue_read_mu)) { - gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); - gpr_mu_unlock(&pi->workqueue_read_mu); + epoll_set *eps) { + if (gpr_mu_trylock(&eps->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&eps->workqueue_items); + gpr_mu_unlock(&eps->workqueue_read_mu); if (n != NULL) { - if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { - workqueue_maybe_wakeup(pi); + if (gpr_atm_full_fetch_add(&eps->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(eps); } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; @@ -888,11 +888,11 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); return true; - } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + } else if (gpr_atm_no_barrier_load(&eps->workqueue_item_count) > 0) { /* n == NULL might mean there's work but it's not available to be popped * yet - try to ensure another workqueue wakes up to check shortly if so */ - workqueue_maybe_wakeup(pi); + workqueue_maybe_wakeup(eps); } } return false; @@ -900,7 +900,7 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, #define GRPC_EPOLL_MAX_EVENTS 100 static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, - polling_island *pi, grpc_error **error) { + epoll_set *eps, grpc_error **error) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; @@ -930,13 +930,13 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, grpc_timer_consume_kick(); append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { + } else if (data_ptr == &eps->workqueue_wakeup_fd) { append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + grpc_wakeup_fd_consume_wakeup(&eps->workqueue_wakeup_fd), err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - gpr_atm_rel_store(&pi->is_shutdown, 1); + maybe_do_workqueue_work(exec_ctx, eps); + } else if (data_ptr == &epoll_set_wakeup_fd) { + gpr_atm_rel_store(&eps->is_shutdown, 1); gpr_log(GPR_INFO, "pollset poller: shutdown set"); } else { grpc_fd *fd = data_ptr; @@ -953,41 +953,41 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, } } -static void polling_island_work(grpc_exec_ctx *exec_ctx, polling_island *pi, +static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, grpc_error **error) { int epoll_fd = -1; - GPR_TIMER_BEGIN("polling_island_work", 0); + GPR_TIMER_BEGIN("epoll_set_work", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the polling island. */ - epoll_fd = pi->epoll_fd; + epoll_fd = eps->epoll_fd; /* Add an extra ref so that the island does not get destroyed (which means the epoll_fd won't be closed) while we are are doing an epoll_wait() on the epoll_fd */ - PI_ADD_REF(pi, "ps_work"); + EPS_ADD_REF(eps, "ps_work"); /* If we get some workqueue work to do, it might end up completing an item on the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ - if (!maybe_do_workqueue_work(exec_ctx, pi)) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); - g_current_thread_polling_island = pi; + if (!maybe_do_workqueue_work(exec_ctx, eps)) { + gpr_atm_no_barrier_fetch_add(&eps->poller_count, 1); + g_current_thread_epoll_set = eps; - do_epoll_wait(exec_ctx, epoll_fd, pi, error); + do_epoll_wait(exec_ctx, epoll_fd, eps, error); - g_current_thread_polling_island = NULL; - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + g_current_thread_epoll_set = NULL; + gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); } /* Before leaving, release the extra ref we added to the polling island. It - is important to use "pi" here (i.e our old copy of pollset->pi + is important to use "eps" here (i.e our old copy of pollset->eps that we got before releasing the polling island lock). This is because - pollset->pi pointer might get udpated in other parts of the + pollset->eps pointer might get udpated in other parts of the code when there is an island merge while we are doing epoll_wait() above */ - PI_UNREF(exec_ctx, pi, "ps_work"); + EPS_UNREF(exec_ctx, eps, "ps_work"); - GPR_TIMER_END("polling_island_work", 0); + GPR_TIMER_END("epoll_set_work", 0); } /* pollset->mu lock must be held by the caller before calling this. @@ -1110,10 +1110,10 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { shutdown_dedicated_poller_threads(); - shutdown_dedicated_polling_islands(); + shutdown_dedicated_epoll_sets(); fd_global_shutdown(); pollset_global_shutdown(); - polling_island_global_shutdown(); + epoll_set_global_shutdown(); gpr_log(GPR_INFO, "ev-epoll-threadpool engine shutdown complete"); } @@ -1158,13 +1158,13 @@ static const grpc_event_engine_vtable vtable = { /***************************************************************************** * Dedicated polling threads and pollsets - Definitions */ -static void add_fd_to_dedicated_pi(grpc_fd *fd) { - GPR_ASSERT(fd->pi == NULL); - GPR_TIMER_BEGIN("add_fd_to_dedicated_pi", 0); +static void add_fd_to_dedicated_eps(grpc_fd *fd) { + GPR_ASSERT(fd->eps == NULL); + GPR_TIMER_BEGIN("add_fd_to_dedicated_eps", 0); grpc_error *error = GRPC_ERROR_NONE; - size_t idx = ((size_t)rand()) % g_num_pi; - polling_island *pi = g_polling_islands[idx]; + size_t idx = ((size_t)rand()) % g_num_eps; + epoll_set *eps = g_epoll_sets[idx]; gpr_mu_lock(&fd->mu); @@ -1173,70 +1173,70 @@ static void add_fd_to_dedicated_pi(grpc_fd *fd) { return; /* Early out */ } - polling_island_add_fd_locked(pi, fd, &error); - PI_ADD_REF(pi, "fd"); - fd->pi = pi; + epoll_set_add_fd_locked(eps, fd, &error); + EPS_ADD_REF(eps, "fd"); + fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_dedicated_pi (fd: %d, pi idx = %ld)", fd->fd, + GRPC_POLLING_TRACE("add_fd_to_dedicated_eps (fd: %d, eps idx = %ld)", fd->fd, idx); gpr_mu_unlock(&fd->mu); - GRPC_LOG_IF_ERROR("add_fd_to_dedicated_pi", error); - GPR_TIMER_END("add_fd_to_dedicated_pi", 0); + GRPC_LOG_IF_ERROR("add_fd_to_dedicated_eps", error); + GPR_TIMER_END("add_fd_to_dedicated_eps", 0); } -static bool init_dedicated_polling_islands() { +static bool init_dedicated_epoll_sets() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; - g_polling_islands = - (polling_island **)malloc(g_num_pi * sizeof(polling_island *)); + g_epoll_sets = + (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); - for (size_t i = 0; i < g_num_pi; i++) { - g_polling_islands[i] = polling_island_create(&error); - if (g_polling_islands[i] == NULL) { + for (size_t i = 0; i < g_num_eps; i++) { + g_epoll_sets[i] = epoll_set_create(&error); + if (g_epoll_sets[i] == NULL) { gpr_log(GPR_ERROR, "Error in creating a dedicated polling island"); - g_num_pi = i; /* Helps cleanup */ - shutdown_dedicated_polling_islands(); + g_num_eps = i; /* Helps cleanup */ + shutdown_dedicated_epoll_sets(); is_success = false; goto done; } - PI_ADD_REF(g_polling_islands[i], "init_dedicated_polling_islands"); + EPS_ADD_REF(g_epoll_sets[i], "init_dedicated_epoll_sets"); } gpr_mu *mu; pollset_init(&g_read_notifier, &mu); done: - GRPC_LOG_IF_ERROR("init_dedicated_polling_islands", error); + GRPC_LOG_IF_ERROR("init_dedicated_epoll_sets", error); return is_success; } -static void shutdown_dedicated_polling_islands() { - if (!g_polling_islands) { +static void shutdown_dedicated_epoll_sets() { + if (!g_epoll_sets) { return; } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - for (size_t i = 0; i < g_num_pi; i++) { - PI_UNREF(&exec_ctx, g_polling_islands[i], - "shutdown_dedicated_polling_islands"); + for (size_t i = 0; i < g_num_eps; i++) { + EPS_UNREF(&exec_ctx, g_epoll_sets[i], + "shutdown_dedicated_epoll_sets"); } grpc_exec_ctx_finish(&exec_ctx); - gpr_free(g_polling_islands); - g_polling_islands = NULL; + gpr_free(g_epoll_sets); + g_epoll_sets = NULL; pollset_destroy(&g_read_notifier); } static void poller_thread_loop(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi = (polling_island *)arg; + epoll_set *eps = (epoll_set *)arg; - while (!gpr_atm_acq_load(&pi->is_shutdown)) { - polling_island_work(&exec_ctx, pi, &error); + while (!gpr_atm_acq_load(&eps->is_shutdown)) { + epoll_set_work(&exec_ctx, eps, &error); grpc_exec_ctx_flush(&exec_ctx); } @@ -1244,37 +1244,37 @@ static void poller_thread_loop(void *arg) { GRPC_LOG_IF_ERROR("poller_thread_loop", error); } -/* g_polling_islands MUST be initialized before calling this */ +/* g_epoll_sets MUST be initialized before calling this */ static void start_dedicated_poller_threads() { - GPR_ASSERT(g_polling_islands); + GPR_ASSERT(g_epoll_sets); gpr_log(GPR_INFO, "Starting poller threads"); /* One thread per pollset */ - g_poller_threads = (gpr_thd_id *)malloc(g_num_pi * sizeof(gpr_thd_id)); + g_poller_threads = (gpr_thd_id *)malloc(g_num_eps * sizeof(gpr_thd_id)); gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); - for (size_t i = 0; i < g_num_pi; i++) { + for (size_t i = 0; i < g_num_eps; i++) { gpr_thd_new(&g_poller_threads[i], poller_thread_loop, - (void *)g_polling_islands[i], &options); + (void *)g_epoll_sets[i], &options); } } static void shutdown_dedicated_poller_threads() { GPR_ASSERT(g_poller_threads); - GPR_ASSERT(g_polling_islands); + GPR_ASSERT(g_epoll_sets); grpc_error *error = GRPC_ERROR_NONE; gpr_log(GPR_INFO, "Shutting down pollers"); - polling_island *pi = NULL; - for (size_t i = 0; i < g_num_pi; i++) { - pi = g_polling_islands[i]; - polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); + epoll_set *eps = NULL; + for (size_t i = 0; i < g_num_eps; i++) { + eps = g_epoll_sets[i]; + epoll_set_add_wakeup_fd_locked(eps, &epoll_set_wakeup_fd, &error); } - for (size_t i = 0; i < g_num_pi; i++) { + for (size_t i = 0; i < g_num_eps; i++) { gpr_thd_join(g_poller_threads[i]); } @@ -1316,12 +1316,12 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!GRPC_LOG_IF_ERROR("polling_island_global_init", - polling_island_global_init())) { + if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", + epoll_set_global_init())) { return NULL; } - if (!init_dedicated_polling_islands()) { + if (!init_dedicated_epoll_sets()) { return NULL; } -- cgit v1.2.3 From 0a3a416dde6b6837900e4086c9366da11ea514b0 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 11:00:53 -0700 Subject: Format --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 36 +++++++++++-------------- 1 file changed, 16 insertions(+), 20 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 448f2685fe..d3e7820149 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -259,7 +259,7 @@ static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static void eps_add_ref_dbg(epoll_set *eps, const char *reason, - const char *file, int line) { + const char *file, int line) { long old_cnt = gpr_atm_acq_load(&eps->ref_count); eps_add_ref(eps); gpr_log(GPR_DEBUG, "Add ref eps: %p, old: %ld -> new:%ld (%s) - (%s, %d)", @@ -267,7 +267,7 @@ static void eps_add_ref_dbg(epoll_set *eps, const char *reason, } static void eps_unref_dbg(grpc_exec_ctx *exec_ctx, epoll_set *eps, - const char *reason, const char *file, int line) { + const char *reason, const char *file, int line) { long old_cnt = gpr_atm_acq_load(&eps->ref_count); eps_unref(exec_ctx, eps); gpr_log(GPR_DEBUG, "Unref eps: %p, old:%ld -> new:%ld (%s) - (%s, %d)", @@ -320,7 +320,7 @@ static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { } static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, - grpc_error **error) { + grpc_error **error) { int err; struct epoll_event ev; char *err_msg; @@ -345,8 +345,8 @@ static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, } static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, - grpc_wakeup_fd *wakeup_fd, - grpc_error **error) { + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { struct epoll_event ev; int err; char *err_msg; @@ -367,8 +367,8 @@ static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, } } -static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, - bool is_fd_closed, grpc_error **error) { +static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, bool is_fd_closed, + grpc_error **error) { int err; char *err_msg; const char *err_desc = "epoll_set_remove_fd"; @@ -823,8 +823,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, - grpc_pollset *ps, char *reason) { +static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, + char *reason) { if (ps->eps != NULL) { EPS_UNREF(exec_ctx, ps->eps, reason); } @@ -871,8 +871,7 @@ static void pollset_destroy(grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, - epoll_set *eps) { +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, epoll_set *eps) { if (gpr_mu_trylock(&eps->workqueue_read_mu)) { gpr_mpscq_node *n = gpr_mpscq_pop(&eps->workqueue_items); gpr_mu_unlock(&eps->workqueue_read_mu); @@ -899,8 +898,8 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, } #define GRPC_EPOLL_MAX_EVENTS 100 -static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, - epoll_set *eps, grpc_error **error) { +static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, + grpc_error **error) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; @@ -954,7 +953,7 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, } static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, - grpc_error **error) { + grpc_error **error) { int epoll_fd = -1; GPR_TIMER_BEGIN("epoll_set_work", 0); @@ -1189,8 +1188,7 @@ static bool init_dedicated_epoll_sets() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; - g_epoll_sets = - (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); + g_epoll_sets = (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); for (size_t i = 0; i < g_num_eps; i++) { g_epoll_sets[i] = epoll_set_create(&error); @@ -1220,8 +1218,7 @@ static void shutdown_dedicated_epoll_sets() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; for (size_t i = 0; i < g_num_eps; i++) { - EPS_UNREF(&exec_ctx, g_epoll_sets[i], - "shutdown_dedicated_epoll_sets"); + EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_dedicated_epoll_sets"); } grpc_exec_ctx_finish(&exec_ctx); @@ -1316,8 +1313,7 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", - epoll_set_global_init())) { + if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", epoll_set_global_init())) { return NULL; } -- cgit v1.2.3 From c5cae77ef5f3fa0d2401cb648020a35295b9c8b5 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 11:39:03 -0700 Subject: Rename a few functions/variables --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 82 ++++++++++++------------- 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index d3e7820149..d11dbbfae8 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -149,7 +149,7 @@ typedef struct epoll_set { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; - /* Is the polling island shutdown */ + /* Is the epoll set shutdown */ gpr_atm is_shutdown; /* The fd of the underlying epoll set */ @@ -197,12 +197,12 @@ gpr_thd_id *g_poller_threads = NULL; * return */ grpc_pollset g_read_notifier; -static void add_fd_to_dedicated_eps(grpc_fd *fd); -static bool init_dedicated_epoll_sets(); -static void shutdown_dedicated_epoll_sets(); +static void add_fd_to_eps(grpc_fd *fd); +static bool init_epoll_sets(); +static void shutdown_epoll_sets(); static void poller_thread_loop(void *arg); -static void start_dedicated_poller_threads(); -static void shutdown_dedicated_poller_threads(); +static void start_poller_threads(); +static void shutdown_poller_threads(); /******************************************************************************* * Common helpers @@ -223,15 +223,15 @@ static bool append_error(grpc_error **composite, grpc_error *error, */ /* The wakeup fd that is used to wake up all threads in a Polling island. This - is useful in the polling island merge operation where we need to wakeup all - the threads currently polling the smaller polling island (so that they can - start polling the new/merged polling island) + is useful in the epoll set merge operation where we need to wakeup all + the threads currently polling the smaller epoll set (so that they can + start polling the new/merged epoll set) NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ static grpc_wakeup_fd epoll_set_wakeup_fd; -/* The polling island being polled right now. +/* The epoll set being polled right now. See comments in workqueue_maybe_wakeup for why this is tracked. */ static __thread epoll_set *g_current_thread_epoll_set; @@ -310,9 +310,9 @@ static void eps_add_ref(epoll_set *eps) { } static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { - /* If ref count went to zero, delete the polling island. This deletion is + /* If ref count went to zero, delete the epoll set. This deletion is not done under a lock since once the ref count goes to zero, we are - guaranteed that no one else holds a reference to the polling island (and + guaranteed that no one else holds a reference to the epoll set (and that there is no racing eps_add_ref() call either).*/ if (1 == gpr_atm_full_fetch_add(&eps->ref_count, -1)) { epoll_set_delete(eps); @@ -588,8 +588,8 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); gpr_free(fd_name); - /* Associate the fd with one of the dedicated eps */ - add_fd_to_dedicated_eps(new_fd); + /* Associate the fd with one of the eps */ + add_fd_to_eps(new_fd); return new_fd; } @@ -625,7 +625,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; - /* Remove the fd from the polling island */ + /* Remove the fd from the epoll set */ if (fd->eps != NULL) { epoll_set_remove_fd(fd->eps, fd, is_fd_closed, &error); unref_eps = fd->eps; @@ -640,8 +640,8 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, add_fd_to_freelist(fd); if (unref_eps != NULL) { - /* Unref stale polling island here, outside the fd lock above. - The polling island owns a workqueue which owns an fd, and unreffing + /* Unref stale epoll set here, outside the fd lock above. + The epoll set owns a workqueue which owns an fd, and unreffing inside the lock can cause an eventual lock loop that makes TSAN very unhappy. */ EPS_UNREF(exec_ctx, unref_eps, "fd_orphan"); @@ -853,7 +853,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() - because it would release the underlying polling island. In such a case, we + because it would release the underlying epoll set. In such a case, we let the last worker call finish_shutdown_locked() from pollset_work() */ if (!pollset_has_workers(pollset)) { GPR_ASSERT(!pollset->finish_shutdown_called); @@ -958,7 +958,7 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, GPR_TIMER_BEGIN("epoll_set_work", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the - polling island. */ + epoll set. */ epoll_fd = eps->epoll_fd; /* Add an extra ref so that the island does not get destroyed (which means @@ -979,9 +979,9 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); } - /* Before leaving, release the extra ref we added to the polling island. It + /* Before leaving, release the extra ref we added to the epoll set. It is important to use "eps" here (i.e our old copy of pollset->eps - that we got before releasing the polling island lock). This is because + that we got before releasing the epoll set lock). This is because pollset->eps pointer might get udpated in other parts of the code when there is an island merge while we are doing epoll_wait() above */ EPS_UNREF(exec_ctx, eps, "ps_work"); @@ -1108,8 +1108,8 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, */ static void shutdown_engine(void) { - shutdown_dedicated_poller_threads(); - shutdown_dedicated_epoll_sets(); + shutdown_poller_threads(); + shutdown_epoll_sets(); fd_global_shutdown(); pollset_global_shutdown(); epoll_set_global_shutdown(); @@ -1157,9 +1157,9 @@ static const grpc_event_engine_vtable vtable = { /***************************************************************************** * Dedicated polling threads and pollsets - Definitions */ -static void add_fd_to_dedicated_eps(grpc_fd *fd) { +static void add_fd_to_eps(grpc_fd *fd) { GPR_ASSERT(fd->eps == NULL); - GPR_TIMER_BEGIN("add_fd_to_dedicated_eps", 0); + GPR_TIMER_BEGIN("add_fd_to_eps", 0); grpc_error *error = GRPC_ERROR_NONE; size_t idx = ((size_t)rand()) % g_num_eps; @@ -1176,15 +1176,15 @@ static void add_fd_to_dedicated_eps(grpc_fd *fd) { EPS_ADD_REF(eps, "fd"); fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_dedicated_eps (fd: %d, eps idx = %ld)", fd->fd, + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, idx); gpr_mu_unlock(&fd->mu); - GRPC_LOG_IF_ERROR("add_fd_to_dedicated_eps", error); - GPR_TIMER_END("add_fd_to_dedicated_eps", 0); + GRPC_LOG_IF_ERROR("add_fd_to_eps", error); + GPR_TIMER_END("add_fd_to_eps", 0); } -static bool init_dedicated_epoll_sets() { +static bool init_epoll_sets() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; @@ -1193,32 +1193,32 @@ static bool init_dedicated_epoll_sets() { for (size_t i = 0; i < g_num_eps; i++) { g_epoll_sets[i] = epoll_set_create(&error); if (g_epoll_sets[i] == NULL) { - gpr_log(GPR_ERROR, "Error in creating a dedicated polling island"); + gpr_log(GPR_ERROR, "Error in creating a epoll set"); g_num_eps = i; /* Helps cleanup */ - shutdown_dedicated_epoll_sets(); + shutdown_epoll_sets(); is_success = false; goto done; } - EPS_ADD_REF(g_epoll_sets[i], "init_dedicated_epoll_sets"); + EPS_ADD_REF(g_epoll_sets[i], "init_epoll_sets"); } gpr_mu *mu; pollset_init(&g_read_notifier, &mu); done: - GRPC_LOG_IF_ERROR("init_dedicated_epoll_sets", error); + GRPC_LOG_IF_ERROR("init_epoll_sets", error); return is_success; } -static void shutdown_dedicated_epoll_sets() { +static void shutdown_epoll_sets() { if (!g_epoll_sets) { return; } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; for (size_t i = 0; i < g_num_eps; i++) { - EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_dedicated_epoll_sets"); + EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_epoll_sets"); } grpc_exec_ctx_finish(&exec_ctx); @@ -1242,7 +1242,7 @@ static void poller_thread_loop(void *arg) { } /* g_epoll_sets MUST be initialized before calling this */ -static void start_dedicated_poller_threads() { +static void start_poller_threads() { GPR_ASSERT(g_epoll_sets); gpr_log(GPR_INFO, "Starting poller threads"); @@ -1258,7 +1258,7 @@ static void start_dedicated_poller_threads() { } } -static void shutdown_dedicated_poller_threads() { +static void shutdown_poller_threads() { GPR_ASSERT(g_poller_threads); GPR_ASSERT(g_epoll_sets); grpc_error *error = GRPC_ERROR_NONE; @@ -1275,8 +1275,8 @@ static void shutdown_dedicated_poller_threads() { gpr_thd_join(g_poller_threads[i]); } - gpr_log(GPR_ERROR, "polling island delete called"); - GRPC_LOG_IF_ERROR("shutdown_dedicated_poller_threads", error); + gpr_log(GPR_ERROR, "epoll set delete called"); + GRPC_LOG_IF_ERROR("shutdown_poller_threads", error); gpr_free(g_poller_threads); g_poller_threads = NULL; } @@ -1317,14 +1317,14 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!init_dedicated_epoll_sets()) { + if (!init_epoll_sets()) { return NULL; } /* TODO (sreek): Maynot be a good idea to start threads here (especially if * this engine doesn't get picked. Consider introducing an engine_init * function in the vtable */ - start_dedicated_poller_threads(); + start_poller_threads(); return &vtable; } -- cgit v1.2.3 From 84f75d448e2eb9c76301598cfaf8630603ef4a98 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 3 May 2017 13:06:35 -0700 Subject: Fix ASAN/TSAN failures - trace system is now thread safe when run with TSAN - fix a race in client_auth_filter.c - allow timer manager to run in single threaded mode for fuzzers --- .../filters/client_channel/channel_connectivity.c | 2 +- .../client_channel/lb_policy/grpclb/grpclb.c | 40 ++++++++-------- .../lb_policy/round_robin/round_robin.c | 22 ++++----- src/core/ext/filters/http/http_filters_plugin.c | 1 + .../message_compress/message_compress_filter.c | 5 +- .../message_compress/message_compress_filter.h | 2 - .../transport/chttp2/transport/chttp2_transport.c | 22 ++++----- .../transport/chttp2/transport/chttp2_transport.h | 5 +- .../transport/chttp2/transport/frame_settings.c | 6 +-- .../ext/transport/chttp2/transport/hpack_parser.c | 6 +-- src/core/ext/transport/chttp2/transport/internal.h | 18 +++---- src/core/ext/transport/chttp2/transport/parsing.c | 8 ++-- src/core/ext/transport/chttp2/transport/writing.c | 9 ++-- src/core/lib/channel/channel_stack.c | 2 +- src/core/lib/channel/channel_stack.h | 4 +- src/core/lib/channel/channel_stack_builder.c | 2 +- src/core/lib/channel/channel_stack_builder.h | 2 +- src/core/lib/debug/trace.c | 16 +++++-- src/core/lib/debug/trace.h | 26 +++++++++- src/core/lib/http/parser.c | 4 +- src/core/lib/http/parser.h | 3 +- src/core/lib/iomgr/combiner.c | 12 ++--- src/core/lib/iomgr/combiner.h | 3 +- src/core/lib/iomgr/iomgr.c | 3 ++ src/core/lib/iomgr/iomgr.h | 3 ++ src/core/lib/iomgr/resource_quota.c | 16 +++---- src/core/lib/iomgr/resource_quota.h | 3 +- src/core/lib/iomgr/tcp_client_posix.c | 8 ++-- src/core/lib/iomgr/tcp_posix.c | 14 +++--- src/core/lib/iomgr/tcp_posix.h | 3 +- src/core/lib/iomgr/tcp_server_posix.c | 2 +- src/core/lib/iomgr/timer_generic.c | 49 ++++++++++--------- src/core/lib/iomgr/timer_manager.c | 56 +++++++++++++++++----- src/core/lib/iomgr/timer_manager.h | 9 ++++ .../lib/security/credentials/jwt/jwt_credentials.c | 2 +- .../credentials/oauth2/oauth2_credentials.c | 2 +- .../lib/security/transport/client_auth_filter.c | 2 +- src/core/lib/security/transport/secure_endpoint.c | 6 +-- src/core/lib/security/transport/secure_endpoint.h | 2 +- src/core/lib/surface/api_trace.c | 2 +- src/core/lib/surface/api_trace.h | 4 +- src/core/lib/surface/call.c | 11 ++--- src/core/lib/surface/call.h | 6 ++- src/core/lib/surface/completion_queue.c | 33 +++++++------ src/core/lib/surface/completion_queue.h | 9 ++-- src/core/lib/surface/init.c | 3 +- src/core/lib/surface/server.c | 4 +- src/core/lib/surface/server.h | 3 +- src/core/lib/transport/bdp_estimator.c | 10 ++-- src/core/lib/transport/bdp_estimator.h | 3 +- src/core/lib/transport/connectivity_state.c | 12 ++--- src/core/lib/transport/connectivity_state.h | 3 +- src/core/tsi/fake_transport_security.c | 8 ++-- src/core/tsi/ssl_transport_security.c | 2 +- src/core/tsi/transport_security.c | 2 +- src/core/tsi/transport_security.h | 3 +- src/core/tsi/transport_security_interface.h | 5 +- test/core/end2end/fuzzers/api_fuzzer.c | 4 ++ test/core/transport/connectivity_state_test.c | 2 +- 59 files changed, 312 insertions(+), 217 deletions(-) (limited to 'src/core') diff --git a/src/core/ext/filters/client_channel/channel_connectivity.c b/src/core/ext/filters/client_channel/channel_connectivity.c index 62f58fb278..f83670db82 100644 --- a/src/core/ext/filters/client_channel/channel_connectivity.c +++ b/src/core/ext/filters/client_channel/channel_connectivity.c @@ -132,7 +132,7 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w, gpr_mu_lock(&w->mu); if (due_to_completion) { - if (grpc_trace_operation_failures) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures)) { GRPC_LOG_IF_ERROR("watch_completion_error", GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 37468101f0..a15476dc23 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -138,7 +138,7 @@ #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -int grpc_lb_glb_trace = 0; +grpc_tracer_flag grpc_lb_glb_trace; /* add lb_token of selected subchannel (address) to the call's initial * metadata */ @@ -224,7 +224,7 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg, } else { grpc_grpclb_client_stats_unref(wc_arg->client_stats); } - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Unreffing RR %p", (void *)wc_arg->rr_policy); } GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "wrapped_rr_closure"); @@ -575,7 +575,7 @@ static bool update_lb_connectivity_status_locked( GPR_ASSERT(new_rr_state_error == GRPC_ERROR_NONE); } - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Setting grpclb's state to %s from new RR policy %p state.", grpc_connectivity_state_name(new_rr_state), @@ -600,7 +600,7 @@ static bool pick_from_internal_rr_locked( (void **)&wc_arg->lb_token, &wc_arg->wrapper_closure); if (pick_done) { /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */ - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Unreffing RR (0x%" PRIxPTR ")", (intptr_t)wc_arg->rr_policy); } @@ -686,7 +686,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, if (!replace_old_rr) { /* dispose of the new RR policy that won't be used after all */ GRPC_LB_POLICY_UNREF(exec_ctx, new_rr_policy, "rr_handover_no_replace"); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Keeping old RR policy (%p) despite new serverlist: new RR " "policy was in %s connectivity state.", @@ -696,7 +696,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, return; } - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Created RR policy (%p) to replace old RR (%p)", (void *)new_rr_policy, (void *)glb_policy->rr_policy); } @@ -741,7 +741,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, pp->wrapped_on_complete_arg.rr_policy = glb_policy->rr_policy; pp->wrapped_on_complete_arg.client_stats = grpc_grpclb_client_stats_ref(glb_policy->client_stats); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Pending pick about to PICK from 0x%" PRIxPTR "", (intptr_t)glb_policy->rr_policy); } @@ -755,7 +755,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_policy->pending_pings = pping->next; GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping"); pping->wrapped_notify_arg.rr_policy = glb_policy->rr_policy; - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Pending ping about to PING from 0x%" PRIxPTR "", (intptr_t)glb_policy->rr_policy); } @@ -908,7 +908,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(uri->path[0] != '\0'); glb_policy->server_name = gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.", glb_policy->server_name); } @@ -1093,7 +1093,7 @@ static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, bool pick_done; if (glb_policy->rr_policy != NULL) { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "grpclb %p about to PICK from RR %p", (void *)glb_policy, (void *)glb_policy->rr_policy); } @@ -1116,7 +1116,7 @@ static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pick_done = pick_from_internal_rr_locked(exec_ctx, glb_policy->rr_policy, pick_args, target, wc_arg); } else { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_DEBUG, "No RR policy in grpclb instance %p. Adding to grpclb's pending " "picks", @@ -1347,7 +1347,7 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, lb_call_init_locked(exec_ctx, glb_policy); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Query for backends (grpclb: %p, lb_call: %p)", (void *)glb_policy, (void *)glb_policy->lb_call); } @@ -1453,7 +1453,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, gpr_time_max(gpr_time_from_seconds(1, GPR_TIMESPAN), grpc_grpclb_duration_to_timespec( &response->client_stats_report_interval)); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "received initial LB response message; " "client load reporting interval = %" PRId64 ".%09d sec", @@ -1466,7 +1466,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, glb_policy->client_load_report_timer_pending = true; GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report"); schedule_next_client_load_report(exec_ctx, glb_policy); - } else if (grpc_lb_glb_trace) { + } else if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "received initial LB response message; " "client load reporting NOT enabled"); @@ -1478,7 +1478,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_grpclb_response_parse_serverlist(response_slice); if (serverlist != NULL) { GPR_ASSERT(glb_policy->lb_call != NULL); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Serverlist with %lu servers received", (unsigned long)serverlist->num_servers); for (size_t i = 0; i < serverlist->num_servers; ++i) { @@ -1495,7 +1495,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (serverlist->num_servers > 0) { if (grpc_grpclb_serverlist_equals(glb_policy->serverlist, serverlist)) { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Incoming server list identical to current, ignoring."); } @@ -1513,7 +1513,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, rr_handover_locked(exec_ctx, glb_policy); } } else { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Received empty server list. Picks will stay pending until " "a response with > 0 servers is received"); @@ -1555,7 +1555,7 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, glb_lb_policy *glb_policy = arg; if (!glb_policy->shutting_down) { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Restaring call to LB server (grpclb %p)", (void *)glb_policy); } @@ -1572,7 +1572,7 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, GPR_ASSERT(glb_policy->lb_call != NULL); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { char *status_details = grpc_slice_to_c_string(glb_policy->lb_call_status_details); gpr_log(GPR_DEBUG, @@ -1591,7 +1591,7 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec next_try = gpr_backoff_step(&glb_policy->lb_call_backoff_state, now); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_DEBUG, "Connection to LB server lost (grpclb: %p)...", (void *)glb_policy); gpr_timespec timeout = gpr_time_sub(next_try, now); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 4c17f9c082..075b7caf5c 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -74,7 +74,7 @@ typedef struct round_robin_lb_policy round_robin_lb_policy; -int grpc_lb_round_robin_trace = 0; +grpc_tracer_flag grpc_lb_round_robin_trace; /** List of entities waiting for a pick. * @@ -198,7 +198,7 @@ static void advance_last_picked_locked(round_robin_lb_policy *p) { GPR_ASSERT(p->ready_list_last_pick == &p->ready_list); } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[READYLIST, RR: %p] ADVANCED LAST PICK. NOW AT NODE %p (SC %p, " "CSC %p)", @@ -228,7 +228,7 @@ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p, p->ready_list.prev->next = new_elem; p->ready_list.prev = new_elem; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (Conn. SC %p)", (void *)new_elem, (void *)sd->subchannel); } @@ -256,7 +256,7 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p, node->next->prev = node->prev; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", (void *)node, (void *)node->subchannel); } @@ -276,7 +276,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { round_robin_lb_policy *p = (round_robin_lb_policy *)pol; ready_list *elem; - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "Destroying Round Robin policy at %p", (void *)pol); } @@ -312,7 +312,7 @@ static void rr_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { pending_pick *pp; size_t i; - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "Shutting down Round Robin policy at %p", (void *)pol); } @@ -421,7 +421,7 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pending_pick *pp; ready_list *selected; - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_INFO, "Round Robin %p trying to pick", (void *)pol); } @@ -434,7 +434,7 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, if (user_data != NULL) { *user_data = selected->user_data; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", (void *)*target, (void *)selected); @@ -566,7 +566,7 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, if (pp->user_data != NULL) { *pp->user_data = selected->user_data; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", (void *)selected->subchannel, (void *)selected); @@ -724,7 +724,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sc_args.args = new_args; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { char *address_uri = grpc_sockaddr_to_uri(&addresses->addresses[i].address); gpr_log(GPR_DEBUG, "Created subchannel %p for address uri %s", @@ -768,7 +768,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE, "round_robin"); - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "Created RR policy at %p with %lu subchannels", (void *)p, (unsigned long)p->num_subchannels); } diff --git a/src/core/ext/filters/http/http_filters_plugin.c b/src/core/ext/filters/http/http_filters_plugin.c index 195a1a8119..856a7dbd91 100644 --- a/src/core/ext/filters/http/http_filters_plugin.c +++ b/src/core/ext/filters/http/http_filters_plugin.c @@ -37,6 +37,7 @@ #include "src/core/ext/filters/http/message_compress/message_compress_filter.h" #include "src/core/ext/filters/http/server/http_server_filter.h" #include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/transport/transport_impl.h" diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 1da8cf69cb..5a54a6ed15 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -47,6 +47,7 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" +#include "src/core/lib/surface/call.h" #include "src/core/lib/transport/static_metadata.h" #define INITIAL_METADATA_UNSEEN 0 @@ -197,7 +198,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, did_compress = grpc_msg_compress(exec_ctx, calld->compression_algorithm, &calld->slices, &tmp); if (did_compress) { - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name; const size_t before_size = calld->slices.length; const size_t after_size = tmp.length; @@ -211,7 +212,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_swap(&calld->slices, &tmp); calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.h b/src/core/ext/filters/http/message_compress/message_compress_filter.h index 75bfa17fba..135da4da62 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.h +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.h @@ -38,8 +38,6 @@ #include "src/core/lib/channel/channel_stack.h" -extern int grpc_compression_trace; - /** Compression filter for outgoing data. * * See for the available compression settings. diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index d6b79bd492..c5c0ec316d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -89,8 +89,8 @@ static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; #define MAX_CLIENT_STREAM_ID 0x7fffffffu -int grpc_http_trace = 0; -int grpc_flowctl_trace = 0; +grpc_tracer_flag grpc_http_trace; +grpc_tracer_flag grpc_flowctl_trace; static const grpc_transport_vtable vtable; @@ -1095,7 +1095,7 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, return; } closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { const char *errstr = grpc_error_string(error); gpr_log(GPR_DEBUG, "complete_closure_step: %p refs=%d flags=0x%04x desc=%s err=%s", @@ -1240,7 +1240,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, grpc_transport_stream_op_batch_payload *op_payload = op->payload; grpc_chttp2_transport *t = s->t; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *str = grpc_transport_stream_op_batch_string(op); gpr_log(GPR_DEBUG, "perform_stream_op_locked: %s; on_complete = %p", str, op->on_complete); @@ -1483,7 +1483,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *str = grpc_transport_stream_op_batch_string(op); gpr_log(GPR_DEBUG, "perform_stream_op[s=%p/%d]: %s", s, s->id, str); gpr_free(str); @@ -2145,7 +2145,7 @@ static void update_bdp(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (delta == 0 || (bdp != 0 && delta > -1024 && delta < 1024)) { return; } - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "%s: update initial window size to %d", t->peer_string, (int)bdp); } @@ -2305,7 +2305,7 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = tp; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "%s: Start BDP ping", t->peer_string); } /* Reset the keepalive ping timer */ @@ -2318,7 +2318,7 @@ static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = tp; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "%s: Complete BDP ping", t->peer_string); } grpc_bdp_estimator_complete_ping(&t->bdp_estimator); @@ -2779,7 +2779,7 @@ static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_chttp2_stream_map_size(&t->stream_map) == 0) { /* Channel with no active streams: send a goaway to try and make it * disconnect cleanly */ - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - send goaway to free memory", t->peer_string); } @@ -2787,7 +2787,7 @@ static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Buffers full"), GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_ENHANCE_YOUR_CALM)); - } else if (error == GRPC_ERROR_NONE && grpc_resource_quota_trace) { + } else if (error == GRPC_ERROR_NONE && GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR " streams", @@ -2808,7 +2808,7 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, t->destructive_reclaimer_registered = false; if (error == GRPC_ERROR_NONE && n > 0) { grpc_chttp2_stream *s = grpc_chttp2_stream_map_rand(&t->stream_map); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - abandon stream id %d", t->peer_string, s->id); } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index c372174f2d..83b17d1936 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -34,11 +34,12 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/transport/transport.h" -extern int grpc_http_trace; -extern int grpc_flowctl_trace; +extern grpc_tracer_flag grpc_http_trace; +extern grpc_tracer_flag grpc_flowctl_trace; grpc_transport *grpc_create_chttp2_transport( grpc_exec_ctx *exec_ctx, const grpc_channel_args *channel_args, diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index e3cd70d3f3..dbaafb5929 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -218,18 +218,18 @@ grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, void *p, parser->incoming_settings[id] != parser->value) { t->initial_window_update += (int64_t)parser->value - parser->incoming_settings[id]; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "adding %d for initial_window change", (int)t->initial_window_update); } } parser->incoming_settings[id] = parser->value; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "CHTTP2:%s:%s: got setting %s = %d", t->is_client ? "CLI" : "SVR", t->peer_string, sp->name, parser->value); } - } else if (grpc_http_trace) { + } else if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_ERROR, "CHTTP2: Ignoring unknown setting %d (value %d)", parser->id, parser->value); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 1846a85fc6..bb98bc4a79 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -50,8 +50,6 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/http2_errors.h" -extern int grpc_http_trace; - typedef enum { NOT_BINARY, BINARY_BEGIN, @@ -666,7 +664,7 @@ static const uint8_t inverse_base64[256] = { /* emission helpers */ static grpc_error *on_hdr(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, grpc_mdelem md, int add_to_table) { - if (grpc_http_trace && !GRPC_MDELEM_IS_INTERNED(md)) { + if (GRPC_TRACER_ON(grpc_http_trace) && !GRPC_MDELEM_IS_INTERNED(md)) { char *k = grpc_slice_to_c_string(GRPC_MDKEY(md)); char *v = grpc_slice_to_c_string(GRPC_MDVALUE(md)); gpr_log( @@ -1052,7 +1050,7 @@ static grpc_error *parse_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, static grpc_error *finish_max_tbl_size(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index); } grpc_error *err = diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 0aaa4aebe5..bb5ce60872 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -629,13 +629,13 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, #define GRPC_CHTTP2_CLIENT_CONNECT_STRLEN \ (sizeof(GRPC_CHTTP2_CLIENT_CONNECT_STRING) - 1) -extern int grpc_http_trace; -extern int grpc_flowctl_trace; +extern grpc_tracer_flag grpc_http_trace; +extern grpc_tracer_flag grpc_flowctl_trace; -#define GRPC_CHTTP2_IF_TRACING(stmt) \ - if (!(grpc_http_trace)) \ - ; \ - else \ +#define GRPC_CHTTP2_IF_TRACING(stmt) \ + if (!(GRPC_TRACER_ON(grpc_http_trace))) \ + ; \ + else \ stmt typedef enum { @@ -648,7 +648,7 @@ typedef enum { dst_var, src_context, src_var) \ do { \ assert(id1 == id2); \ - if (grpc_flowctl_trace) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace( \ __FILE__, __LINE__, phase, GRPC_CHTTP2_FLOWCTL_MOVE, #dst_context, \ #dst_var, #src_context, #src_var, transport->is_client, id1, \ @@ -671,7 +671,7 @@ typedef enum { #define GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ - if (grpc_flowctl_trace) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ GRPC_CHTTP2_FLOWCTL_CREDIT, #dst_context, \ #dst_var, NULL, #amount, transport->is_client, \ @@ -729,7 +729,7 @@ typedef enum { #define GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ - if (grpc_flowctl_trace) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ GRPC_CHTTP2_FLOWCTL_DEBIT, #dst_context, \ #dst_var, NULL, #amount, transport->is_client, \ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 638b137316..cee3c3f432 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -324,7 +324,7 @@ static grpc_error *init_frame_parser(grpc_exec_ctx *exec_ctx, case GRPC_CHTTP2_FRAME_GOAWAY: return init_goaway_parser(exec_ctx, t); default: - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_ERROR, "Unknown frame type %02x", t->incoming_frame_type); } return init_skip_frame_parser(exec_ctx, t, 0); @@ -494,7 +494,7 @@ static void on_initial_header(grpc_exec_ctx *exec_ctx, void *tp, GPR_ASSERT(s != NULL); - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *key = grpc_slice_to_c_string(GRPC_MDKEY(md)); char *value = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX | GPR_DUMP_ASCII); @@ -574,7 +574,7 @@ static void on_trailing_header(grpc_exec_ctx *exec_ctx, void *tp, GPR_ASSERT(s != NULL); - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *key = grpc_slice_to_c_string(GRPC_MDKEY(md)); char *value = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX | GPR_DUMP_ASCII); @@ -807,7 +807,7 @@ static grpc_error *parse_frame_slice(grpc_exec_ctx *exec_ctx, if (err == GRPC_ERROR_NONE) { return err; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { const char *msg = grpc_error_string(err); gpr_log(GPR_ERROR, "%s", msg); } diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 069780ae5a..54ff0aadad 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -74,7 +74,8 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_INFLIGHT])) { /* ping already in-flight: wait */ - if (grpc_http_trace || grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: already pinging", t->peer_string); } return; @@ -82,7 +83,8 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, if (t->ping_state.pings_before_data_required == 0 && t->ping_policy.max_pings_without_data != 0) { /* need to send something of substance before sending a ping again */ - if (grpc_http_trace || grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: too many recent pings: %d/%d", t->peer_string, t->ping_state.pings_before_data_required, t->ping_policy.max_pings_without_data); @@ -96,7 +98,8 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, (int)t->ping_policy.min_time_between_pings.tv_nsec);*/ if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < 0) { /* not enough elapsed time between successive pings */ - if (grpc_http_trace || grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: not enough time elapsed since last ping", t->peer_string); diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 94382980eb..107d5997fd 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -38,7 +38,7 @@ #include #include -int grpc_trace_channel = 0; +grpc_tracer_flag grpc_trace_channel; /* Memory layouts. diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index fdbcbdb018..c26d61b2ef 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -307,10 +307,10 @@ void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx, grpc_call_element *cur_elem, grpc_error *error); -extern int grpc_trace_channel; +extern grpc_tracer_flag grpc_trace_channel; #define GRPC_CALL_LOG_OP(sev, elem, op) \ - if (grpc_trace_channel) grpc_call_log_op(sev, elem, op) + if (GRPC_TRACER_ON(grpc_trace_channel)) grpc_call_log_op(sev, elem, op) #ifdef __cplusplus } diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index 88c02edb70..332635f58f 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -38,7 +38,7 @@ #include #include -int grpc_trace_channel_stack_builder = 0; +grpc_tracer_flag grpc_trace_channel_stack_builder; typedef struct filter_node { struct filter_node *next; diff --git a/src/core/lib/channel/channel_stack_builder.h b/src/core/lib/channel/channel_stack_builder.h index c78111b00d..8cb36eb117 100644 --- a/src/core/lib/channel/channel_stack_builder.h +++ b/src/core/lib/channel/channel_stack_builder.h @@ -165,7 +165,7 @@ grpc_error *grpc_channel_stack_builder_finish( void grpc_channel_stack_builder_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder); -extern int grpc_trace_channel_stack_builder; +extern grpc_tracer_flag grpc_trace_channel_stack_builder; #ifdef __cplusplus } diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c index c56046785b..54653868b8 100644 --- a/src/core/lib/debug/trace.c +++ b/src/core/lib/debug/trace.c @@ -42,17 +42,23 @@ typedef struct tracer { const char *name; - int *flag; + grpc_tracer_flag *flag; struct tracer *next; } tracer; static tracer *tracers; -void grpc_register_tracer(const char *name, int *flag) { +#ifdef GRPC_THREADSAFE_TRACER +#define TRACER_SET(flag, on) gpr_atm_no_barrier_store(&(flag).value, (on)) +#else +#define TRACER_SET(flag, on) (flag).value = (on) +#endif + +void grpc_register_tracer(const char *name, grpc_tracer_flag *flag) { tracer *t = gpr_malloc(sizeof(*t)); t->name = name; t->flag = flag; t->next = tracers; - *flag = 0; + TRACER_SET(*flag, false); tracers = t; } @@ -121,13 +127,13 @@ int grpc_tracer_set_enabled(const char *name, int enabled) { tracer *t; if (0 == strcmp(name, "all")) { for (t = tracers; t; t = t->next) { - *t->flag = enabled; + TRACER_SET(*t->flag, enabled); } } else { int found = 0; for (t = tracers; t; t = t->next) { if (0 == strcmp(name, t->name)) { - *t->flag = enabled; + TRACER_SET(*t->flag, enabled); found = 1; } } diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index 7afc38db7e..bf3bfee306 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -34,9 +34,33 @@ #ifndef GRPC_CORE_LIB_DEBUG_TRACE_H #define GRPC_CORE_LIB_DEBUG_TRACE_H +#include #include +#include -void grpc_register_tracer(const char *name, int *flag); +#if defined(__has_feature) && __has_feature(thread_sanitizer) +#define GRPC_THREADSAFE_TRACER +#endif + +typedef struct { +#ifdef GRPC_THREADSAFE_TRACER + gpr_atm value; +#else + bool value; +#endif +} grpc_tracer_flag; + +#ifdef GRPC_THREADSAFE_TRACER +#define GRPC_TRACER_ON(flag) (gpr_atm_no_barrier_load(&(flag).value) != 0) +#define GRPC_TRACER_INITIALIZER(on) \ + { (gpr_atm)(on) } +#else +#define GRPC_TRACER_ON(flag) ((flag).value) +#define GRPC_TRACER_INITIALIZER(on) \ + { (on) } +#endif + +void grpc_register_tracer(const char *name, grpc_tracer_flag *flag); void grpc_tracer_init(const char *env_var_name); void grpc_tracer_shutdown(void); diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index aac506b800..b8883f5953 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -40,7 +40,7 @@ #include #include -int grpc_http1_trace = 0; +grpc_tracer_flag grpc_http1_trace; static char *buf2str(void *buffer, size_t length) { char *out = gpr_malloc(length + 1); @@ -308,7 +308,7 @@ static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte, case GRPC_HTTP_FIRST_LINE: case GRPC_HTTP_HEADERS: if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) { - if (grpc_http1_trace) + if (GRPC_TRACER_ON(grpc_http1_trace)) gpr_log(GPR_ERROR, "HTTP header max line length (%d) exceeded", GRPC_HTTP_PARSER_MAX_HEADER_LENGTH); return GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h index a68011dd43..a155fecf11 100644 --- a/src/core/lib/http/parser.h +++ b/src/core/lib/http/parser.h @@ -36,6 +36,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/error.h" /* Maximum length of a header string of the form 'Key: Value\r\n' */ @@ -121,6 +122,6 @@ grpc_error *grpc_http_parser_eof(grpc_http_parser *parser); void grpc_http_request_destroy(grpc_http_request *request); void grpc_http_response_destroy(grpc_http_response *response); -extern int grpc_http1_trace; +extern grpc_tracer_flag grpc_http1_trace; #endif /* GRPC_CORE_LIB_HTTP_PARSER_H */ diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 05cdbdad2b..fa7d576133 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -42,13 +42,13 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" -int grpc_combiner_trace = 0; +grpc_tracer_flag grpc_combiner_trace; -#define GRPC_COMBINER_TRACE(fn) \ - do { \ - if (grpc_combiner_trace) { \ - fn; \ - } \ +#define GRPC_COMBINER_TRACE(fn) \ + do { \ + if (GRPC_TRACER_ON(grpc_combiner_trace)) { \ + fn; \ + } \ } while (0) #define STATE_UNORPHANED 1 diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 75dcb0b70a..6ab7a2b26b 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -37,6 +37,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/mpscq.h" @@ -78,6 +79,6 @@ grpc_closure_scheduler *grpc_combiner_finally_scheduler(grpc_combiner *lock, bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx); -extern int grpc_combiner_trace; +extern grpc_tracer_flag grpc_combiner_trace; #endif /* GRPC_CORE_LIB_IOMGR_COMBINER_H */ diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 445f7aa422..37804d8600 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -66,6 +66,9 @@ void grpc_iomgr_init(void) { g_root_object.name = "root"; grpc_network_status_init(); grpc_iomgr_platform_init(); +} + +void grpc_iomgr_start(void) { grpc_timer_manager_init(); } diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h index 245a1e08aa..6e2e023615 100644 --- a/src/core/lib/iomgr/iomgr.h +++ b/src/core/lib/iomgr/iomgr.h @@ -40,6 +40,9 @@ /** Initializes the iomgr. */ void grpc_iomgr_init(void); +/** Starts any background threads for iomgr. */ +void grpc_iomgr_start(void); + /** Signals the intention to shutdown the iomgr. Expects to be able to flush * exec_ctx. */ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx); diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index c3ee878651..52ccfc0aad 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -44,7 +44,7 @@ #include "src/core/lib/iomgr/combiner.h" -int grpc_resource_quota_trace = 0; +grpc_tracer_flag grpc_resource_quota_trace; #define MEMORY_USAGE_ESTIMATION_MAX 65536 @@ -307,13 +307,13 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, resource_user->free_pool = 0; resource_quota->free_pool -= amt; rq_update_estimate(resource_quota); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64 " bytes; rq_free_pool -> %" PRId64, resource_quota->name, resource_user->name, amt, resource_quota->free_pool); } - } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) { + } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && resource_user->free_pool >= 0) { gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request", resource_quota->name, resource_user->name); } @@ -342,7 +342,7 @@ static bool rq_reclaim_from_per_user_free_pool( resource_user->free_pool = 0; resource_quota->free_pool += amt; rq_update_estimate(resource_quota); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 " bytes; rq_free_pool -> %" PRId64, resource_quota->name, resource_user->name, amt, @@ -365,7 +365,7 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx, : GRPC_RULIST_RECLAIMER_BENIGN; grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list); if (resource_user == NULL) return false; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation", resource_quota->name, resource_user->name, destructive ? "destructive" : "benign"); @@ -786,7 +786,7 @@ void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&resource_user->mu); ru_ref_by(resource_user, (gpr_atm)size); resource_user->free_pool -= (int64_t)size; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, resource_user->free_pool); @@ -810,7 +810,7 @@ void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&resource_user->mu); bool was_zero_or_negative = resource_user->free_pool <= 0; resource_user->free_pool += (int64_t)size; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, resource_user->free_pool); @@ -839,7 +839,7 @@ void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx, grpc_resource_user *resource_user) { - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete", resource_user->resource_quota->name, resource_user->name); } diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index 6f99be0d51..51122dad01 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -36,6 +36,7 @@ #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" /** \file Tracks resource usage against a pool. @@ -75,7 +76,7 @@ maintain lists of users (which users arrange to leave before they are destroyed) */ -extern int grpc_resource_quota_trace; +extern grpc_tracer_flag grpc_resource_quota_trace; grpc_resource_quota *grpc_resource_quota_ref_internal( grpc_resource_quota *resource_quota); diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index a2692707d9..ed3fd94a98 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -58,7 +58,7 @@ #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/support/string.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; typedef struct { gpr_mu mu; @@ -114,7 +114,7 @@ done: static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { int done; async_connect *ac = acp; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", ac->addr_str, str); @@ -152,7 +152,7 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { GRPC_ERROR_REF(error); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: error=%s", ac->addr_str, str); @@ -330,7 +330,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, grpc_schedule_on_exec_ctx); ac->channel_args = grpc_channel_args_copy(channel_args); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", ac->addr_str); } diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 5f4b38de2b..233dea9bec 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -74,7 +74,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type; typedef size_t msg_iovlen_type; #endif -int grpc_tcp_trace = 0; +grpc_tracer_flag grpc_tcp_trace; typedef struct { grpc_endpoint base; @@ -221,7 +221,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, grpc_error *error) { grpc_closure *cb = tcp->read_cb; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -468,14 +468,14 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, } if (!tcp_flush(tcp, &error)) { - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "write: delayed"); } grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { cb = tcp->write_cb; tcp->write_cb = NULL; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write: %s", str); } @@ -490,7 +490,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_tcp *tcp = (grpc_tcp *)ep; grpc_error *error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; for (i = 0; i < buf->count; i++) { @@ -521,12 +521,12 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, if (!tcp_flush(tcp, &error)) { TCP_REF(tcp, "write"); tcp->write_cb = cb; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "write: delayed"); } grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write: %s", str); } diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index 1ad5788331..4ad60c116e 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -44,10 +44,11 @@ otherwise specified. */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/ev_posix.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; /* Create a tcp endpoint given a file desciptor and a read slice size. Takes ownership of fd. */ diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index e66ffc9b1c..08997b5e2b 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -257,7 +257,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { addr_str = grpc_sockaddr_to_uri(&addr); gpr_asprintf(&name, "tcp-server-connection:%s", addr_str); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "SERVER_CONNECT: incoming connection: %s", addr_str); } diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index d8e6068431..ec0bb169cb 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -56,8 +56,8 @@ #define MIN_QUEUE_WINDOW_DURATION 0.01 #define MAX_QUEUE_WINDOW_DURATION 1 -int grpc_timer_trace = 0; -int grpc_timer_check_trace = 0; +grpc_tracer_flag grpc_timer_trace; +grpc_tracer_flag grpc_timer_check_trace; typedef struct { gpr_mu mu; @@ -232,14 +232,13 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, GPR_ASSERT(deadline.clock_type == g_clock_type); GPR_ASSERT(now.clock_type == g_clock_type); timer->closure = closure; - timer->deadline = timespec_to_atm_round_up(deadline); + gpr_atm deadline_atm = timer->deadline = timespec_to_atm_round_up(deadline); - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, "TIMER %p: SET %" PRId64 ".%09d [%" PRIdPTR "] now %" PRId64 ".%09d [%" PRIdPTR "] call %p[%p]", - timer, deadline.tv_sec, deadline.tv_nsec, timer->deadline, - now.tv_sec, now.tv_nsec, timespec_to_atm_round_down(now), closure, - closure->cb); + timer, deadline.tv_sec, deadline.tv_nsec, deadline_atm, now.tv_sec, + now.tv_nsec, timespec_to_atm_round_down(now), closure, closure->cb); } if (!g_shared_mutables.initialized) { @@ -262,13 +261,13 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, grpc_time_averaged_stats_add_sample(&shard->stats, ts_to_dbl(gpr_time_sub(deadline, now))); - if (timer->deadline < shard->queue_deadline_cap) { + if (deadline_atm < shard->queue_deadline_cap) { is_first_timer = grpc_timer_heap_add(&shard->heap, timer); } else { timer->heap_index = INVALID_HEAP_INDEX; list_join(&shard->list, timer); } - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, " .. add to shard %d with queue_deadline_cap=%" PRIdPTR " => is_first_timer=%s", (int)(shard - g_shards), shard->queue_deadline_cap, @@ -289,16 +288,16 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, grpc_timer_check. */ if (is_first_timer) { gpr_mu_lock(&g_shared_mutables.mu); - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, " .. old shard min_deadline=%" PRIdPTR, shard->min_deadline); } - if (timer->deadline < shard->min_deadline) { + if (deadline_atm < shard->min_deadline) { gpr_atm old_min_deadline = g_shard_queue[0]->min_deadline; - shard->min_deadline = timer->deadline; + shard->min_deadline = deadline_atm; note_deadline_change(shard); - if (shard->shard_queue_index == 0 && timer->deadline < old_min_deadline) { - gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, timer->deadline); + if (shard->shard_queue_index == 0 && deadline_atm < old_min_deadline) { + gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, deadline_atm); grpc_kick_poller(); } } @@ -319,7 +318,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)]; gpr_mu_lock(&shard->mu); - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer, timer->pending ? "true" : "false"); } @@ -355,7 +354,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) { saturating_add(GPR_MAX(now, shard->queue_deadline_cap), (gpr_atm)(deadline_delta * 1000.0)); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. shard[%d]->queue_deadline_cap --> %" PRIdPTR, (int)(shard - g_shards), shard->queue_deadline_cap); } @@ -363,7 +362,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) { next = timer->next; if (timer->deadline < shard->queue_deadline_cap) { - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. add timer with deadline %" PRIdPTR " to heap", timer->deadline); } @@ -380,7 +379,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) { static grpc_timer *pop_one(shard_type *shard, gpr_atm now) { grpc_timer *timer; for (;;) { - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. shard[%d]: heap_empty=%s", (int)(shard - g_shards), grpc_timer_heap_is_empty(&shard->heap) ? "true" : "false"); @@ -390,13 +389,13 @@ static grpc_timer *pop_one(shard_type *shard, gpr_atm now) { if (!refill_queue(shard, now)) return NULL; } timer = grpc_timer_heap_top(&shard->heap); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. check top timer deadline=%" PRIdPTR " now=%" PRIdPTR, timer->deadline, now); } if (timer->deadline > now) return NULL; - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, "TIMER %p: FIRE %" PRIdPTR "ms late", timer, now - timer->deadline); } @@ -436,7 +435,7 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now, if (gpr_spinlock_trylock(&g_shared_mutables.checker_mu)) { gpr_mu_lock(&g_shared_mutables.mu); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. shard[%d]->min_deadline = %" PRIdPTR, (int)(g_shard_queue[0] - g_shards), g_shard_queue[0]->min_deadline); @@ -452,7 +451,7 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now, n += pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline, error); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. popped --> %" PRIdPTR ", shard[%d]->min_deadline %" PRIdPTR " --> %" PRIdPTR ", now=%" PRIdPTR, @@ -509,7 +508,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, *next = atm_to_timespec(GPR_MIN(timespec_to_atm_round_up(*next), min_timer)); } - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, "TIMER CHECK SKIP: now_atm=%" PRIdPTR " min_timer=%" PRIdPTR, now_atm, min_timer); @@ -523,7 +522,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, : GRPC_ERROR_CREATE_FROM_STATIC_STRING("Shutting down timer system"); // tracing - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { char *next_str; if (next == NULL) { next_str = gpr_strdup("NULL"); @@ -549,7 +548,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, *next = atm_to_timespec(next_atm); } // tracing - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { char *next_str; if (next == NULL) { next_str = gpr_strdup("NULL"); diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 00e868de01..1d83341299 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -45,11 +45,11 @@ typedef struct completed_thread { } completed_thread; static gpr_mu g_mu; +static bool g_threaded; static gpr_cv g_cv_wait; static gpr_cv g_cv_shutdown; static int g_thread_count; static int g_waiter_count; -static bool g_shutdown; static completed_thread *g_completed_threads; static bool g_kicked; @@ -83,6 +83,14 @@ static void start_timer_thread_and_unlock(void) { gpr_thd_new(&thd, timer_thread, NULL, &opt); } +void grpc_timer_manager_tick() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + grpc_timer_check(&exec_ctx, now, &next); + grpc_exec_ctx_finish(&exec_ctx); +} + static void timer_thread(void *unused) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); @@ -93,7 +101,7 @@ static void timer_thread(void *unused) { gpr_mu_lock(&g_mu); --g_waiter_count; bool start_thread = g_waiter_count == 0; - if (start_thread && !g_shutdown) { + if (start_thread && g_threaded) { start_timer_thread_and_unlock(); } else { gpr_mu_unlock(&g_mu); @@ -105,7 +113,7 @@ static void timer_thread(void *unused) { gpr_mu_unlock(&g_mu); } else { gpr_mu_lock(&g_mu); - if (g_shutdown) break; + if (!g_threaded) break; if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) { if (g_kicked) { grpc_timer_consume_kick(); @@ -130,34 +138,58 @@ static void timer_thread(void *unused) { gpr_log(GPR_DEBUG, "End timer thread"); } +static void start_threads(void) { + gpr_mu_lock(&g_mu); + if (!g_threaded) { + g_threaded = true; + start_timer_thread_and_unlock(); + } else { + g_threaded = false; + gpr_mu_unlock(&g_mu); + } +} + void grpc_timer_manager_init(void) { gpr_mu_init(&g_mu); gpr_cv_init(&g_cv_wait); gpr_cv_init(&g_cv_shutdown); + g_threaded = false; g_thread_count = 0; g_waiter_count = 0; - g_shutdown = false; g_completed_threads = NULL; - gpr_mu_lock(&g_mu); - start_timer_thread_and_unlock(); + start_threads(); } -void grpc_timer_manager_shutdown(void) { +static void stop_threads(void) { gpr_mu_lock(&g_mu); - g_shutdown = true; - gpr_cv_broadcast(&g_cv_wait); - while (g_thread_count > 0) { - gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); - gc_completed_threads(); + if (g_threaded) { + g_threaded = false; + gpr_cv_broadcast(&g_cv_wait); + while (g_thread_count > 0) { + gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + gc_completed_threads(); + } } gpr_mu_unlock(&g_mu); +} + +void grpc_timer_manager_shutdown(void) { + stop_threads(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_cv_wait); gpr_cv_destroy(&g_cv_shutdown); } +void grpc_timer_manager_set_threading(bool threaded) { + if (threaded) { + start_threads(); + } else { + stop_threads(); + } +} + void grpc_kick_poller(void) { gpr_mu_lock(&g_mu); g_kicked = true; diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h index a24c9da328..0b21262b1a 100644 --- a/src/core/lib/iomgr/timer_manager.h +++ b/src/core/lib/iomgr/timer_manager.h @@ -34,10 +34,19 @@ #ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H #define GRPC_CORE_IOMGR_TIMER_MANAGER_H +#include + /* Timer Manager tries to keep one thread waiting for the next timeout at all times */ void grpc_timer_manager_init(void); void grpc_timer_manager_shutdown(void); +/* enable/disable threading - must be called after grpc_timer_manager_init and + * before grpc_timer_manager_shutdown */ +void grpc_timer_manager_set_threading(bool enabled); +/* explicitly perform one tick of the timer system - for when threading is + * disabled */ +void grpc_timer_manager_tick(void); + #endif diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c index 178ce89aa6..0e7c1afb02 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.c +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c @@ -171,7 +171,7 @@ static char *redact_private_key(const char *json_key) { grpc_call_credentials *grpc_service_account_jwt_access_credentials_create( const char *json_key, gpr_timespec token_lifetime, void *reserved) { - if (grpc_api_trace) { + if (GRPC_TRACER_ON(grpc_api_trace)) { char *clean_json = redact_private_key(json_key); gpr_log(GPR_INFO, "grpc_service_account_jwt_access_credentials_create(" diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index ccfb3566c1..29235b6eb3 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -412,7 +412,7 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create( const char *json_refresh_token, void *reserved) { grpc_auth_refresh_token token = grpc_auth_refresh_token_create_from_string(json_refresh_token); - if (grpc_api_trace) { + if (GRPC_TRACER_ON(grpc_api_trace)) { char *loggable_token = create_loggable_refresh_token(&token); gpr_log(GPR_INFO, "grpc_refresh_token_credentials_create(json_refresh_token=%s, " diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index 1f0daf7325..dff05633ec 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -253,7 +253,7 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_linked_mdelem *l; grpc_client_security_context *sec_ctx = NULL; - if (calld->security_context_set == 0 && !op->cancel_stream) { + if (!op->cancel_stream && calld->security_context_set == 0) { calld->security_context_set = 1; GPR_ASSERT(op->payload->context != NULL); if (op->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) { diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 0d5c7432c6..c8dc50206a 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -75,7 +75,7 @@ typedef struct { gpr_refcount ref; } secure_endpoint; -int grpc_trace_secure_endpoint = 0; +grpc_tracer_flag grpc_trace_secure_endpoint; static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; @@ -137,7 +137,7 @@ static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur, static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, grpc_error *error) { - if (grpc_trace_secure_endpoint) { + if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) { size_t i; for (i = 0; i < ep->read_buffer->count; i++) { char *data = grpc_dump_slice(ep->read_buffer->slices[i], @@ -269,7 +269,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &ep->output_buffer); - if (grpc_trace_secure_endpoint) { + if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) { for (i = 0; i < slices->count; i++) { char *data = grpc_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); diff --git a/src/core/lib/security/transport/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h index a61f40a4fa..f1a5c8cb6d 100644 --- a/src/core/lib/security/transport/secure_endpoint.h +++ b/src/core/lib/security/transport/secure_endpoint.h @@ -39,7 +39,7 @@ struct tsi_frame_protector; -extern int grpc_trace_secure_endpoint; +extern grpc_tracer_flag grpc_trace_secure_endpoint; /* Takes ownership of protector and to_wrap, and refs leftover_slices. */ grpc_endpoint *grpc_secure_endpoint_create( diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 79e3e5ca9b..3b20e042b4 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -33,4 +33,4 @@ #include "src/core/lib/surface/api_trace.h" -int grpc_api_trace = 0; +grpc_tracer_flag grpc_api_trace; diff --git a/src/core/lib/surface/api_trace.h b/src/core/lib/surface/api_trace.h index c60aaba5e9..d4fbc8d90d 100644 --- a/src/core/lib/surface/api_trace.h +++ b/src/core/lib/surface/api_trace.h @@ -37,7 +37,7 @@ #include #include "src/core/lib/debug/trace.h" -extern int grpc_api_trace; +extern grpc_tracer_flag grpc_api_trace; /* Provide unwrapping macros because we're in C89 and variadic macros weren't introduced until C99... */ @@ -58,7 +58,7 @@ extern int grpc_api_trace; /* Due to the limitations of C89's preprocessor, the arity of the var-arg list 'nargs' must be specified. */ #define GRPC_API_TRACE(fmt, nargs, args) \ - if (grpc_api_trace) { \ + if (GRPC_TRACER_ON(grpc_api_trace)) { \ gpr_log(GPR_INFO, fmt GRPC_API_TRACE_UNWRAP##nargs args); \ } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 7525806583..fbced00860 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -244,8 +244,8 @@ struct grpc_call { void *saved_receiving_stream_ready_bctlp; }; -int grpc_call_error_trace = 0; -int grpc_compression_trace = 0; +grpc_tracer_flag grpc_call_error_trace; +grpc_tracer_flag grpc_compression_trace; #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) @@ -702,7 +702,7 @@ static void get_final_status(grpc_call *call, for (i = 0; i < STATUS_SOURCE_COUNT; i++) { status[i] = unpack_received_status(gpr_atm_acq_load(&call->status[i])); } - if (grpc_call_error_trace) { + if (GRPC_TRACER_ON(grpc_call_error_trace)) { gpr_log(GPR_DEBUG, "get_final_status %s", call->is_client ? "CLI" : "SVR"); for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (status[i].is_set) { @@ -1259,7 +1259,7 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } if (error != GRPC_ERROR_NONE) { - if (grpc_trace_operation_failures) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures)) { GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); } grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); @@ -1355,8 +1355,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, GPR_ASSERT(call->encodings_accepted_by_peer != 0); if (!GPR_BITGET(call->encodings_accepted_by_peer, call->incoming_compression_algorithm)) { - extern int grpc_compression_trace; - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name = NULL; grpc_compression_algorithm_name(call->incoming_compression_algorithm, &algo_name); diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 7d4d0db28d..256a5fa2fe 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -117,7 +117,8 @@ void grpc_call_context_set(grpc_call *call, grpc_context_index elem, void *grpc_call_context_get(grpc_call *call, grpc_context_index elem); #define GRPC_CALL_LOG_BATCH(sev, call, ops, nops, tag) \ - if (grpc_api_trace) grpc_call_log_batch(sev, call, ops, nops, tag) + if (GRPC_TRACER_ON(grpc_api_trace)) \ + grpc_call_log_batch(sev, call, ops, nops, tag) uint8_t grpc_call_is_client(grpc_call *call); @@ -126,7 +127,8 @@ uint8_t grpc_call_is_client(grpc_call *call); grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level); -extern int grpc_call_error_trace; +extern grpc_tracer_flag grpc_call_error_trace; +extern grpc_tracer_flag grpc_compression_trace; #ifdef __cplusplus } diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 048564e32f..f31f0bc4c7 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -50,9 +50,9 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/event_string.h" -int grpc_trace_operation_failures; +grpc_tracer_flag grpc_trace_operation_failures; #ifndef NDEBUG -int grpc_trace_pending_tags; +grpc_tracer_flag grpc_trace_pending_tags; #endif typedef struct { @@ -242,15 +242,16 @@ struct grpc_completion_queue { #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) #define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1) -int grpc_cq_pluck_trace; -int grpc_cq_event_timeout_trace; +grpc_tracer_flag grpc_cq_pluck_trace = GRPC_TRACER_INITIALIZER(true); +grpc_tracer_flag grpc_cq_event_timeout_trace = GRPC_TRACER_INITIALIZER(true); -#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \ - if (grpc_api_trace && \ - (grpc_cq_pluck_trace || (event)->type != GRPC_QUEUE_TIMEOUT)) { \ - char *_ev = grpc_event_string(event); \ - gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \ - gpr_free(_ev); \ +#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \ + if (GRPC_TRACER_ON(grpc_api_trace) && \ + (GRPC_TRACER_ON(grpc_cq_pluck_trace) || \ + (event)->type != GRPC_QUEUE_TIMEOUT)) { \ + char *_ev = grpc_event_string(event); \ + gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \ + gpr_free(_ev); \ } static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, @@ -375,14 +376,16 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, #endif GPR_TIMER_BEGIN("grpc_cq_end_op", 0); - if (grpc_api_trace || - (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) { + if (GRPC_TRACER_ON(grpc_api_trace) || + (GRPC_TRACER_ON(grpc_trace_operation_failures) && + error != GRPC_ERROR_NONE)) { const char *errmsg = grpc_error_string(error); GRPC_API_TRACE( "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, " "done_arg=%p, storage=%p)", 7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage)); - if (grpc_trace_operation_failures && error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures) && + error != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg); } } @@ -481,7 +484,7 @@ static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) { #ifndef NDEBUG static void dump_pending_tags(grpc_completion_queue *cc) { - if (!grpc_trace_pending_tags) return; + if (!GRPC_TRACER_ON(grpc_trace_pending_tags)) return; gpr_strvec v; gpr_strvec_init(&v); @@ -677,7 +680,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, abort(); } - if (grpc_cq_pluck_trace) { + if (GRPC_TRACER_ON(grpc_cq_pluck_trace)) { GRPC_API_TRACE( "grpc_completion_queue_pluck(" "cc=%p, tag=%p, " diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index a932087939..a750687b27 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -37,15 +37,16 @@ /* Internal API for completion queues */ #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/pollset.h" /* These trace flags default to 1. The corresponding lines are only traced if grpc_api_trace is also truthy */ -extern int grpc_cq_pluck_trace; -extern int grpc_cq_event_timeout_trace; -extern int grpc_trace_operation_failures; +extern grpc_tracer_flag grpc_cq_pluck_trace; +extern grpc_tracer_flag grpc_cq_event_timeout_trace; +extern grpc_tracer_flag grpc_trace_operation_failures; #ifndef NDEBUG -extern int grpc_trace_pending_tags; +extern grpc_tracer_flag grpc_trace_pending_tags; #endif typedef struct grpc_cq_completion { diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index 4b381b1954..6163776152 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -145,10 +145,8 @@ void grpc_init(void) { grpc_register_tracer("server_channel", &grpc_server_channel_trace); grpc_register_tracer("bdp_estimator", &grpc_bdp_estimator_trace); // Default pluck trace to 1 - grpc_cq_pluck_trace = 1; grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace); // Default timeout trace to 1 - grpc_cq_event_timeout_trace = 1; grpc_register_tracer("op_failure", &grpc_trace_operation_failures); grpc_register_tracer("resource_quota", &grpc_resource_quota_trace); grpc_register_tracer("call_error", &grpc_call_error_trace); @@ -173,6 +171,7 @@ void grpc_init(void) { grpc_tracer_init("GRPC_TRACE"); /* no more changes to channel init pipelines */ grpc_channel_init_finalize(); + grpc_iomgr_start(); } gpr_mu_unlock(&g_init_mu); GRPC_API_TRACE("grpc_init(void)", 0, ()); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 934ca0431a..99020926c9 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -73,7 +73,7 @@ typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; -int grpc_server_channel_trace = 0; +grpc_tracer_flag grpc_server_channel_trace; typedef struct requested_call { requested_call_type type; @@ -456,7 +456,7 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand, grpc_closure_init(&chand->finish_destroy_channel_closure, finish_destroy_channel, chand, grpc_schedule_on_exec_ctx); - if (grpc_server_channel_trace && error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_server_channel_trace) && error != GRPC_ERROR_NONE) { const char *msg = grpc_error_string(error); gpr_log(GPR_INFO, "Disconnected client: %s", msg); } diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index a85d9f4964..cd2fca0fe0 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -36,12 +36,13 @@ #include #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/transport/transport.h" extern const grpc_channel_filter grpc_server_top_filter; /** Lightweight tracing of server channel state */ -extern int grpc_server_channel_trace; +extern grpc_tracer_flag grpc_server_channel_trace; /* Add a listener to the server: when the server starts, it will call start, and when it shuts down, it will call destroy */ diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c index e1483677fd..c4c0078cf2 100644 --- a/src/core/lib/transport/bdp_estimator.c +++ b/src/core/lib/transport/bdp_estimator.c @@ -38,7 +38,7 @@ #include #include -int grpc_bdp_estimator_trace = 0; +grpc_tracer_flag grpc_bdp_estimator_trace; void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) { estimator->estimate = 65536; @@ -67,7 +67,7 @@ bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator, } void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } @@ -77,7 +77,7 @@ void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { } void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } @@ -87,14 +87,14 @@ void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) { } void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_STARTED); if (estimator->accumulator > 2 * estimator->estimate / 3) { estimator->estimate *= 2; - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]: estimate increased to %" PRId64, estimator->name, estimator->estimate); } diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h index df8d1f6fc0..1b125d8d61 100644 --- a/src/core/lib/transport/bdp_estimator.h +++ b/src/core/lib/transport/bdp_estimator.h @@ -36,11 +36,12 @@ #include #include +#include "src/core/lib/debug/trace.h" #define GRPC_BDP_SAMPLES 16 #define GRPC_BDP_MIN_SAMPLES_FOR_ESTIMATE 3 -extern int grpc_bdp_estimator_trace; +extern grpc_tracer_flag grpc_bdp_estimator_trace; typedef enum { GRPC_BDP_PING_UNSCHEDULED, diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 3757b25267..0b8dc703e2 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -39,7 +39,7 @@ #include #include -int grpc_connectivity_state_trace = 0; +grpc_tracer_flag grpc_connectivity_state_trace; const char *grpc_connectivity_state_name(grpc_connectivity_state state) { switch (state) { @@ -94,7 +94,7 @@ grpc_connectivity_state grpc_connectivity_state_check( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); } @@ -106,7 +106,7 @@ grpc_connectivity_state grpc_connectivity_state_get( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); } @@ -127,7 +127,7 @@ bool grpc_connectivity_state_notify_on_state_change( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { if (current == NULL) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker, tracker->name, notify); @@ -180,7 +180,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); grpc_connectivity_state_watcher *w; - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { const char *error_string = grpc_error_string(error); gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker, tracker->name, grpc_connectivity_state_name(cur), @@ -208,7 +208,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, while ((w = tracker->watchers) != NULL) { *w->current = state; tracker->watchers = w->next; - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "NOTIFY: %p %s: %p", tracker, tracker->name, w->notify); } diff --git a/src/core/lib/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h index c9604c34dd..cdc2930c11 100644 --- a/src/core/lib/transport/connectivity_state.h +++ b/src/core/lib/transport/connectivity_state.h @@ -35,6 +35,7 @@ #define GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" typedef struct grpc_connectivity_state_watcher { @@ -57,7 +58,7 @@ typedef struct { char *name; } grpc_connectivity_state_tracker; -extern int grpc_connectivity_state_trace; +extern grpc_tracer_flag grpc_connectivity_state_trace; /** enum --> string conversion */ const char *grpc_connectivity_state_name(grpc_connectivity_state state); diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index 1836beefc4..4925d19f96 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -396,7 +396,7 @@ static tsi_result fake_handshaker_get_bytes_to_send_to_peer( if (next_message_to_send > TSI_FAKE_HANDSHAKE_MESSAGE_MAX) { next_message_to_send = TSI_FAKE_HANDSHAKE_MESSAGE_MAX; } - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%s prepared %s.", impl->is_client ? "Client" : "Server", tsi_fake_handshake_message_to_string(impl->next_message_to_send)); @@ -408,7 +408,7 @@ static tsi_result fake_handshaker_get_bytes_to_send_to_peer( if (!impl->is_client && impl->next_message_to_send == TSI_FAKE_HANDSHAKE_MESSAGE_MAX) { /* We're done. */ - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "Server is done."); } impl->result = TSI_OK; @@ -445,7 +445,7 @@ static tsi_result fake_handshaker_process_bytes_from_peer( tsi_fake_handshake_message_to_string(received_msg), tsi_fake_handshake_message_to_string(expected_msg)); } - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%s received %s.", impl->is_client ? "Client" : "Server", tsi_fake_handshake_message_to_string(received_msg)); } @@ -453,7 +453,7 @@ static tsi_result fake_handshaker_process_bytes_from_peer( impl->needs_incoming_message = 0; if (impl->next_message_to_send == TSI_FAKE_HANDSHAKE_MESSAGE_MAX) { /* We're done. */ - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%s is done.", impl->is_client ? "Client" : "Server"); } impl->result = TSI_OK; diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index e1d634a1fa..59fd2b1c93 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -180,7 +180,7 @@ static const char *ssl_error_string(int error) { /* TODO(jboeuf): Remove when we are past the debugging phase with this code. */ static void ssl_log_where_info(const SSL *ssl, int where, int flag, const char *msg) { - if ((where & flag) && tsi_tracing_enabled) { + if ((where & flag) && GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%20.20s - %30.30s - %5.10s", msg, SSL_state_string_long(ssl), SSL_state_string(ssl)); } diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index b11c00c43c..9a2d4669c6 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -41,7 +41,7 @@ /* --- Tracing. --- */ -int tsi_tracing_enabled = 0; +grpc_tracer_flag tsi_tracing_enabled; /* --- tsi_result common implementation. --- */ diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index a4c9cbc001..2422f92076 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -36,13 +36,14 @@ #include +#include "src/core/lib/debug/trace.h" #include "src/core/tsi/transport_security_interface.h" #ifdef __cplusplus extern "C" { #endif -extern int tsi_tracing_enabled; +extern grpc_tracer_flag tsi_tracing_enabled; /* Base for tsi_frame_protector implementations. See transport_security_interface.h for documentation. */ diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index f2112b62b6..8a3fff6a17 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -37,6 +37,8 @@ #include #include +#include "src/core/lib/debug/trace.h" + #ifdef __cplusplus extern "C" { #endif @@ -73,8 +75,7 @@ const char *tsi_result_to_string(tsi_result result); /* --- tsi tracing --- */ -/* Set this early to avoid races */ -extern int tsi_tracing_enabled; +extern grpc_tracer_flag tsi_tracing_enabled; /* --- tsi_frame_protector object --- diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 88a0e301da..b33b43dac5 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -44,6 +44,7 @@ #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/tcp_client.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/transport/metadata.h" @@ -722,6 +723,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_tcp_client_connect_impl = my_tcp_client_connect; gpr_now_impl = now_impl; grpc_init(); + grpc_timer_manager_set_threading(false); grpc_resolve_address = my_resolve_address; GPR_ASSERT(g_channel == NULL); @@ -769,6 +771,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { g_now = gpr_time_add(g_now, gpr_time_from_seconds(1, GPR_TIMESPAN)); } + grpc_timer_manager_tick(); + switch (next_byte(&inp)) { // terminate on bad bytes default: diff --git a/test/core/transport/connectivity_state_test.c b/test/core/transport/connectivity_state_test.c index 8314a5f619..96db59ba2d 100644 --- a/test/core/transport/connectivity_state_test.c +++ b/test/core/transport/connectivity_state_test.c @@ -151,7 +151,7 @@ static void test_subscribe_with_failure_then_destroy(void) { int main(int argc, char **argv) { grpc_test_init(argc, argv); - grpc_connectivity_state_trace = 1; + grpc_connectivity_state_trace.value = 1; test_connectivity_state_name(); test_check(); test_subscribe_then_unsubscribe(); -- cgit v1.2.3 From f3003885911eb303db69bcfef98161b8326b7dae Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 3 May 2017 13:56:15 -0700 Subject: Better? feature test --- src/core/lib/debug/trace.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index bf3bfee306..ba432574d0 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -38,9 +38,11 @@ #include #include -#if defined(__has_feature) && __has_feature(thread_sanitizer) +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) #define GRPC_THREADSAFE_TRACER #endif +#endif typedef struct { #ifdef GRPC_THREADSAFE_TRACER -- cgit v1.2.3 From 9f276e19a8ea1cbf5d445ed44635ebb6142c62ec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 3 May 2017 14:07:10 -0700 Subject: Fix mac build --- src/core/lib/surface/api_trace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 3b20e042b4..55c69cd5b0 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -31,6 +31,7 @@ * */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/surface/api_trace.h" -grpc_tracer_flag grpc_api_trace; +grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false); -- cgit v1.2.3 From c0a9d1f4252763fe0a37ad9cb6046918ce94a034 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 18:24:06 -0700 Subject: Turnstile polling per dedicated epoll set --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 68 ++++++++++++++----------- 1 file changed, 38 insertions(+), 30 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index d11dbbfae8..8679f9c85d 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -110,7 +110,7 @@ static void fd_global_init(void); static void fd_global_shutdown(void); /******************************************************************************* - * Polling island Declarations + * epoll set Declarations */ #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG @@ -130,6 +130,10 @@ static void fd_global_shutdown(void); typedef struct epoll_set { grpc_closure_scheduler workqueue_scheduler; + /* Mutex poller should acquire to poll this. This enforces that only one + * poller can be polling on epoll_set at any time */ + gpr_mu mu; + /* Ref count. Use EPS_ADD_REF() and EPS_UNREF() macros to increment/decrement the refcount. Once the ref count becomes zero, this structure is destroyed which means we should ensure that there is never a scenario where a @@ -137,7 +141,7 @@ typedef struct epoll_set { zero. */ gpr_atm ref_count; - /* Number of threads currently polling on this island */ + /* Number of threads currently polling on this epoll set*/ gpr_atm poller_count; /* Mutex guarding the read end of the workqueue (must be held to pop from * workqueue_items) */ @@ -189,6 +193,7 @@ struct grpc_pollset_set {}; size_t g_num_eps = 1; struct epoll_set **g_epoll_sets = NULL; +gpr_atm g_next_eps; size_t g_num_threads_per_eps = 1; gpr_thd_id *g_poller_threads = NULL; @@ -219,16 +224,13 @@ static bool append_error(grpc_error **composite, grpc_error *error, } /******************************************************************************* - * Polling island Definitions + * epoll set Definitions */ -/* The wakeup fd that is used to wake up all threads in a Polling island. This - is useful in the epoll set merge operation where we need to wakeup all - the threads currently polling the smaller epoll set (so that they can - start polling the new/merged epoll set) - - NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the - threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +/* The wakeup fd that is used to wake up all threads in an epoll_set informing + that the epoll set is shutdown. This wakeup fd initialized to be readable + and MUST NOT be consumed i.e the threads that woke up MUST NOT call + grpc_wakeup_fd_consume_wakeup() */ static grpc_wakeup_fd epoll_set_wakeup_fd; /* The epoll set being polled right now. @@ -399,6 +401,7 @@ static epoll_set *epoll_set_create(grpc_error **error) { eps->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; eps->epoll_fd = -1; + gpr_mu_init(&eps->mu); gpr_mu_init(&eps->workqueue_read_mu); gpr_mpscq_init(&eps->workqueue_items); gpr_atm_rel_store(&eps->workqueue_item_count, 0); @@ -437,6 +440,7 @@ static void epoll_set_delete(epoll_set *eps) { } GPR_ASSERT(gpr_atm_no_barrier_load(&eps->workqueue_item_count) == 0); + gpr_mu_destroy(&eps->mu); gpr_mu_destroy(&eps->workqueue_read_mu); gpr_mpscq_destroy(&eps->workqueue_items); grpc_wakeup_fd_destroy(&eps->workqueue_wakeup_fd); @@ -897,6 +901,19 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, epoll_set *eps) { return false; } +/* Blocking call */ +static void acquire_epoll_lease(epoll_set *eps) { + if (g_num_threads_per_eps > 1) { + gpr_mu_lock(&eps->mu); + } +} + +static void release_epoll_lease(epoll_set *eps) { + if (g_num_threads_per_eps > 1) { + gpr_mu_unlock(&eps->mu); + } +} + #define GRPC_EPOLL_MAX_EVENTS 100 static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, grpc_error **error) { @@ -908,7 +925,9 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, int timeout_ms = -1; GRPC_SCHEDULING_START_BLOCKING_REGION; + acquire_epoll_lease(eps); ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); + release_epoll_lease(eps); GRPC_SCHEDULING_END_BLOCKING_REGION; if (ep_rv < 0) { @@ -961,11 +980,6 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, epoll set. */ epoll_fd = eps->epoll_fd; - /* Add an extra ref so that the island does not get destroyed (which means - the epoll_fd won't be closed) while we are are doing an epoll_wait() on the - epoll_fd */ - EPS_ADD_REF(eps, "ps_work"); - /* If we get some workqueue work to do, it might end up completing an item on the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ @@ -979,13 +993,6 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); } - /* Before leaving, release the extra ref we added to the epoll set. It - is important to use "eps" here (i.e our old copy of pollset->eps - that we got before releasing the epoll set lock). This is because - pollset->eps pointer might get udpated in other parts of the - code when there is an island merge while we are doing epoll_wait() above */ - EPS_UNREF(exec_ctx, eps, "ps_work"); - GPR_TIMER_END("epoll_set_work", 0); } @@ -1162,7 +1169,7 @@ static void add_fd_to_eps(grpc_fd *fd) { GPR_TIMER_BEGIN("add_fd_to_eps", 0); grpc_error *error = GRPC_ERROR_NONE; - size_t idx = ((size_t)rand()) % g_num_eps; + size_t idx = (size_t)gpr_atm_no_barrier_fetch_add(&g_next_eps, 1) % g_num_eps; epoll_set *eps = g_epoll_sets[idx]; gpr_mu_lock(&fd->mu); @@ -1176,8 +1183,7 @@ static void add_fd_to_eps(grpc_fd *fd) { EPS_ADD_REF(eps, "fd"); fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, - idx); + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, idx); gpr_mu_unlock(&fd->mu); GRPC_LOG_IF_ERROR("add_fd_to_eps", error); @@ -1203,6 +1209,7 @@ static bool init_epoll_sets() { EPS_ADD_REF(g_epoll_sets[i], "init_epoll_sets"); } + gpr_atm_no_barrier_store(&g_next_eps, 0); gpr_mu *mu; pollset_init(&g_read_notifier, &mu); @@ -1247,14 +1254,14 @@ static void start_poller_threads() { gpr_log(GPR_INFO, "Starting poller threads"); - /* One thread per pollset */ - g_poller_threads = (gpr_thd_id *)malloc(g_num_eps * sizeof(gpr_thd_id)); + size_t num_threads = g_num_eps * g_num_threads_per_eps; + g_poller_threads = (gpr_thd_id *)malloc(num_threads * sizeof(gpr_thd_id)); gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); - for (size_t i = 0; i < g_num_eps; i++) { + for (size_t i = 0; i < num_threads; i++) { gpr_thd_new(&g_poller_threads[i], poller_thread_loop, - (void *)g_epoll_sets[i], &options); + (void *)g_epoll_sets[i % g_num_eps], &options); } } @@ -1266,7 +1273,8 @@ static void shutdown_poller_threads() { gpr_log(GPR_INFO, "Shutting down pollers"); epoll_set *eps = NULL; - for (size_t i = 0; i < g_num_eps; i++) { + size_t num_threads = g_num_eps * g_num_threads_per_eps; + for (size_t i = 0; i < num_threads; i++) { eps = g_epoll_sets[i]; epoll_set_add_wakeup_fd_locked(eps, &epoll_set_wakeup_fd, &error); } -- cgit v1.2.3 From 3b654361537dee42dba9855c7d2501302c272c7a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 08:11:17 -0700 Subject: Add initializers for all tracers --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 2 +- .../ext/filters/client_channel/lb_policy/round_robin/round_robin.c | 2 +- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 7 ++++--- src/core/lib/channel/channel_stack.c | 2 +- src/core/lib/channel/channel_stack_builder.c | 3 ++- src/core/lib/http/parser.c | 2 +- src/core/lib/iomgr/combiner.c | 2 +- src/core/lib/iomgr/resource_quota.c | 5 +++-- src/core/lib/iomgr/tcp_posix.c | 2 +- src/core/lib/iomgr/timer_generic.c | 4 ++-- src/core/lib/security/transport/secure_endpoint.c | 2 +- src/core/lib/surface/call.c | 4 ++-- src/core/lib/surface/completion_queue.c | 4 ++-- src/core/lib/surface/server.c | 2 +- src/core/lib/transport/bdp_estimator.c | 2 +- src/core/lib/transport/connectivity_state.c | 2 +- src/core/tsi/transport_security.c | 2 +- 17 files changed, 26 insertions(+), 23 deletions(-) (limited to 'src/core') diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index a15476dc23..18b36dde8a 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -138,7 +138,7 @@ #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -grpc_tracer_flag grpc_lb_glb_trace; +grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false); /* add lb_token of selected subchannel (address) to the call's initial * metadata */ diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 075b7caf5c..6e7f410635 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -74,7 +74,7 @@ typedef struct round_robin_lb_policy round_robin_lb_policy; -grpc_tracer_flag grpc_lb_round_robin_trace; +grpc_tracer_flag grpc_lb_round_robin_trace = GRPC_TRACER_INITIALIZER(false); /** List of entities waiting for a pick. * diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index c5c0ec316d..ab74ff6ad3 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -89,8 +89,8 @@ static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; #define MAX_CLIENT_STREAM_ID 0x7fffffffu -grpc_tracer_flag grpc_http_trace; -grpc_tracer_flag grpc_flowctl_trace; +grpc_tracer_flag grpc_http_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_flowctl_trace = GRPC_TRACER_INITIALIZER(false); static const grpc_transport_vtable vtable; @@ -2787,7 +2787,8 @@ static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Buffers full"), GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_ENHANCE_YOUR_CALM)); - } else if (error == GRPC_ERROR_NONE && GRPC_TRACER_ON(grpc_resource_quota_trace)) { + } else if (error == GRPC_ERROR_NONE && + GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR " streams", diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 107d5997fd..7db54d1107 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -38,7 +38,7 @@ #include #include -grpc_tracer_flag grpc_trace_channel; +grpc_tracer_flag grpc_trace_channel = GRPC_TRACER_INITIALIZER(false); /* Memory layouts. diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index 332635f58f..44b030e4d1 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -38,7 +38,8 @@ #include #include -grpc_tracer_flag grpc_trace_channel_stack_builder; +grpc_tracer_flag grpc_trace_channel_stack_builder = + GRPC_TRACER_INITIALIZER(false); typedef struct filter_node { struct filter_node *next; diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index b8883f5953..a4357978e4 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -40,7 +40,7 @@ #include #include -grpc_tracer_flag grpc_http1_trace; +grpc_tracer_flag grpc_http1_trace = GRPC_TRACER_INITIALIZER(false); static char *buf2str(void *buffer, size_t length) { char *out = gpr_malloc(length + 1); diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index fa7d576133..863f22c614 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -42,7 +42,7 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" -grpc_tracer_flag grpc_combiner_trace; +grpc_tracer_flag grpc_combiner_trace = GRPC_TRACER_INITIALIZER(false); #define GRPC_COMBINER_TRACE(fn) \ do { \ diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 52ccfc0aad..6b2b85cce0 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -44,7 +44,7 @@ #include "src/core/lib/iomgr/combiner.h" -grpc_tracer_flag grpc_resource_quota_trace; +grpc_tracer_flag grpc_resource_quota_trace = GRPC_TRACER_INITIALIZER(false); #define MEMORY_USAGE_ESTIMATION_MAX 65536 @@ -313,7 +313,8 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, resource_quota->name, resource_user->name, amt, resource_quota->free_pool); } - } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && resource_user->free_pool >= 0) { + } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && + resource_user->free_pool >= 0) { gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request", resource_quota->name, resource_user->name); } diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 233dea9bec..5d360b0b80 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -74,7 +74,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type; typedef size_t msg_iovlen_type; #endif -grpc_tracer_flag grpc_tcp_trace; +grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { grpc_endpoint base; diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index ec0bb169cb..b28340b71c 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -56,8 +56,8 @@ #define MIN_QUEUE_WINDOW_DURATION 0.01 #define MAX_QUEUE_WINDOW_DURATION 1 -grpc_tracer_flag grpc_timer_trace; -grpc_tracer_flag grpc_timer_check_trace; +grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { gpr_mu mu; diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index c8dc50206a..48d368a2a7 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -75,7 +75,7 @@ typedef struct { gpr_refcount ref; } secure_endpoint; -grpc_tracer_flag grpc_trace_secure_endpoint; +grpc_tracer_flag grpc_trace_secure_endpoint = GRPC_TRACER_INITIALIZER(false); static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index fbced00860..1bd158f614 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -244,8 +244,8 @@ struct grpc_call { void *saved_receiving_stream_ready_bctlp; }; -grpc_tracer_flag grpc_call_error_trace; -grpc_tracer_flag grpc_compression_trace; +grpc_tracer_flag grpc_call_error_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_compression_trace = GRPC_TRACER_INITIALIZER(false); #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index f31f0bc4c7..f5ce96f62d 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -50,9 +50,9 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/event_string.h" -grpc_tracer_flag grpc_trace_operation_failures; +grpc_tracer_flag grpc_trace_operation_failures = GRPC_TRACER_INITIALIZER(false); #ifndef NDEBUG -grpc_tracer_flag grpc_trace_pending_tags; +grpc_tracer_flag grpc_trace_pending_tags = GRPC_TRACER_INITIALIZER(false); #endif typedef struct { diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 99020926c9..795429ed1e 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -73,7 +73,7 @@ typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; -grpc_tracer_flag grpc_server_channel_trace; +grpc_tracer_flag grpc_server_channel_trace = GRPC_TRACER_INITIALIZER(false); typedef struct requested_call { requested_call_type type; diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c index c4c0078cf2..eaf9caaa64 100644 --- a/src/core/lib/transport/bdp_estimator.c +++ b/src/core/lib/transport/bdp_estimator.c @@ -38,7 +38,7 @@ #include #include -grpc_tracer_flag grpc_bdp_estimator_trace; +grpc_tracer_flag grpc_bdp_estimator_trace = GRPC_TRACER_INITIALIZER(false); void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) { estimator->estimate = 65536; diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 0b8dc703e2..e30cd523fa 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -39,7 +39,7 @@ #include #include -grpc_tracer_flag grpc_connectivity_state_trace; +grpc_tracer_flag grpc_connectivity_state_trace = GRPC_TRACER_INITIALIZER(false); const char *grpc_connectivity_state_name(grpc_connectivity_state state) { switch (state) { diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index 9a2d4669c6..4efcf8f43d 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -41,7 +41,7 @@ /* --- Tracing. --- */ -grpc_tracer_flag tsi_tracing_enabled; +grpc_tracer_flag tsi_tracing_enabled = GRPC_TRACER_INITIALIZER(false); /* --- tsi_result common implementation. --- */ -- cgit v1.2.3 From 9ba9f704c30640a6cd98195b4eaf4b1fab660bec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 08:17:30 -0700 Subject: Fix race --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index ab74ff6ad3..654ff0c09d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1485,7 +1485,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (GRPC_TRACER_ON(grpc_http_trace)) { char *str = grpc_transport_stream_op_batch_string(op); - gpr_log(GPR_DEBUG, "perform_stream_op[s=%p/%d]: %s", s, s->id, str); + gpr_log(GPR_DEBUG, "perform_stream_op[s=%p]: %s", s, str); gpr_free(str); } -- cgit v1.2.3 From 1dbd063dbf08cd39ab398c75c1c458ee6bcf4993 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 16:48:32 -0700 Subject: Fix old trace vars --- src/core/ext/filters/http/server/http_server_filter.c | 2 -- src/core/ext/transport/chttp2/transport/hpack_encoder.c | 6 +++--- src/core/ext/transport/chttp2/transport/hpack_table.c | 7 ++++--- src/core/lib/iomgr/tcp_client_uv.c | 6 +++--- src/core/lib/iomgr/tcp_uv.h | 3 ++- test/core/iomgr/timer_list_test.c | 13 +++++++------ 6 files changed, 19 insertions(+), 18 deletions(-) (limited to 'src/core') diff --git a/src/core/ext/filters/http/server/http_server_filter.c b/src/core/ext/filters/http/server/http_server_filter.c index ff857878e4..9e495f4d42 100644 --- a/src/core/ext/filters/http/server/http_server_filter.c +++ b/src/core/ext/filters/http/server/http_server_filter.c @@ -46,8 +46,6 @@ #define EXPECTED_CONTENT_TYPE "application/grpc" #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1 -extern int grpc_http_trace; - typedef struct call_data { grpc_linked_mdelem status; grpc_linked_mdelem content_type; diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 8fdd4ee77c..126e012aac 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -69,7 +69,7 @@ static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; static const grpc_slice terminal_slice = {&terminal_slice_refcount, .data.refcounted = {0, 0}}; -extern int grpc_http_trace; +extern grpc_tracer_flag grpc_http_trace; typedef struct { int is_first_frame; @@ -425,7 +425,7 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, "Reserved header (colon-prefixed) happening after regular ones."); } - if (grpc_http_trace && !GRPC_MDELEM_IS_INTERNED(elem)) { + if (GRPC_TRACER_ON(grpc_http_trace) && !GRPC_MDELEM_IS_INTERNED(elem)) { char *k = grpc_slice_to_c_string(GRPC_MDKEY(elem)); char *v = grpc_slice_to_c_string(GRPC_MDVALUE(elem)); gpr_log( @@ -616,7 +616,7 @@ void grpc_chttp2_hpack_compressor_set_max_table_size( } } c->advertise_table_size_change = 1; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "set max table size from encoder to %d", max_table_size); } } diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c index 9dd41fdbe1..7aaff55339 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.c +++ b/src/core/ext/transport/chttp2/transport/hpack_table.c @@ -40,9 +40,10 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/support/murmur_hash.h" -extern int grpc_http_trace; +extern grpc_tracer_flag grpc_http_trace; static struct { const char *key; @@ -260,7 +261,7 @@ void grpc_chttp2_hptbl_set_max_bytes(grpc_exec_ctx *exec_ctx, if (tbl->max_bytes == max_bytes) { return; } - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "Update hpack parser max size to %d", max_bytes); } while (tbl->mem_used > max_bytes) { @@ -284,7 +285,7 @@ grpc_error *grpc_chttp2_hptbl_set_current_table_size(grpc_exec_ctx *exec_ctx, gpr_free(msg); return err; } - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "Update hpack parser table size to %d", bytes); } while (tbl->mem_used > bytes) { diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index 682c24ed56..f0856a76d4 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -46,7 +46,7 @@ #include "src/core/lib/iomgr/tcp_uv.h" #include "src/core/lib/iomgr/timer.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; typedef struct grpc_uv_tcp_connect { uv_connect_t connect_req; @@ -72,7 +72,7 @@ static void uv_tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { int done; grpc_uv_tcp_connect *connect = acp; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", connect->addr_name, str); @@ -156,7 +156,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, uv_tcp_init(uv_default_loop(), connect->tcp_handle); connect->connect_req.data = connect; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", connect->addr_name); } diff --git a/src/core/lib/iomgr/tcp_uv.h b/src/core/lib/iomgr/tcp_uv.h index 970fcafe4a..106bec5eca 100644 --- a/src/core/lib/iomgr/tcp_uv.h +++ b/src/core/lib/iomgr/tcp_uv.h @@ -44,11 +44,12 @@ otherwise specified. */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; #define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 diff --git a/test/core/iomgr/timer_list_test.c b/test/core/iomgr/timer_list_test.c index 46e41dd449..88a9f6b855 100644 --- a/test/core/iomgr/timer_list_test.c +++ b/test/core/iomgr/timer_list_test.c @@ -41,12 +41,13 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "test/core/util/test_config.h" #define MAX_CB 30 -extern int grpc_timer_trace; -extern int grpc_timer_check_trace; +extern grpc_tracer_flag grpc_timer_trace; +extern grpc_tracer_flag grpc_timer_check_trace; static int cb_called[MAX_CB][2]; @@ -63,8 +64,8 @@ static void add_test(void) { gpr_log(GPR_INFO, "add_test"); grpc_timer_list_init(start); - grpc_timer_trace = 1; - grpc_timer_check_trace = 1; + grpc_timer_trace.value = 1; + grpc_timer_check_trace.value = 1; memset(cb_called, 0, sizeof(cb_called)); /* 10 ms timers. will expire in the current epoch */ @@ -138,8 +139,8 @@ void destruction_test(void) { gpr_log(GPR_INFO, "destruction_test"); grpc_timer_list_init(gpr_time_0(GPR_CLOCK_REALTIME)); - grpc_timer_trace = 1; - grpc_timer_check_trace = 1; + grpc_timer_trace.value = 1; + grpc_timer_check_trace.value = 1; memset(cb_called, 0, sizeof(cb_called)); grpc_timer_init( -- cgit v1.2.3 From 14a7f9a2a834c6429309a44312ae089711ddedac Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 21:16:03 -0700 Subject: Reduce wakeups, comment code --- src/core/lib/iomgr/timer_manager.c | 73 ++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 10 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 1d83341299..8b83cc0106 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -52,8 +52,10 @@ static int g_thread_count; static int g_waiter_count; static completed_thread *g_completed_threads; static bool g_kicked; +static bool g_has_timed_waiter; +static uint64_t g_timed_waiter_generation; -#define MAX_WAITERS 3 +#define MAX_WAITERS 2 static void timer_thread(void *unused); @@ -92,39 +94,87 @@ void grpc_timer_manager_tick() { } static void timer_thread(void *unused) { + // this threads exec_ctx: we try to run things through to completion here + // since it's easy to spin up new threads grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + const gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_MONOTONIC); for (;;) { - gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec next = inf_future; gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + // check timer state, updates next to the next time to run a check if (grpc_timer_check(&exec_ctx, now, &next)) { + // if there's something to execute... gpr_mu_lock(&g_mu); + // remove a waiter from the pool, and start another thread if necessary --g_waiter_count; bool start_thread = g_waiter_count == 0; if (start_thread && g_threaded) { start_timer_thread_and_unlock(); } else { + // if there's no thread waiting with a timeout, kick an existing waiter + // so that the next deadline is not missed + if (!g_has_timed_waiter) { + gpr_log(GPR_DEBUG, "kick untimed waiter"); + gpr_cv_signal(&g_cv_wait); + } gpr_mu_unlock(&g_mu); } + // without our lock, flush the exec_ctx grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(&g_mu); + // garbage collect any threads hanging out that are dead gc_completed_threads(); + // get ready to wait again ++g_waiter_count; gpr_mu_unlock(&g_mu); } else { gpr_mu_lock(&g_mu); + // if we're not threaded anymore, leave if (!g_threaded) break; - if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) { - if (g_kicked) { - grpc_timer_consume_kick(); - g_kicked = false; - } else if (g_waiter_count > MAX_WAITERS) { - break; - } + // if there's no timed waiter, we should become one: that waiter waits + // only until the next timer should expire + // all other timers wait forever + uint64_t my_timed_waiter_generation = g_timed_waiter_generation - 1; + if (!g_has_timed_waiter) { + g_has_timed_waiter = true; + // we use a generation counter to track the timed waiter so we can + // cancel an existing one quickly (and when it actually times out it'll + // figure stuff out instead of incurring a wakeup) + my_timed_waiter_generation = ++g_timed_waiter_generation; + gpr_log(GPR_DEBUG, "sleep for a while"); + } else { + next = inf_future; + gpr_log(GPR_DEBUG, "sleep until kicked"); + } + bool timed_out = gpr_cv_wait(&g_cv_wait, &g_mu, next); + // if we timed out and we have too many waiters, maybe exit this thread + bool should_stop = (timed_out && g_waiter_count > MAX_WAITERS); + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d timed_out:%d kicked:%d", + my_timed_waiter_generation == g_timed_waiter_generation, + timed_out, g_kicked); + // if this was the timed waiter, then we need to check timers, and flag + // that there's now no timed waiter... we'll look for a replacement if + // there's work to do after checking timers (code above) + if (my_timed_waiter_generation == g_timed_waiter_generation) { + g_has_timed_waiter = false; + should_stop = false; + } + // if this was a kick from the timer system, consume it (and don't stop + // this thread yet) + if (g_kicked) { + grpc_timer_consume_kick(); + g_kicked = false; + should_stop = false; + } + if (should_stop) { + break; } - gpr_mu_unlock(&g_mu); } + gpr_mu_unlock(&g_mu); } + // terminate the thread: drop the waiter count, thread count, and let whomever + // stopped the threading stuff know that we're done --g_waiter_count; --g_thread_count; if (0 == g_thread_count) { @@ -135,6 +185,7 @@ static void timer_thread(void *unused) { ct->next = g_completed_threads; g_completed_threads = ct; gpr_mu_unlock(&g_mu); + grpc_exec_ctx_finish(&exec_ctx); gpr_log(GPR_DEBUG, "End timer thread"); } @@ -193,6 +244,8 @@ void grpc_timer_manager_set_threading(bool threaded) { void grpc_kick_poller(void) { gpr_mu_lock(&g_mu); g_kicked = true; + g_has_timed_waiter = false; + ++g_timed_waiter_generation; gpr_cv_signal(&g_cv_wait); gpr_mu_unlock(&g_mu); } -- cgit v1.2.3 From 2e8993f07c90432b847cf615628afadbb5f876b7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 21:25:08 -0700 Subject: Eliminate some complexity --- src/core/lib/iomgr/timer_manager.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 8b83cc0106..dd7120695c 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -44,19 +44,27 @@ typedef struct completed_thread { struct completed_thread *next; } completed_thread; +// global mutex static gpr_mu g_mu; +// are we multi-threaded static bool g_threaded; +// cv to wait until a thread is needed static gpr_cv g_cv_wait; +// cv for notification when threading ends static gpr_cv g_cv_shutdown; +// number of threads in the system static int g_thread_count; +// number of threads sitting around waiting static int g_waiter_count; +// linked list of threads that have completed (and need joining) static completed_thread *g_completed_threads; +// was the manager kicked by the timer system static bool g_kicked; +// is there a thread waiting until the next timer should fire? static bool g_has_timed_waiter; +// generation counter to track which thread is waiting for the next timer static uint64_t g_timed_waiter_generation; -#define MAX_WAITERS 2 - static void timer_thread(void *unused); static void gc_completed_threads(void) { @@ -147,28 +155,21 @@ static void timer_thread(void *unused) { next = inf_future; gpr_log(GPR_DEBUG, "sleep until kicked"); } - bool timed_out = gpr_cv_wait(&g_cv_wait, &g_mu, next); - // if we timed out and we have too many waiters, maybe exit this thread - bool should_stop = (timed_out && g_waiter_count > MAX_WAITERS); - gpr_log(GPR_DEBUG, "wait ended: was_timed:%d timed_out:%d kicked:%d", + gpr_cv_wait(&g_cv_wait, &g_mu, next); + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", my_timed_waiter_generation == g_timed_waiter_generation, - timed_out, g_kicked); + g_kicked); // if this was the timed waiter, then we need to check timers, and flag // that there's now no timed waiter... we'll look for a replacement if // there's work to do after checking timers (code above) if (my_timed_waiter_generation == g_timed_waiter_generation) { g_has_timed_waiter = false; - should_stop = false; } // if this was a kick from the timer system, consume it (and don't stop // this thread yet) if (g_kicked) { grpc_timer_consume_kick(); g_kicked = false; - should_stop = false; - } - if (should_stop) { - break; } } gpr_mu_unlock(&g_mu); -- cgit v1.2.3 From 66918a6106476cbd60137c725529cb22ab958b14 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 21:27:01 -0700 Subject: simplify --- src/core/lib/iomgr/timer_manager.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index dd7120695c..464907b454 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -116,8 +116,7 @@ static void timer_thread(void *unused) { gpr_mu_lock(&g_mu); // remove a waiter from the pool, and start another thread if necessary --g_waiter_count; - bool start_thread = g_waiter_count == 0; - if (start_thread && g_threaded) { + if (g_waiter_count == 0 && g_threaded) { start_timer_thread_and_unlock(); } else { // if there's no thread waiting with a timeout, kick an existing waiter -- cgit v1.2.3 From cfaa046a0127f9e40e63eaf5490164794d6ab253 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 15:27:40 +0000 Subject: Integrate new tracers --- src/core/lib/iomgr/ev_epoll1_linux.c | 14 ------- src/core/lib/iomgr/ev_epollex_linux.c | 73 +++++++++------------------------- src/core/lib/iomgr/ev_epollsig_linux.c | 3 +- src/core/lib/iomgr/ev_posix.c | 2 +- src/core/lib/iomgr/ev_posix.h | 3 +- 5 files changed, 23 insertions(+), 72 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 90c5546555..8aa69cd73a 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -63,12 +63,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ static grpc_wakeup_fd global_wakeup_fd; static int g_epfd; -static gpr_atm g_timer_kick; /******************************************************************************* * Fd Declarations @@ -512,9 +508,6 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - if (gpr_atm_no_barrier_cas(&g_timer_kick, 1, 0)) { - grpc_timer_consume_kick(); - } gpr_mu_lock(&g_wq_mu); grpc_closure_list_move(&g_wq_items, &exec_ctx->closure_list); gpr_mu_unlock(&g_wq_mu); @@ -799,11 +792,6 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) {} -static grpc_error *kick_poller(void) { - gpr_atm_no_barrier_store(&g_timer_kick, 1); - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - /******************************************************************************* * Workqueue Definitions */ @@ -951,8 +939,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b9f60d8e3e..5a7c0448b6 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -64,11 +64,6 @@ #include "src/core/lib/support/block_annotate.h" #include "src/core/lib/support/spinlock.h" -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - /******************************************************************************* * Pollset-set sibling link */ @@ -560,16 +555,6 @@ static grpc_error *pollable_materialize(pollable *p) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { return GRPC_OS_ERROR(errno, "epoll_create1"); - } else { - struct epoll_event ev = { - .events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), - .data.ptr = &global_wakeup_fd}; - if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != - 0) { - grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); - close(new_epfd); - return err; - } } grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); if (err != GRPC_ERROR_NONE) { @@ -639,22 +624,16 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); -static bool global_wakeup_fd_initialized = false; /* Global state management */ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - grpc_error *error = GRPC_ERROR_NONE; - static const char *err_desc = "pollset_global_init"; - global_wakeup_fd_initialized = - append_error(&error, grpc_wakeup_fd_init(&global_wakeup_fd), err_desc); pollable_init(&g_empty_pollable, PO_EMPTY_POLLABLE); - return error; + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - if (global_wakeup_fd_initialized) grpc_wakeup_fd_destroy(&global_wakeup_fd); pollable_destroy(&g_empty_pollable); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); @@ -687,7 +666,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p " "root_worker=(pollset:%p pollable:%p)", @@ -698,13 +677,13 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { if (pollset->root_worker == NULL) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); } pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; } else { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_via_wakeup_fd", p); } grpc_error *err = pollable_materialize(p); @@ -712,25 +691,25 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, return grpc_wakeup_fd_wakeup(&p->wakeup); } } else { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_but_awake", p); } return GRPC_ERROR_NONE; } } else if (specific_worker->kicked) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_already_kicked", p); } return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); } specific_worker->kicked = true; return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_wakeup_fd", p); } grpc_error *err = pollable_materialize(p); @@ -738,7 +717,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, specific_worker->kicked = true; return grpc_wakeup_fd_wakeup(&p->wakeup); } else { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); } specific_worker->kicked = true; @@ -761,10 +740,6 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollable_init(&pollset->pollable, PO_POLLSET); pollset->current_pollable = &g_empty_pollable; @@ -865,7 +840,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, int timeout = poll_deadline_to_millis_timeout(deadline, now); - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p for %dms", pollset, p, timeout); } @@ -882,23 +857,15 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p got %d events", pollset, p, r); } grpc_error *error = GRPC_ERROR_NONE; for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll %p got global_wakeup_fd", pollset, p); - } - - grpc_timer_consume_kick(); - append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &p->wakeup) { - if (grpc_polling_trace) { + if (data_ptr == &p->wakeup) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p got pollset_wakeup", pollset, p); } append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); @@ -908,7 +875,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p got fd %p: is_wq=%d cancel=%d read=%d " "write=%d", @@ -994,25 +961,25 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (worker->pollable != &pollset->pollable) { gpr_mu_unlock(&pollset->pollable.po.mu); } - if (grpc_polling_trace && worker->pollable->root_worker != worker) { + if (GRPC_TRACER_ON(grpc_polling_trace) && worker->pollable->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, worker->pollable, worker, poll_deadline_to_millis_timeout(deadline, *now)); } while (do_poll && worker->pollable->root_worker != worker) { if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, worker->pollable, worker); } do_poll = false; } else if (worker->kicked) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, worker); } do_poll = false; - } else if (grpc_polling_trace && + } else if (GRPC_TRACER_ON(grpc_polling_trace) && worker->pollable->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, worker->pollable, worker); @@ -1056,7 +1023,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - if (0 && grpc_polling_trace) { + if (0 && GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, @@ -1484,8 +1451,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 11be0e70a4..d9ba77c6f0 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -65,9 +65,8 @@ #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) -/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ #define GRPC_POLLING_TRACE(fmt, ...) \ - if (grpc_polling_trace) { \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index d3302f4a80..e3d53d6d3d 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -51,7 +51,7 @@ #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" -int grpc_polling_trace = 0; /* Disabled by default */ +grpc_tracer_flag grpc_polling_trace = GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ /** Default poll() function - a pointer so that it can be overridden by some * tests */ diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index d70ed74ae1..e013a6c953 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -41,8 +41,9 @@ #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/debug/trace.h" -extern int grpc_polling_trace; /* Disabled by default */ +extern grpc_tracer_flag grpc_polling_trace; /* Disabled by default */ typedef struct grpc_fd grpc_fd; -- cgit v1.2.3 From 6b82f8c4bfaaab5bb20171eb25ec7d7b0510f5f1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 16:13:28 +0000 Subject: Fix misplaced unlock, cleanup tracing --- src/core/lib/iomgr/timer_manager.c | 43 ++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 464907b454..24085093e7 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -37,6 +37,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/timer.h" typedef struct completed_thread { @@ -44,6 +45,8 @@ typedef struct completed_thread { struct completed_thread *next; } completed_thread; +extern grpc_tracer_flag grpc_timer_check_trace; + // global mutex static gpr_mu g_mu; // are we multi-threaded @@ -83,10 +86,13 @@ static void gc_completed_threads(void) { } static void start_timer_thread_and_unlock(void) { + GPR_ASSERT(g_threaded); ++g_waiter_count; ++g_thread_count; gpr_mu_unlock(&g_mu); - gpr_log(GPR_DEBUG, "Spawn timer thread"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "Spawn timer thread"); + } gpr_thd_id thd; gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); @@ -122,7 +128,9 @@ static void timer_thread(void *unused) { // if there's no thread waiting with a timeout, kick an existing waiter // so that the next deadline is not missed if (!g_has_timed_waiter) { - gpr_log(GPR_DEBUG, "kick untimed waiter"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "kick untimed waiter"); + } gpr_cv_signal(&g_cv_wait); } gpr_mu_unlock(&g_mu); @@ -149,15 +157,21 @@ static void timer_thread(void *unused) { // cancel an existing one quickly (and when it actually times out it'll // figure stuff out instead of incurring a wakeup) my_timed_waiter_generation = ++g_timed_waiter_generation; - gpr_log(GPR_DEBUG, "sleep for a while"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "sleep for a while"); + } } else { next = inf_future; - gpr_log(GPR_DEBUG, "sleep until kicked"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "sleep until kicked"); + } } gpr_cv_wait(&g_cv_wait, &g_mu, next); - gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", - my_timed_waiter_generation == g_timed_waiter_generation, - g_kicked); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", + my_timed_waiter_generation == g_timed_waiter_generation, + g_kicked); + } // if this was the timed waiter, then we need to check timers, and flag // that there's now no timed waiter... we'll look for a replacement if // there's work to do after checking timers (code above) @@ -170,8 +184,8 @@ static void timer_thread(void *unused) { grpc_timer_consume_kick(); g_kicked = false; } + gpr_mu_unlock(&g_mu); } - gpr_mu_unlock(&g_mu); } // terminate the thread: drop the waiter count, thread count, and let whomever // stopped the threading stuff know that we're done @@ -186,7 +200,9 @@ static void timer_thread(void *unused) { g_completed_threads = ct; gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_log(GPR_DEBUG, "End timer thread"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "End timer thread"); + } } static void start_threads(void) { @@ -214,11 +230,20 @@ void grpc_timer_manager_init(void) { static void stop_threads(void) { gpr_mu_lock(&g_mu); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "stop timer threads: threaded=%d", g_threaded); + } if (g_threaded) { g_threaded = false; gpr_cv_broadcast(&g_cv_wait); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count); + } while (g_thread_count > 0) { gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count); + } gc_completed_threads(); } } -- cgit v1.2.3 From 6c8383af699211963debb76e2f96dee09497c26a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 16:54:42 +0000 Subject: Fix missing edge, add tracing --- src/core/lib/iomgr/ev_epoll1_linux.c | 1 - src/core/lib/iomgr/ev_epollex_linux.c | 26 +++++++++++++++++++++++--- src/core/lib/iomgr/lockfree_event.c | 16 ++++++++++++++++ src/core/lib/iomgr/tcp_client_posix.c | 4 ++-- 4 files changed, 41 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8aa69cd73a..d31eafc70b 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -57,7 +57,6 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/lockfree_event.h" -#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 5a7c0448b6..cb6814e89e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -582,6 +582,10 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { const int epfd = p->epfd; GPR_ASSERT(epfd != -1); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "add fd %p to pollable %p", fd, p); + } + gpr_mu_lock(&fd->orphaned_mu); if (fd->orphaned) { gpr_mu_unlock(&fd->orphaned_mu); @@ -961,7 +965,8 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (worker->pollable != &pollset->pollable) { gpr_mu_unlock(&pollset->pollable.po.mu); } - if (GRPC_TRACER_ON(grpc_polling_trace) && worker->pollable->root_worker != worker) { + if (GRPC_TRACER_ON(grpc_polling_trace) && + worker->pollable->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, worker->pollable, worker, poll_deadline_to_millis_timeout(deadline, *now)); @@ -1080,6 +1085,10 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; if (pollset->current_pollable == &g_empty_pollable) { + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, + "PS:%p add fd %p; transition pollable from empty to fd", pollset, + fd); /* empty pollable --> single fd pollable */ append_error(&error, pollset_kick_all(pollset), err_desc); pollset->current_pollable = &fd->pollable; @@ -1088,10 +1097,17 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); REF_BY(fd, 2, "pollset_pollable"); } else if (pollset->current_pollable == &pollset->pollable) { + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); append_error(&error, pollable_add_fd(pollset->current_pollable, fd), err_desc); } else if (pollset->current_pollable != &fd->pollable) { grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, + "PS:%p add fd %p; transition pollable from fd %p to multipoller", + pollset, fd, had_fd); + append_error(&error, pollset_kick_all(pollset), err_desc); pollset->current_pollable = &pollset->pollable; if (append_error(&error, pollable_materialize(&pollset->pollable), err_desc)) { @@ -1458,7 +1474,8 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requested) { +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1483,7 +1500,10 @@ const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requeste #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requested) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c index 17e3bbf727..898ec1cb1b 100644 --- a/src/core/lib/iomgr/lockfree_event.c +++ b/src/core/lib/iomgr/lockfree_event.c @@ -35,6 +35,10 @@ #include +#include "src/core/lib/debug/trace.h" + +extern grpc_tracer_flag grpc_polling_trace; + /* 'state' holds the to call when the fd is readable or writable respectively. It can contain one of the following values: CLOSURE_READY : The fd has an I/O event of interest but there is no @@ -93,6 +97,10 @@ void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, grpc_closure *closure) { while (true) { gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_notify_on: %p curr=%p closure=%p", state, + (void *)curr, closure); + } switch (curr) { case CLOSURE_NOT_READY: { /* CLOSURE_NOT_READY -> . @@ -155,6 +163,10 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state, while (true) { gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_set_shutdown: %p curr=%p err=%s", state, + (void *)curr, grpc_error_string(shutdown_err)); + } switch (curr) { case CLOSURE_READY: case CLOSURE_NOT_READY: @@ -200,6 +212,10 @@ void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) { while (true) { gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_set_ready: %p curr=%p", state, (void *)curr); + } + switch (curr) { case CLOSURE_READY: { /* Already ready. We are done here */ diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index ed3fd94a98..5c7da999e0 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -331,8 +331,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, ac->channel_args = grpc_channel_args_copy(channel_args); if (GRPC_TRACER_ON(grpc_tcp_trace)) { - gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", - ac->addr_str); + gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting fd %p", + ac->addr_str, fdobj); } gpr_mu_lock(&ac->mu); -- cgit v1.2.3 From 3e04383817b60a3fae9292fa21aaf50a9b0ffb8f Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 5 May 2017 10:11:46 -0700 Subject: Remove extra log line --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 1 - 1 file changed, 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 8679f9c85d..f46dee8a04 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -1283,7 +1283,6 @@ static void shutdown_poller_threads() { gpr_thd_join(g_poller_threads[i]); } - gpr_log(GPR_ERROR, "epoll set delete called"); GRPC_LOG_IF_ERROR("shutdown_poller_threads", error); gpr_free(g_poller_threads); g_poller_threads = NULL; -- cgit v1.2.3 From 924353a7173ed6184f3e0baef8c62d3ccbfa39f5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 17:36:31 +0000 Subject: Finish merge, disable new pollers until they stabilize --- CMakeLists.txt | 7 ---- Makefile | 7 ---- binding.gyp | 1 - build.yaml | 1 - config.m4 | 1 - gRPC-Core.podspec | 1 - grpc.gemspec | 1 - package.xml | 1 - src/core/lib/iomgr/ev_epoll1_linux.c | 3 ++ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 41 ++++++++-------------- src/core/lib/iomgr/ev_epoll_thread_pool_linux.h | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ src/core/lib/iomgr/ev_epollsig_linux.c | 3 +- src/python/grpcio/grpc_core_dependencies.py | 1 - tools/doxygen/Doxyfile.c++.internal | 1 - tools/doxygen/Doxyfile.core.internal | 1 - tools/run_tests/generated/sources_and_headers.json | 1 - vsprojects/vcxproj/grpc++/grpc++.vcxproj | 2 -- vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 3 -- .../grpc++_unsecure/grpc++_unsecure.vcxproj | 2 -- .../grpc++_unsecure.vcxproj.filters | 3 -- vsprojects/vcxproj/grpc/grpc.vcxproj | 2 -- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 -- .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 2 -- .../grpc_test_util/grpc_test_util.vcxproj.filters | 3 -- .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 2 -- .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 3 -- 27 files changed, 22 insertions(+), 78 deletions(-) (limited to 'src/core') diff --git a/CMakeLists.txt b/CMakeLists.txt index 007982644f..581d96bc06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -940,7 +940,6 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -1274,7 +1273,6 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -1591,7 +1589,6 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -1853,7 +1850,6 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -2280,7 +2276,6 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -2611,7 +2606,6 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -3386,7 +3380,6 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c diff --git a/Makefile b/Makefile index b731dcb831..e83b73827b 100644 --- a/Makefile +++ b/Makefile @@ -2923,7 +2923,6 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -3255,7 +3254,6 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -3571,7 +3569,6 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -3805,7 +3802,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -4209,7 +4205,6 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -4548,7 +4543,6 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -5313,7 +5307,6 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ diff --git a/binding.gyp b/binding.gyp index 37e41efb5c..bf3d14135d 100644 --- a/binding.gyp +++ b/binding.gyp @@ -675,7 +675,6 @@ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_epollsig_linux.c', diff --git a/build.yaml b/build.yaml index 3b062048b6..1223e05250 100644 --- a/build.yaml +++ b/build.yaml @@ -311,7 +311,6 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll1_linux.c - - src/core/lib/iomgr/ev_epoll_linux.c - src/core/lib/iomgr/ev_epoll_thread_pool_linux.c - src/core/lib/iomgr/ev_epollex_linux.c - src/core/lib/iomgr/ev_epollsig_linux.c diff --git a/config.m4 b/config.m4 index 8c201f3bbf..e2ad0e11b7 100644 --- a/config.m4 +++ b/config.m4 @@ -109,7 +109,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2bbdd910e3..c81a522dfe 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -494,7 +494,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_epollsig_linux.c', diff --git a/grpc.gemspec b/grpc.gemspec index d8226fda7d..95174d4faf 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -410,7 +410,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.c ) - s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epollex_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.c ) diff --git a/package.xml b/package.xml index 2eba84bafb..3ba1df3155 100644 --- a/package.xml +++ b/package.xml @@ -419,7 +419,6 @@ - diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index d31eafc70b..ad69f808cd 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -948,6 +948,9 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { + /* TODO(ctiller): temporary, until this stabilizes */ + if (!explicit_request) return NULL; + if (!grpc_has_wakeup_fd()) { return NULL; } diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 8679f9c85d..cf20fc3346 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -66,9 +66,8 @@ #include "src/core/lib/support/block_annotate.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -static int grpc_polling_trace = 0; /* Disabled by default */ #define GRPC_POLLING_TRACE(fmt, ...) \ - if (grpc_polling_trace) { \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } @@ -77,13 +76,10 @@ static int grpc_polling_trace = 0; /* Disabled by default */ * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a * case occurs. */ -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - struct epoll_set; +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1) + /******************************************************************************* * Fd Declarations */ @@ -362,7 +358,7 @@ static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); @@ -423,7 +419,6 @@ static epoll_set *epoll_set_create(grpc_error **error) { goto done; } - epoll_set_add_wakeup_fd_locked(eps, &global_wakeup_fd, error); epoll_set_add_wakeup_fd_locked(eps, &eps->workqueue_wakeup_fd, error); done: @@ -703,11 +698,10 @@ static void pollset_worker_init(grpc_pollset_worker *worker) { static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } @@ -802,10 +796,6 @@ static grpc_error *pollset_kick(grpc_pollset *p, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; @@ -870,7 +860,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. So other * than destroying the mutexes, there is nothing special that needs to be done * here */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); gpr_mu_destroy(&pollset->mu); } @@ -944,11 +934,7 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, for (int i = 0; i < ep_rv; ++i) { void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &eps->workqueue_wakeup_fd) { + if (data_ptr == &eps->workqueue_wakeup_fd) { append_error(error, grpc_wakeup_fd_consume_wakeup(&eps->workqueue_wakeup_fd), err_desc); @@ -1152,8 +1138,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, @@ -1227,11 +1211,12 @@ static void shutdown_epoll_sets() { for (size_t i = 0; i < g_num_eps; i++) { EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_epoll_sets"); } - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_free(g_epoll_sets); g_epoll_sets = NULL; - pollset_destroy(&g_read_notifier); + pollset_destroy(&exec_ctx, &g_read_notifier); + grpc_exec_ctx_finish(&exec_ctx); } static void poller_thread_loop(void *arg) { @@ -1306,7 +1291,9 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly) { + if (!requested_explicitly) return NULL; + if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1341,6 +1328,6 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll_linux(bool requested_explicitly) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h index f4959e3fee..b94dbe7d5d 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -37,6 +37,6 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void); +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index cb6814e89e..7cb6085e25 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1476,6 +1476,8 @@ static const grpc_event_engine_vtable vtable = { const grpc_event_engine_vtable *grpc_init_epollex_linux( bool explicitly_requested) { + if (!explicitly_requested) return NULL; + if (!grpc_has_wakeup_fd()) { return NULL; } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index d9ba77c6f0..32096ed55f 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1920,7 +1920,8 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux( } if (!is_grpc_wakeup_signal_initialized) { - if (explicit_request) { + /* TODO(ctiller): when other epoll engines are ready, remove the true || to force this to be explitly chosen if needed */ + if (true || explicit_request) { grpc_use_signal(SIGRTMIN + 6); } else { return NULL; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index c7c341fee7..7bf3381f60 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -98,7 +98,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_epollsig_linux.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e56d36dc53..7455d0c0d2 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -941,7 +941,6 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll1_linux.c \ src/core/lib/iomgr/ev_epoll1_linux.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_epollex_linux.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 8b2e6aea75..96726a5357 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1080,7 +1080,6 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll1_linux.c \ src/core/lib/iomgr/ev_epoll1_linux.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_epollex_linux.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index ece8695f86..4d434e7f6b 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7914,7 +7914,6 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.c", "src/core/lib/iomgr/ev_epoll1_linux.h", - "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_epollex_linux.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 88250ff3ae..f71af04092 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -618,8 +618,6 @@ - - diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index d9fa3bf741..5ddcb0924b 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -187,9 +187,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 3750bb83f7..6ab9653178 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -602,8 +602,6 @@ - - diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index d628067bd5..3fbd96324b 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -172,9 +172,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 89e5522b1b..b9ec423444 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -559,8 +559,6 @@ - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 6f5add03a2..9568c8db90 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -67,9 +67,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 5b0a632949..45ef342c93 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -391,8 +391,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index b96e653a57..f3d9d069a3 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -124,9 +124,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 34f2493a6e..ba6b989b55 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -526,8 +526,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 76ba99fdf7..620720d05a 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -70,9 +70,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr -- cgit v1.2.3 From bc0ab08652fb93bb030aa97955bd8c109a3d3215 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 10:42:44 -0700 Subject: clang-format --- src/core/ext/transport/chttp2/transport/internal.h | 2 +- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 20 ++++++++++++-------- src/core/lib/iomgr/ev_epoll_thread_pool_linux.h | 3 ++- src/core/lib/iomgr/ev_epollex_linux.h | 3 ++- src/core/lib/iomgr/ev_epollsig_linux.c | 9 +++++---- src/core/lib/iomgr/ev_posix.c | 7 ++++--- src/core/lib/iomgr/ev_posix.h | 2 +- src/core/lib/iomgr/iomgr.c | 4 +--- src/core/lib/surface/api_trace.c | 2 +- 9 files changed, 29 insertions(+), 23 deletions(-) (limited to 'src/core') diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index bb5ce60872..b9d06a84d3 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -671,7 +671,7 @@ typedef enum { #define GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ - if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ GRPC_CHTTP2_FLOWCTL_CREDIT, #dst_context, \ #dst_var, NULL, #amount, transport->is_client, \ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index cf20fc3346..a3a9e76ed4 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -66,9 +66,9 @@ #include "src/core/lib/support/block_annotate.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -#define GRPC_POLLING_TRACE(fmt, ...) \ - if (GRPC_TRACER_ON(grpc_polling_trace)) { \ - gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } /* The alarm system needs to be able to wakeup 'some poller' sometimes @@ -78,7 +78,7 @@ struct epoll_set; -#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1) +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /******************************************************************************* * Fd Declarations @@ -358,8 +358,8 @@ static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), - errno, strerror(errno)); + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -1291,7 +1291,8 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly) { +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly) { if (!requested_explicitly) return NULL; if (!grpc_has_wakeup_fd()) { @@ -1328,6 +1329,9 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(bool requested_explicitly) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll_linux( + bool requested_explicitly) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h index b94dbe7d5d..9af776a52e 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly); +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.h b/src/core/lib/iomgr/ev_epollex_linux.h index a865c49116..a078a7f19a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.h +++ b/src/core/lib/iomgr/ev_epollex_linux.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requested); +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 32096ed55f..52362a62f4 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -65,9 +65,9 @@ #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) -#define GRPC_POLLING_TRACE(fmt, ...) \ - if (GRPC_TRACER_ON(grpc_polling_trace)) { \ - gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } /* Uncomment the following to enable extra checks on poll_object operations */ @@ -1920,7 +1920,8 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux( } if (!is_grpc_wakeup_signal_initialized) { - /* TODO(ctiller): when other epoll engines are ready, remove the true || to force this to be explitly chosen if needed */ + /* TODO(ctiller): when other epoll engines are ready, remove the true || to + * force this to be explitly chosen if needed */ if (true || explicit_request) { grpc_use_signal(SIGRTMIN + 6); } else { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 0b0535c852..02c6c746ac 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,14 +45,15 @@ #include #include "src/core/lib/debug/trace.h" -#include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_epoll1_linux.h" -#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" +#include "src/core/lib/iomgr/ev_epollex_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" -grpc_tracer_flag grpc_polling_trace = GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ +grpc_tracer_flag grpc_polling_trace = + GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ /** Default poll() function - a pointer so that it can be overridden by some * tests */ diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index e013a6c953..80619aab5f 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -36,12 +36,12 @@ #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" -#include "src/core/lib/debug/trace.h" extern grpc_tracer_flag grpc_polling_trace; /* Disabled by default */ diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 37804d8600..1fd41c2f88 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -68,9 +68,7 @@ void grpc_iomgr_init(void) { grpc_iomgr_platform_init(); } -void grpc_iomgr_start(void) { - grpc_timer_manager_init(); -} +void grpc_iomgr_start(void) { grpc_timer_manager_init(); } static size_t count_objects(void) { grpc_iomgr_object *obj; diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 55c69cd5b0..d8941cdf42 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/debug/trace.h" #include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/debug/trace.h" grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false); -- cgit v1.2.3 From 6e0df1478ebfa37c3b5f198b41c119b4e9e3e464 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 10:42:57 -0700 Subject: Header formatting --- src/core/lib/iomgr/timer_manager.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h index 0b21262b1a..46729ccea6 100644 --- a/src/core/lib/iomgr/timer_manager.h +++ b/src/core/lib/iomgr/timer_manager.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H -#define GRPC_CORE_IOMGR_TIMER_MANAGER_H +#ifndef GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H +#define GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H #include @@ -49,4 +49,4 @@ void grpc_timer_manager_set_threading(bool enabled); * disabled */ void grpc_timer_manager_tick(void); -#endif +#endif /* GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H */ -- cgit v1.2.3 From 5a1a3b49f47afeeb1b7eb2e4ea84432cce98c314 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 5 May 2017 13:14:45 -0700 Subject: Separate this into a new poller --- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + .../lib/iomgr/ev_epoll_limited_pollers_linux.c | 2156 ++++++++++++++++++++ .../lib/iomgr/ev_epoll_limited_pollers_linux.h | 42 + src/core/lib/iomgr/ev_epoll_linux.c | 399 +--- src/core/lib/iomgr/ev_posix.c | 2 + src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + tools/run_tests/generated/sources_and_headers.json | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util/grpc_test_util.vcxproj.filters | 6 + .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 6 + 26 files changed, 2365 insertions(+), 312 deletions(-) create mode 100644 src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c create mode 100644 src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h (limited to 'src/core') diff --git a/CMakeLists.txt b/CMakeLists.txt index f6c5f9e56d..68c58470eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -938,6 +938,7 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -1266,6 +1267,7 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -1577,6 +1579,7 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -1833,6 +1836,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -2254,6 +2258,7 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -2579,6 +2584,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -3348,6 +3354,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c diff --git a/Makefile b/Makefile index 1a8ee553ca..1fdf64d03f 100644 --- a/Makefile +++ b/Makefile @@ -2921,6 +2921,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -3247,6 +3248,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -3557,6 +3559,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -3785,6 +3788,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -4183,6 +4187,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -4516,6 +4521,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -5275,6 +5281,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ diff --git a/binding.gyp b/binding.gyp index c8dd5c1cfd..bf7c6a1529 100644 --- a/binding.gyp +++ b/binding.gyp @@ -674,6 +674,7 @@ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c', 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', diff --git a/build.yaml b/build.yaml index b293060234..2ae2f1b408 100644 --- a/build.yaml +++ b/build.yaml @@ -197,6 +197,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair.h - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h + - src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h - src/core/lib/iomgr/ev_epoll_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h @@ -304,6 +305,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_uv.c - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c + - src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c - src/core/lib/iomgr/ev_epoll_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c diff --git a/config.m4 b/config.m4 index 1c0c6d92fc..f54033c0c2 100644 --- a/config.m4 +++ b/config.m4 @@ -108,6 +108,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 241eba01dd..cdc5e5837d 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -279,6 +279,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h', 'src/core/lib/iomgr/ev_epoll_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', @@ -487,6 +488,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c', 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', @@ -744,6 +746,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h', 'src/core/lib/iomgr/ev_epoll_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index b96f3cbab5..22151fb29b 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -195,6 +195,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair.h ) s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) + s.files += %w( src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) @@ -403,6 +404,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_uv.c ) s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) + s.files += %w( src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) diff --git a/package.xml b/package.xml index d47708efa4..15c573bd48 100644 --- a/package.xml +++ b/package.xml @@ -204,6 +204,7 @@ + @@ -412,6 +413,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c new file mode 100644 index 0000000000..1b1c597d82 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c @@ -0,0 +1,2156 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/env.h" + +/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ +static int grpc_polling_trace = 0; /* Disabled by default */ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (grpc_polling_trace) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +/* Uncomment the following to enable extra checks on poll_object operations */ +/* #define PO_DEBUG */ + +/* The maximum number of polling threads per polling island. By default no + limit */ +static int g_max_pollers_per_pi = INT_MAX; + +static int grpc_wakeup_signal = -1; +static bool is_grpc_wakeup_signal_initialized = false; + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +/* Implements the function defined in grpc_posix.h. This function might be + * called before even calling grpc_init() to set either a different signal to + * use. If signum == -1, then the use of signals is disabled */ +static void grpc_use_signal(int signum) { + grpc_wakeup_signal = signum; + is_grpc_wakeup_signal_initialized = true; + + if (grpc_wakeup_signal < 0) { + gpr_log(GPR_INFO, + "Use of signals is disabled. Epoll engine will not be used"); + } else { + gpr_log(GPR_INFO, "epoll engine will be using signal: %d", + grpc_wakeup_signal); + } +} + +struct polling_island; + +typedef enum { + POLL_OBJ_FD, + POLL_OBJ_POLLSET, + POLL_OBJ_POLLSET_SET +} poll_obj_type; + +typedef struct poll_obj { +#ifdef PO_DEBUG + poll_obj_type obj_type; +#endif + gpr_mu mu; + struct polling_island *pi; +} poll_obj; + +static const char *poll_obj_string(poll_obj_type po_type) { + switch (po_type) { + case POLL_OBJ_FD: + return "fd"; + case POLL_OBJ_POLLSET: + return "pollset"; + case POLL_OBJ_POLLSET_SET: + return "pollset_set"; + } + + GPR_UNREACHABLE_CODE(return "UNKNOWN"); +} + +/******************************************************************************* + * Fd Declarations + */ + +#define FD_FROM_PO(po) ((grpc_fd *)(po)) + +struct grpc_fd { + poll_obj po; + + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Polling island Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define PI_UNREF(exec_ctx, p, r) \ + pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define PI_ADD_REF(p, r) pi_add_ref((p)) +#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ + +typedef struct worker_node { + struct worker_node *next; + struct worker_node *prev; +} worker_node; + +/* This is also used as grpc_workqueue (by directly casing it) */ +typedef struct polling_island { + grpc_closure_scheduler workqueue_scheduler; + + gpr_mu mu; + /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + the refcount. + Once the ref count becomes zero, this structure is destroyed which means + we should ensure that there is never a scenario where a PI_ADD_REF() is + racing with a PI_UNREF() that just made the ref_count zero. */ + gpr_atm ref_count; + + /* Pointer to the polling_island this merged into. + * merged_to value is only set once in polling_island's lifetime (and that too + * only if the island is merged with another island). Because of this, we can + * use gpr_atm type here so that we can do atomic access on this and reduce + * lock contention on 'mu' mutex. + * + * Note that if this field is not NULL (i.e not 0), all the remaining fields + * (except mu and ref_count) are invalid and must be ignored. */ + gpr_atm merged_to; + + /* Number of threads currently polling on this island */ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* The list of workers waiting to do polling on this polling island */ + gpr_mu worker_list_mu; + worker_node worker_list_head; + + /* The fd of the underlying epoll set */ + int epoll_fd; + + /* The file descriptors in the epoll set */ + size_t fd_cnt; + size_t fd_capacity; + grpc_fd **fds; +} polling_island; + +/******************************************************************************* + * Pollset Declarations + */ +#define WORKER_FROM_WORKER_LIST_NODE(p) \ + (struct grpc_pollset_worker *)(((char *)(p)) - \ + offsetof(grpc_pollset_worker, pi_list_link)) +struct grpc_pollset_worker { + /* Thread id of this worker */ + pthread_t pt_id; + + /* Used to prevent a worker from getting kicked multiple times */ + gpr_atm is_kicked; + + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; + + /* Indicates if it is this worker's turn to do epoll */ + gpr_atm is_polling_turn; + + /* Node in the polling island's worker list. */ + worker_node pi_list_link; +}; + +struct grpc_pollset { + poll_obj po; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + poll_obj po; +}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Polling island Definitions + */ + +/* The wakeup fd that is used to wake up all threads in a Polling island. This + is useful in the polling island merge operation where we need to wakeup all + the threads currently polling the smaller polling island (so that they can + start polling the new/merged polling island) + + NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the + threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd polling_island_wakeup_fd; + +/* The polling island being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread polling_island *g_current_thread_polling_island; + +/* Forward declaration */ +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void pi_add_ref(polling_island *pi); +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void pi_add_ref_dbg(polling_island *pi, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_add_ref(pi); + gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, old_cnt + 1, reason, file, line); +} + +static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_unref(exec_ctx, pi); + gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_add_ref((polling_island *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_unref(exec_ctx, (polling_island *)workqueue); + } +} +#endif + +static void pi_add_ref(polling_island *pi) { + gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +} + +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { + /* If ref count went to zero, delete the polling island. + Note that this deletion not be done under a lock. Once the ref count goes + to zero, we are guaranteed that no one else holds a reference to the + polling island (and that there is no racing pi_add_ref() call either). + + Also, if we are deleting the polling island and the merged_to field is + non-empty, we should remove a ref to the merged_to polling island + */ + if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + polling_island_delete(exec_ctx, pi); + if (next != NULL) { + PI_UNREF(exec_ctx, next, "pi_delete"); /* Recursive call */ + } + } +} + +static void worker_node_init(worker_node *node) { + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void push_back_worker_node(worker_node *head, worker_node *node) { + node->next = head; + node->prev = head->prev; + head->prev->next = node; + head->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void remove_worker_node(worker_node *node) { + node->next->prev = node->prev; + node->prev->next = node->next; + /* If node's next and prev point to itself, the node is considered detached + * from the list*/ + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static worker_node *pop_front_worker_node(worker_node *head) { + worker_node *node = head->next; + if (node != head) { + remove_worker_node(node); + } else { + node = NULL; + } + + return node; +} + +/* Returns true if the node's next and prev are pointing to itself (which + indicates that the node is not in the list */ +static bool is_worker_node_detached(worker_node *node) { + return (node->next == node->prev && node->next == node); +} + +/* The caller is expected to hold pi->mu lock before calling this function + */ +static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, + size_t fd_count, bool add_fd_refs, + grpc_error **error) { + int err; + size_t i; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "polling_island_add_fds"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + for (i = 0; i < fd_count; i++) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fds[i]; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); + + if (err < 0) { + if (errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + continue; + } + + if (pi->fd_cnt == pi->fd_capacity) { + pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); + pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); + } + + pi->fds[pi->fd_cnt++] = fds[i]; + if (add_fd_refs) { + GRPC_FD_REF(fds[i], "polling_island"); + } + } +} + +/* The caller is expected to hold pi->mu before calling this */ +static void polling_island_add_wakeup_fd_locked(polling_island *pi, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "polling_island_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_all_fds_locked(polling_island *pi, + bool remove_fd_refs, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fds"; + + for (i = 0; i < pi->fd_cnt; i++) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with " + "error: %d (%s)", + pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + if (remove_fd_refs) { + GRPC_FD_UNREF(pi->fds[i], "polling_island"); + } + } + + pi->fd_cnt = 0; +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } + + for (i = 0; i < pi->fd_cnt; i++) { + if (pi->fds[i] == fd) { + pi->fds[i] = pi->fds[--pi->fd_cnt]; + GRPC_FD_UNREF(fd, "polling_island"); + break; + } + } +} + +/* Might return NULL in case of an error */ +static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, + grpc_fd *initial_fd, + grpc_error **error) { + polling_island *pi = NULL; + const char *err_desc = "polling_island_create"; + + *error = GRPC_ERROR_NONE; + + pi = gpr_malloc(sizeof(*pi)); + pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + gpr_mu_init(&pi->mu); + pi->fd_cnt = 0; + pi->fd_capacity = 0; + pi->fds = NULL; + pi->epoll_fd = -1; + + gpr_mu_init(&pi->workqueue_read_mu); + gpr_mpscq_init(&pi->workqueue_items); + gpr_atm_rel_store(&pi->workqueue_item_count, 0); + + gpr_atm_rel_store(&pi->ref_count, 0); + gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); + + gpr_mu_init(&pi->worker_list_mu); + worker_node_init(&pi->worker_list_head); + + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (pi->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); + polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + + if (initial_fd != NULL) { + polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); + } + +done: + if (*error != GRPC_ERROR_NONE) { + polling_island_delete(exec_ctx, pi); + pi = NULL; + } + return pi; +} + +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { + GPR_ASSERT(pi->fd_cnt == 0); + + if (pi->epoll_fd >= 0) { + close(pi->epoll_fd); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); + gpr_mu_destroy(&pi->workqueue_read_mu); + gpr_mpscq_destroy(&pi->workqueue_items); + gpr_mu_destroy(&pi->mu); + grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + gpr_mu_destroy(&pi->worker_list_mu); + GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); + + gpr_free(pi->fds); + gpr_free(pi); +} + +/* Attempts to gets the last polling island in the linked list (liked by the + * 'merged_to' field). Since this does not lock the polling island, there are no + * guarantees that the island returned is the last island */ +static polling_island *polling_island_maybe_get_latest(polling_island *pi) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + while (next != NULL) { + pi = next; + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + } + + return pi; +} + +/* Gets the lock on the *latest* polling island i.e the last polling island in + the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the + returned polling island's mu. + Usage: To lock/unlock polling island "pi", do the following: + polling_island *pi_latest = polling_island_lock(pi); + ... + ... critical section .. + ... + gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */ +static polling_island *polling_island_lock(polling_island *pi) { + polling_island *next = NULL; + + while (true) { + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* Looks like 'pi' is the last node in the linked list but unless we check + this by holding the pi->mu lock, we cannot be sure (i.e without the + pi->mu lock, we don't prevent island merges). + To be absolutely sure, check once more by holding the pi->mu lock */ + gpr_mu_lock(&pi->mu); + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* pi is infact the last node and we have the pi->mu lock. we're done */ + break; + } + + /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu + * isn't the lock we are interested in. Continue traversing the list */ + gpr_mu_unlock(&pi->mu); + } + + pi = next; + } + + return pi; +} + +/* Gets the lock on the *latest* polling islands in the linked lists pointed by + *p and *q (and also updates *p and *q to point to the latest polling islands) + + This function is needed because calling the following block of code to obtain + locks on polling islands (*p and *q) is prone to deadlocks. + { + polling_island_lock(*p, true); + polling_island_lock(*q, true); + } + + Usage/example: + polling_island *p1; + polling_island *p2; + .. + polling_island_lock_pair(&p1, &p2); + .. + .. Critical section with both p1 and p2 locked + .. + // Release locks: Always call polling_island_unlock_pair() to release locks + polling_island_unlock_pair(p1, p2); +*/ +static void polling_island_lock_pair(polling_island **p, polling_island **q) { + polling_island *pi_1 = *p; + polling_island *pi_2 = *q; + polling_island *next_1 = NULL; + polling_island *next_2 = NULL; + + /* The algorithm is simple: + - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and + keep updating pi_1 and pi_2) + - Then obtain locks on the islands by following a lock order rule of + locking polling_island with lower address first + Special case: Before obtaining the locks, check if pi_1 and pi_2 are + pointing to the same island. If that is the case, we can just call + polling_island_lock() + - After obtaining both the locks, double check that the polling islands + are still the last polling islands in their respective linked lists + (this is because there might have been polling island merges before + we got the lock) + - If the polling islands are the last islands, we are done. If not, + release the locks and continue the process from the first step */ + while (true) { + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + while (next_1 != NULL) { + pi_1 = next_1; + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + } + + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + while (next_2 != NULL) { + pi_2 = next_2; + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + } + + if (pi_1 == pi_2) { + pi_1 = pi_2 = polling_island_lock(pi_1); + break; + } + + if (pi_1 < pi_2) { + gpr_mu_lock(&pi_1->mu); + gpr_mu_lock(&pi_2->mu); + } else { + gpr_mu_lock(&pi_2->mu); + gpr_mu_lock(&pi_1->mu); + } + + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + if (next_1 == NULL && next_2 == NULL) { + break; + } + + gpr_mu_unlock(&pi_1->mu); + gpr_mu_unlock(&pi_2->mu); + } + + *p = pi_1; + *q = pi_2; +} + +static void polling_island_unlock_pair(polling_island *p, polling_island *q) { + if (p == q) { + gpr_mu_unlock(&p->mu); + } else { + gpr_mu_unlock(&p->mu); + gpr_mu_unlock(&q->mu); + } +} + +static void workqueue_maybe_wakeup(polling_island *pi) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_polling_island == pi); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + } +} + +static void workqueue_move_items_to_parent(polling_island *q) { + polling_island *p = (polling_island *)gpr_atm_no_barrier_load(&q->merged_to); + if (p == NULL) { + return; + } + gpr_mu_lock(&q->workqueue_read_mu); + int num_added = 0; + while (gpr_atm_no_barrier_load(&q->workqueue_item_count) > 0) { + gpr_mpscq_node *n = gpr_mpscq_pop(&q->workqueue_items); + if (n != NULL) { + gpr_atm_no_barrier_fetch_add(&q->workqueue_item_count, -1); + gpr_atm_no_barrier_fetch_add(&p->workqueue_item_count, 1); + gpr_mpscq_push(&p->workqueue_items, n); + num_added++; + } + } + gpr_mu_unlock(&q->workqueue_read_mu); + if (num_added > 0) { + workqueue_maybe_wakeup(p); + } + workqueue_move_items_to_parent(p); +} + +static polling_island *polling_island_merge(polling_island *p, + polling_island *q, + grpc_error **error) { + /* Get locks on both the polling islands */ + polling_island_lock_pair(&p, &q); + + if (p != q) { + /* Make sure that p points to the polling island with fewer fds than q */ + if (p->fd_cnt > q->fd_cnt) { + GPR_SWAP(polling_island *, p, q); + } + + /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q + Note that the refcounts on the fds being moved will not change here. + This is why the last param in the following two functions is 'false') */ + polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error); + polling_island_remove_all_fds_locked(p, false, error); + + /* Wakeup all the pollers (if any) on p so that they pickup this change */ + polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error); + + /* Add the 'merged_to' link from p --> q */ + gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); + PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ + + workqueue_move_items_to_parent(p); + } + /* else if p == q, nothing needs to be done */ + + polling_island_unlock_pair(p, q); + + /* Return the merged polling island (Note that no merge would have happened + if p == q which is ok) */ + return q; +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + polling_island *pi = (polling_island *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(pi); + } + workqueue_move_items_to_parent(pi); + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + polling_island *pi = (polling_island *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &pi->workqueue_scheduler; +} + +static grpc_error *polling_island_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + } + + return error; +} + +static void polling_island_global_shutdown() { + grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->po.mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->po.mu); + } + + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->po.mu); + new_fd->po.pi = NULL; +#ifdef PO_DEBUG + new_fd->po.obj_type = POLL_OBJ_FD; +#endif + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->po.mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->po.mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->po.mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->po.mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->po.pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->po.pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->po.pi != NULL) { + polling_island *pi_latest = polling_island_lock(fd->po.pi); + polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi_latest->mu); + + unref_pi = fd->po.pi; + fd->po.pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->po.mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + gpr_mu_lock(&fd->po.mu); + grpc_workqueue *workqueue = + GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); + gpr_mu_unlock(&fd->po.mu); + return workqueue; +} + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static __thread bool g_initialized_sigmask; +static __thread sigset_t g_orig_sigmask; +static __thread sigset_t g_wakeup_sig_set; + +static void sig_handler(int sig_num) { +#ifdef GRPC_EPOLL_DEBUG + gpr_log(GPR_INFO, "Received signal %d", sig_num); +#endif +} + +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->pt_id = pthread_self(); + worker->next = worker->prev = NULL; + gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); + gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); + worker_node_init(&worker->pi_list_link); +} + +static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + poller_kick_init(); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *worker_kick(grpc_pollset_worker *worker, + gpr_atm *is_kicked) { + grpc_error *err = GRPC_ERROR_NONE; + + /* Kick the worker only if it was not already kicked */ + if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { + GRPC_POLLING_TRACE( + "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", + (void *)worker, (long int)worker->pt_id); + int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); + if (err_num != 0) { + err = GRPC_OS_ERROR(err_num, "pthread_kill"); + } + } + return err; +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_kicked); +} + +static grpc_error *poller_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_polling_turn); +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->po.mu); + *mu = &pollset->po.mu; + pollset->po.pi = NULL; +#ifdef PO_DEBUG + pollset->po.obj_type = POLL_OBJ_POLLSET; +#endif + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; +} + +/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, + grpc_pollset *ps, char *reason) { + if (ps->po.pi != NULL) { + PI_UNREF(exec_ctx, ps->po.pi, reason); + } + ps->po.pi = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->po.pi to NULL */ + pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying polling island. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->po.mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, + polling_island *pi) { + if (gpr_mu_trylock(&pi->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); + gpr_mu_unlock(&pi->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(pi); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(pi); + } + } + return false; +} + +/* NOTE: This function may modify 'now' */ +static bool acquire_polling_lease(grpc_pollset_worker *worker, + polling_island *pi, gpr_timespec deadline, + gpr_timespec *now) { + bool is_lease_acquired = false; + + gpr_mu_lock(&pi->worker_list_mu); // LOCK + long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + + if (num_pollers >= g_max_pollers_per_pi) { + push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + + bool is_timeout = false; + int ret; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); + if (timeout_ms == -1) { + ret = sigwaitinfo(&g_wakeup_sig_set, NULL); + } else { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + GRPC_SCHEDULING_START_BLOCKING_REGION; + ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (ret == -1) { + if (errno == EAGAIN) { + is_timeout = true; + } else { + /* NOTE: This should not happen. If we see these log messages, it means + we are most likely doing something incorrect in the setup * needed + for sigwaitinfo/sigtimedwait */ + gpr_log(GPR_ERROR, + "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, + timeout_ms); + } + } + + /* Did the worker come out of sigtimedwait due to a thread that just + exited epoll and kicking it (in release_polling_lease function). */ + bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); + + /* Did the worker come out of sigtimedwait due to a thread alerting it that + some completion event was (likely) available in the completion queue */ + bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + + if (is_kicked || is_timeout) { + *now = deadline; /* Essentially make the epoll timeout = 0 */ + } else if (is_polling_turn) { + *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ + } + + gpr_mu_lock(&pi->worker_list_mu); // LOCK + /* The node might have already been removed from the list by the poller + that kicked this. However it is safe to call 'remove_worker_node' on + an already detached node */ + remove_worker_node(&worker->pi_list_link); + /* It is important to read the num_pollers again under the lock so that we + * have the latest num_pollers value that doesn't change while we are doing + * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ + num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + } + + if (num_pollers < g_max_pollers_per_pi) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); + is_lease_acquired = true; + } + + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + return is_lease_acquired; +} + +static void release_polling_lease(polling_island *pi, grpc_error **error) { + gpr_mu_lock(&pi->worker_list_mu); + + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + worker_node *node = pop_front_worker_node(&pi->worker_list_head); + if (node != NULL) { + grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); + append_error(error, poller_kick(next_worker), "poller kick error"); + } + + gpr_mu_unlock(&pi->worker_list_mu); +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + /* Only g_max_pollers_per_pi threads can be doing polling in parallel. + If we cannot get a lease, we cannot continue to do epoll_pwait() */ + if (!acquire_polling_lease(worker, pi, deadline, &now)) { + return; + } + + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + /* timeout_ms is the time between 'now' and 'deadline' */ + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = + epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + /* Give back the lease right away so that some other thread can enter */ + release_polling_lease(pi, error); + + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + +/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ +static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + int epoll_fd = -1; + polling_island *pi = NULL; + GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + + /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the + latest polling island pointed by pollset->po.pi + + Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. There is however a possibility that the polling island from + which we got the epoll_fd, got merged with another island in the meantime. + This is okay because in such a case, we will wakeup right-away from + epoll_pwait() (because any merge will poison the old polling island's epoll + set 'polling_island_wakeup_fd') and then pick up the latest polling_island + the next time this function - pollset_work_and_unlock()) is called */ + + if (pollset->po.pi == NULL) { + pollset->po.pi = polling_island_create(exec_ctx, NULL, error); + if (pollset->po.pi == NULL) { + GPR_TIMER_END("pollset_work_and_unlock", 0); + return; /* Fatal error. Cannot continue */ + } + + PI_ADD_REF(pollset->po.pi, "ps"); + GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", + (void *)pollset, (void *)pollset->po.pi); + } + + pi = polling_island_maybe_get_latest(pollset->po.pi); + epoll_fd = pi->epoll_fd; + + /* Update the pollset->po.pi since the island being pointed by + pollset->po.pi maybe older than the one pointed by pi) */ + if (pollset->po.pi != pi) { + /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the + polling island to be deleted */ + PI_ADD_REF(pi, "ps"); + PI_UNREF(exec_ctx, pollset->po.pi, "ps"); + pollset->po.pi = pi; + } + + /* Add an extra ref so that the island does not get destroyed (which means + the epoll_fd won't be closed) while we are are doing an epoll_wait() on the + epoll_fd */ + PI_ADD_REF(pi, "ps_work"); + gpr_mu_unlock(&pollset->po.mu); + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, pi)) { + g_current_thread_polling_island = pi; + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, + deadline, sig_mask, error); + g_current_thread_polling_island = NULL; + } + + GPR_ASSERT(pi != NULL); + + /* Before leaving, release the extra ref we added to the polling island. It + is important to use "pi" here (i.e our old copy of pollset->po.pi + that we got before releasing the polling island lock). This is because + pollset->po.pi pointer might get udpated in other parts of the + code when there is an island merge while we are doing epoll_wait() above */ + PI_UNREF(exec_ctx, pi, "ps_work"); + + GPR_TIMER_END("pollset_work_and_unlock", 0); +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + + grpc_pollset_worker worker; + pollset_worker_init(&worker); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up + (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the + worker that there is some pending work that needs immediate attention + (like an event on the completion queue, or a polling island merge that + results in a new epoll-fd to wait on) and that the worker should not + spend time waiting in epoll_pwait(). + + A worker can be kicked anytime from the point it is added to the pollset + via push_front_worker() (or push_back_worker()) to the point it is + removed via remove_worker(). + If the worker is kicked before/during it calls epoll_pwait(), it should + immediately exit from epoll_wait(). If the worker is kicked after it + returns from epoll_wait(), then nothing really needs to be done. + + To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all + times *except* when it is in epoll_pwait(). This way, the worker never + misses acting on a kick */ + + if (!g_initialized_sigmask) { + sigemptyset(&g_wakeup_sig_set); + sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); + sigdelset(&g_orig_sigmask, grpc_wakeup_signal); + g_initialized_sigmask = true; + /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. + This is the mask used at all times *except during + epoll_wait()*" + g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and + this is the mask to use *during epoll_wait()* + + The new_mask is set on the worker before it is added to the pollset + (i.e before it can be kicked) */ + } + + push_front_worker(pollset, &worker); /* Add worker to pollset */ + + pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, + &g_orig_sigmask, &error); + grpc_exec_ctx_flush(exec_ctx); + + gpr_mu_lock(&pollset->po.mu); + + /* Note: There is no need to reset worker.is_kicked to 0 since we are no + longer going to use this worker */ + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->po.mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag, + poll_obj_type bag_type, poll_obj *item, + poll_obj_type item_type) { + GPR_TIMER_BEGIN("add_poll_object", 0); + +#ifdef PO_DEBUG + GPR_ASSERT(item->obj_type == item_type); + GPR_ASSERT(bag->obj_type == bag_type); +#endif + + grpc_error *error = GRPC_ERROR_NONE; + polling_island *pi_new = NULL; + + gpr_mu_lock(&bag->mu); + gpr_mu_lock(&item->mu); + +retry: + /* + * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing + * 2) If item->pi and bag->pi are both NULL, create a new polling island (with + * a refcount of 2) and point item->pi and bag->pi to the new island + * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to + * the other's non-NULL pi + * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the + * polling islands and update item->pi and bag->pi to point to the new + * island + */ + + /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already + * orphaned */ + if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) { + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + return; + } + + if (item->pi == bag->pi) { + pi_new = item->pi; + if (pi_new == NULL) { + /* GPR_ASSERT(item->pi == bag->pi == NULL) */ + + /* If we are adding an fd to a bag (i.e pollset or pollset_set), then + * we need to do some extra work to make TSAN happy */ + if (item_type == POLL_OBJ_FD) { + /* Unlock before creating a new polling island: the polling island will + create a workqueue which creates a file descriptor, and holding an fd + lock here can eventually cause a loop to appear to TSAN (making it + unhappy). We don't think it's a real loop (there's an epoch point + where that loop possibility disappears), but the advantages of + keeping TSAN happy outweigh any performance advantage we might have + by keeping the lock held. */ + gpr_mu_unlock(&item->mu); + pi_new = polling_island_create(exec_ctx, FD_FROM_PO(item), &error); + gpr_mu_lock(&item->mu); + + /* Need to reverify any assumptions made between the initial lock and + getting to this branch: if they've changed, we need to throw away our + work and figure things out again. */ + if (item->pi != NULL) { + GRPC_POLLING_TRACE( + "add_poll_object: Raced creating new polling island. pi_new: %p " + "(fd: %d, %s: %p)", + (void *)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type), + (void *)bag); + /* No need to lock 'pi_new' here since this is a new polling island + and no one has a reference to it yet */ + polling_island_remove_all_fds_locked(pi_new, true, &error); + + /* Ref and unref so that the polling island gets deleted during unref + */ + PI_ADD_REF(pi_new, "dance_of_destruction"); + PI_UNREF(exec_ctx, pi_new, "dance_of_destruction"); + goto retry; + } + } else { + pi_new = polling_island_create(exec_ctx, NULL, &error); + } + + GRPC_POLLING_TRACE( + "add_poll_object: Created new polling island. pi_new: %p (%s: %p, " + "%s: %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + GRPC_POLLING_TRACE( + "add_poll_object: Same polling island. pi: %p (%s, %s)", + (void *)pi_new, poll_obj_string(item_type), + poll_obj_string(bag_type)); + } + } else if (item->pi == NULL) { + /* GPR_ASSERT(bag->pi != NULL) */ + /* Make pi_new point to latest pi*/ + pi_new = polling_island_lock(bag->pi); + + if (item_type == POLL_OBJ_FD) { + grpc_fd *fd = FD_FROM_PO(item); + polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); + } + + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else if (bag->pi == NULL) { + /* GPR_ASSERT(item->pi != NULL) */ + /* Make pi_new to point to latest pi */ + pi_new = polling_island_lock(item->pi); + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + pi_new = polling_island_merge(item->pi, bag->pi, &error); + GRPC_POLLING_TRACE( + "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } + + /* At this point, pi_new is the polling island that both item->pi and bag->pi + MUST be pointing to */ + + if (item->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(item_type)); + if (item->pi != NULL) { + PI_UNREF(exec_ctx, item->pi, poll_obj_string(item_type)); + } + item->pi = pi_new; + } + + if (bag->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(bag_type)); + if (bag->pi != NULL) { + PI_UNREF(exec_ctx, bag->pi, poll_obj_string(bag_type)); + } + bag->pi = pi_new; + } + + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + + GRPC_LOG_IF_ERROR("add_poll_object", error); + GPR_TIMER_END("add_poll_object", 0); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pollset->po, POLL_OBJ_POLLSET, &fd->po, + POLL_OBJ_FD); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + gpr_mu_init(&pss->po.mu); + pss->po.pi = NULL; +#ifdef PO_DEBUG + pss->po.obj_type = POLL_OBJ_POLLSET_SET; +#endif + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + gpr_mu_destroy(&pss->po.mu); + + if (pss->po.pi != NULL) { + PI_UNREF(exec_ctx, pss->po.pi, "pss_destroy"); + } + + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &fd->po, + POLL_OBJ_FD); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &ps->po, + POLL_OBJ_POLLSET); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + add_poll_object(exec_ctx, &bag->po, POLL_OBJ_POLLSET_SET, &item->po, + POLL_OBJ_POLLSET_SET); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +/* This is mainly for testing purposes. Checks to see if environment variable + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to + * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ +static void set_max_pollers_per_island() { + char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); + if (s) { + g_max_pollers_per_pi = (int)strtol(s, NULL, 10); + if (g_max_pollers_per_pi < 0) { + g_max_pollers_per_pi = INT_MAX; + } + } else { + g_max_pollers_per_pi = INT_MAX; + } + + gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", + g_max_pollers_per_pi); +} + +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux(void) { + /* If use of signals is disabled, we cannot use epoll engine*/ + if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { + return NULL; + } + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + if (!is_grpc_wakeup_signal_initialized) { + grpc_use_signal(SIGRTMIN + 6); + } + + set_max_pollers_per_island(); + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux(void) { + return NULL; +} +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h new file mode 100644 index 0000000000..23ff8570c1 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 64ea7b192a..e603a75593 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -40,7 +40,6 @@ #include #include -#include #include #include #include @@ -63,7 +62,6 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -#include "src/core/lib/support/env.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ static int grpc_polling_trace = 0; /* Disabled by default */ @@ -75,10 +73,6 @@ static int grpc_polling_trace = 0; /* Disabled by default */ /* Uncomment the following to enable extra checks on poll_object operations */ /* #define PO_DEBUG */ -/* The maximum number of polling threads per polling island. By default no - limit */ -static int g_max_pollers_per_pi = INT_MAX; - static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; @@ -201,11 +195,6 @@ static void fd_global_shutdown(void); #endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ -typedef struct worker_node { - struct worker_node *next; - struct worker_node *prev; -} worker_node; - /* This is also used as grpc_workqueue (by directly casing it) */ typedef struct polling_island { grpc_closure_scheduler workqueue_scheduler; @@ -240,10 +229,6 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; - /* The list of workers waiting to do polling on this polling island */ - gpr_mu worker_list_mu; - worker_node worker_list_head; - /* The fd of the underlying epoll set */ int epoll_fd; @@ -256,24 +241,14 @@ typedef struct polling_island { /******************************************************************************* * Pollset Declarations */ -#define WORKER_FROM_WORKER_LIST_NODE(p) \ - (struct grpc_pollset_worker *)(((char *)(p)) - \ - offsetof(grpc_pollset_worker, pi_list_link)) struct grpc_pollset_worker { /* Thread id of this worker */ pthread_t pt_id; /* Used to prevent a worker from getting kicked multiple times */ gpr_atm is_kicked; - struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; - - /* Indicates if it is this worker's turn to do epoll */ - gpr_atm is_polling_turn; - - /* Node in the polling island's worker list. */ - worker_node pi_list_link; }; struct grpc_pollset { @@ -417,47 +392,7 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { } } -static void worker_node_init(worker_node *node) { - node->next = node->prev = node; -} - -/* Not thread safe. Do under a list-level lock */ -static void push_back_worker_node(worker_node *head, worker_node *node) { - node->next = head; - node->prev = head->prev; - head->prev->next = node; - head->prev = node; -} - -/* Not thread safe. Do under a list-level lock */ -static void remove_worker_node(worker_node *node) { - node->next->prev = node->prev; - node->prev->next = node->next; - /* If node's next and prev point to itself, the node is considered detached - * from the list*/ - node->next = node->prev = node; -} - -/* Not thread safe. Do under a list-level lock */ -static worker_node *pop_front_worker_node(worker_node *head) { - worker_node *node = head->next; - if (node != head) { - remove_worker_node(node); - } else { - node = NULL; - } - - return node; -} - -/* Returns true if the node's next and prev are pointing to itself (which - indicates that the node is not in the list */ -static bool is_worker_node_detached(worker_node *node) { - return (node->next == node->prev && node->next == node); -} - -/* The caller is expected to hold pi->mu lock before calling this function - */ +/* The caller is expected to hold pi->mu lock before calling this function */ static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, size_t fd_count, bool add_fd_refs, grpc_error **error) { @@ -611,9 +546,6 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, gpr_atm_rel_store(&pi->poller_count, 0); gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); - gpr_mu_init(&pi->worker_list_mu); - worker_node_init(&pi->worker_list_head); - if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), err_desc)) { goto done; @@ -652,9 +584,6 @@ static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { gpr_mpscq_destroy(&pi->workqueue_items); gpr_mu_destroy(&pi->mu); grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); - gpr_mu_destroy(&pi->worker_list_mu); - GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); - gpr_free(pi->fds); gpr_free(pi); } @@ -1173,7 +1102,6 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static __thread bool g_initialized_sigmask; static __thread sigset_t g_orig_sigmask; -static __thread sigset_t g_wakeup_sig_set; static void sig_handler(int sig_num) { #ifdef GRPC_EPOLL_DEBUG @@ -1181,14 +1109,6 @@ static void sig_handler(int sig_num) { #endif } -static void pollset_worker_init(grpc_pollset_worker *worker) { - worker->pt_id = pthread_self(); - worker->next = worker->prev = NULL; - gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); - gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); - worker_node_init(&worker->pi_list_link); -} - static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } /* Global state management */ @@ -1205,12 +1125,11 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *worker_kick(grpc_pollset_worker *worker, - gpr_atm *is_kicked) { +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { grpc_error *err = GRPC_ERROR_NONE; /* Kick the worker only if it was not already kicked */ - if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { + if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { GRPC_POLLING_TRACE( "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", (void *)worker, (long int)worker->pt_id); @@ -1222,14 +1141,6 @@ static grpc_error *worker_kick(grpc_pollset_worker *worker, return err; } -static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { - return worker_kick(worker, &worker->is_kicked); -} - -static grpc_error *poller_kick(grpc_pollset_worker *worker) { - return worker_kick(worker, &worker->is_polling_turn); -} - /* Return 1 if the pollset has active threads in pollset_work (pollset must * be locked) */ static int pollset_has_workers(grpc_pollset *p) { @@ -1335,22 +1246,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->shutdown_done = NULL; } -/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ -static struct timespec millis_to_timespec(int millis) { - struct timespec linux_ts; - gpr_timespec gpr_ts; - - if (millis == -1) { - gpr_ts = gpr_inf_future(GPR_TIMESPAN); - } else { - gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); - } - - linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; - linux_ts.tv_nsec = gpr_ts.tv_nsec; - return linux_ts; -} - /* Convert a timespec to milliseconds: - Very small or negative poll times are clamped to zero to do a non-blocking poll (which becomes spin polling) @@ -1469,200 +1364,35 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, return false; } -/* NOTE: This function may modify 'now' */ -static bool acquire_polling_lease(grpc_pollset_worker *worker, - polling_island *pi, gpr_timespec deadline, - gpr_timespec *now) { - bool is_lease_acquired = false; - - gpr_mu_lock(&pi->worker_list_mu); // LOCK - long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - - if (num_pollers >= g_max_pollers_per_pi) { - push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); - gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK - - bool is_timeout = false; - int ret; - int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); - if (timeout_ms == -1) { - ret = sigwaitinfo(&g_wakeup_sig_set, NULL); - } else { - struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); - GRPC_SCHEDULING_START_BLOCKING_REGION; - ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); - GRPC_SCHEDULING_END_BLOCKING_REGION; - } - - if (ret == -1) { - if (errno == EAGAIN) { - is_timeout = true; - } else { - /* NOTE: This should not happen. If we see these log messages, it means - we are most likely doing something incorrect in the setup * needed - for sigwaitinfo/sigtimedwait */ - gpr_log(GPR_ERROR, - "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, - timeout_ms); - } - } - - /* Did the worker come out of sigtimedwait due to a thread that just - exited epoll and kicking it (in release_polling_lease function). */ - bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); - - /* Did the worker come out of sigtimedwait due to a thread alerting it that - some completion event was (likely) available in the completion queue */ - bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); - - if (is_kicked || is_timeout) { - *now = deadline; /* Essentially make the epoll timeout = 0 */ - } else if (is_polling_turn) { - *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ - } - - gpr_mu_lock(&pi->worker_list_mu); // LOCK - /* The node might have already been removed from the list by the poller - that kicked this. However it is safe to call 'remove_worker_node' on - an already detached node */ - remove_worker_node(&worker->pi_list_link); - /* It is important to read the num_pollers again under the lock so that we - * have the latest num_pollers value that doesn't change while we are doing - * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ - num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - } - - if (num_pollers < g_max_pollers_per_pi) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); - is_lease_acquired = true; - } - - gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK - return is_lease_acquired; -} - -static void release_polling_lease(polling_island *pi, grpc_error **error) { - gpr_mu_lock(&pi->worker_list_mu); - - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); - worker_node *node = pop_front_worker_node(&pi->worker_list_head); - if (node != NULL) { - grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); - append_error(error, poller_kick(next_worker), "poller kick error"); - } - - gpr_mu_unlock(&pi->worker_list_mu); -} - #define GRPC_EPOLL_MAX_EVENTS 100 -static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, - grpc_pollset *pollset, polling_island *pi, - grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline, - sigset_t *sig_mask, grpc_error **error) { - /* Only g_max_pollers_per_pi threads can be doing polling in parallel. - If we cannot get a lease, we cannot continue to do epoll_pwait() */ - if (!acquire_polling_lease(worker, pi, deadline, &now)) { - return; - } - - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; - int ep_rv; - char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; - - /* timeout_ms is the time between 'now' and 'deadline' */ - int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); - - GRPC_SCHEDULING_START_BLOCKING_REGION; - ep_rv = - epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); - GRPC_SCHEDULING_END_BLOCKING_REGION; - - /* Give back the lease right away so that some other thread can enter */ - release_polling_lease(pi, error); - - if (ep_rv < 0) { - if (errno != EINTR) { - gpr_asprintf(&err_msg, - "epoll_wait() epoll fd: %d failed with error: %d (%s)", - epoll_fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - } else { - /* We were interrupted. Save an interation by doing a zero timeout - epoll_wait to see if there are any other events of interest */ - GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", - (void *)pollset, (void *)worker); - ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - } - } - -#ifdef GRPC_TSAN - /* See the definition of g_poll_sync for more details */ - gpr_atm_acq_load(&g_epoll_sync); -#endif /* defined(GRPC_TSAN) */ - - for (int i = 0; i < ep_rv; ++i) { - void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { - append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), - err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " - "%d) got merged", - (void *)pollset, (void *)worker, epoll_fd); - /* This means that our polling island is merged with a different - island. We do not have to do anything here since the subsequent call - to the function pollset_work_and_unlock() will pick up the correct - epoll_fd */ - } else { - grpc_fd *fd = data_ptr; - int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write_ev = ep_ev[i].events & EPOLLOUT; - if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); - } - if (write_ev || cancel) { - fd_become_writable(exec_ctx, fd); - } - } - } -} - /* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline, + grpc_pollset_worker *worker, int timeout_ms, sigset_t *sig_mask, grpc_error **error) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int epoll_fd = -1; + int ep_rv; polling_island *pi = NULL; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the latest polling island pointed by pollset->po.pi - Since epoll_fd is immutable, it is safe to read it without a lock on the - polling island. There is however a possibility that the polling island from - which we got the epoll_fd, got merged with another island in the meantime. - This is okay because in such a case, we will wakeup right-away from - epoll_pwait() (because any merge will poison the old polling island's epoll - set 'polling_island_wakeup_fd') and then pick up the latest polling_island - the next time this function - pollset_work_and_unlock()) is called */ + Since epoll_fd is immutable, we can read it without obtaining the polling + island lock. There is however a possibility that the polling island (from + which we got the epoll_fd) got merged with another island while we are + in this function. This is still okay because in such a case, we will wakeup + right-away from epoll_wait() and pick up the latest polling_island the next + this function (i.e pollset_work_and_unlock()) is called */ if (pollset->po.pi == NULL) { pollset->po.pi = polling_island_create(exec_ctx, NULL, error); if (pollset->po.pi == NULL) { GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. Cannot continue */ + return; /* Fatal error. We cannot continue */ } PI_ADD_REF(pollset->po.pi, "ps"); @@ -1693,10 +1423,70 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ if (!maybe_do_workqueue_work(exec_ctx, pi)) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); g_current_thread_polling_island = pi; - pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, - deadline, sig_mask, error); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, + sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + g_current_thread_polling_island = NULL; + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); } GPR_ASSERT(pi != NULL); @@ -1720,9 +1510,14 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec now, gpr_timespec deadline) { GPR_TIMER_BEGIN("pollset_work", 0); grpc_error *error = GRPC_ERROR_NONE; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + sigset_t new_mask; grpc_pollset_worker worker; - pollset_worker_init(&worker); + worker.next = worker.prev = NULL; + worker.pt_id = pthread_self(); + gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); if (worker_hdl) *worker_hdl = &worker; @@ -1756,9 +1551,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, misses acting on a kick */ if (!g_initialized_sigmask) { - sigemptyset(&g_wakeup_sig_set); - sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); - pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); + sigemptyset(&new_mask); + sigaddset(&new_mask, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); sigdelset(&g_orig_sigmask, grpc_wakeup_signal); g_initialized_sigmask = true; /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. @@ -1773,7 +1568,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, push_front_worker(pollset, &worker); /* Add worker to pollset */ - pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, + pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms, &g_orig_sigmask, &error); grpc_exec_ctx_flush(exec_ctx); @@ -2126,24 +1921,6 @@ static bool is_epoll_available() { return true; } -/* This is mainly for testing purposes. Checks to see if environment variable - * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to - * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ -static void set_max_pollers_per_island() { - char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); - if (s) { - g_max_pollers_per_pi = (int)strtol(s, NULL, 10); - if (g_max_pollers_per_pi < 0) { - g_max_pollers_per_pi = INT_MAX; - } - } else { - g_max_pollers_per_pi = INT_MAX; - } - - gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", - g_max_pollers_per_pi); -} - const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { @@ -2162,8 +1939,6 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { grpc_use_signal(SIGRTMIN + 6); } - set_max_pollers_per_island(); - fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de8..07a9ed570a 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,6 +44,7 @@ #include #include +#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" #include "src/core/lib/iomgr/ev_epoll_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -68,6 +69,7 @@ static const event_engine_factory g_factories[] = { {"epoll", grpc_init_epoll_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, + {"epoll-limited", grpc_init_epoll_limited_pollers_linux}, }; static void add(const char *beg, const char *end, char ***ss, size_t *ns) { diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 42c13b2cb6..c3053612e4 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -97,6 +97,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c', 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f..5ce89e7a8e 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -939,6 +939,8 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 51d77c2b52..54a7ed4e97 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1078,6 +1078,8 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index c962823437..f2c467e4ea 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7750,6 +7750,7 @@ "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", + "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h", "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", @@ -7891,6 +7892,8 @@ "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", + "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c", + "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a58..4d44d450a2 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -396,6 +396,7 @@ + @@ -610,6 +611,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc297..b2afed1e50 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -184,6 +184,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -908,6 +911,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651..fd207a2c79 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -390,6 +390,7 @@ + @@ -594,6 +595,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab3..531ae52047 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -169,6 +169,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -875,6 +878,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 86b5856d68..9e0fb5de8d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -320,6 +320,7 @@ + @@ -551,6 +552,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 943ad521f7..4abf85b10f 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -64,6 +64,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -875,6 +878,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97..60deba9243 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -215,6 +215,7 @@ + @@ -383,6 +384,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4..44a29fadb7 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -121,6 +121,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -623,6 +626,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 2db0ffa692..6fdf6bb690 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -310,6 +310,7 @@ + @@ -518,6 +519,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index c7d6670db1..707ab9e098 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -67,6 +67,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -785,6 +788,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr -- cgit v1.2.3 From 93f307a9ba6ca01ba43ef7bdbff39e687450d40d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:26:22 -0700 Subject: Fix trace dependency mess --- src/core/lib/debug/trace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c index 54653868b8..4dfea44c57 100644 --- a/src/core/lib/debug/trace.c +++ b/src/core/lib/debug/trace.c @@ -35,11 +35,12 @@ #include -#include #include #include #include "src/core/lib/support/env.h" +int grpc_tracer_set_enabled(const char *name, int enabled); + typedef struct tracer { const char *name; grpc_tracer_flag *flag; -- cgit v1.2.3 From f70fe1dfc4779970fb43a70814a8bfd8307e264e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:27:23 -0700 Subject: Fix mac build --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index ab23c2feaa..d31d4631a0 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -1328,7 +1328,7 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux( +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( bool requested_explicitly) { return NULL; } -- cgit v1.2.3 From 020176dd5c9cbec8b32348271e39fa6121b7b2f9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:36:44 -0700 Subject: Fix tracer decl --- src/core/lib/iomgr/tcp_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c index 8e8db9f7b4..aba8136ccb 100644 --- a/src/core/lib/iomgr/tcp_uv.c +++ b/src/core/lib/iomgr/tcp_uv.c @@ -52,7 +52,7 @@ #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" -int grpc_tcp_trace = 0; +grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { grpc_endpoint base; -- cgit v1.2.3 From 341bcc581294b703e9413bcad200541bb6833989 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:37:44 -0700 Subject: Fix tracer usage --- src/core/lib/iomgr/tcp_uv.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c index aba8136ccb..e7157537f6 100644 --- a/src/core/lib/iomgr/tcp_uv.c +++ b/src/core/lib/iomgr/tcp_uv.c @@ -158,7 +158,7 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, sub = grpc_slice_sub_no_ref(tcp->read_slice, 0, (size_t)nread); grpc_slice_buffer_add(tcp->read_slices, sub); error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -199,7 +199,7 @@ static void uv_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice_from_static_string(uv_strerror(status))); grpc_closure_sched(exec_ctx, cb, error); } - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "Initiating read on %p: error=%s", tcp, str); } @@ -217,7 +217,7 @@ static void write_callback(uv_write_t *req, int status) { } else { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed"); } - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write complete on %p: error=%s", tcp, str); } @@ -238,7 +238,7 @@ static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice *slice; uv_write_t *write_req; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t j; for (j = 0; j < write_slices->count; j++) { @@ -346,7 +346,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle, char *peer_string) { grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "Creating TCP endpoint %p", tcp); } -- cgit v1.2.3 From 9cd102a4ebf7d8a76ba481748b20e450929bd65b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:38:41 -0700 Subject: Fix printf --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index d31d4631a0..bb44321922 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -1167,7 +1167,8 @@ static void add_fd_to_eps(grpc_fd *fd) { EPS_ADD_REF(eps, "fd"); fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, idx); + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %" PRIdPTR ")", fd->fd, + idx); gpr_mu_unlock(&fd->mu); GRPC_LOG_IF_ERROR("add_fd_to_eps", error); -- cgit v1.2.3 From 99ee8d53085d10a2d008f394321cf6fdb9623d8b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 12:36:20 -0700 Subject: Solve uv link problem --- src/core/lib/iomgr/timer_uv.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 8e8a07578c..3830d78ba8 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -42,6 +42,9 @@ #include +grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false); + static void timer_close_callback(uv_handle_t *handle) { gpr_free(handle); } static void stop_uv_timer(uv_timer_t *handle) { -- cgit v1.2.3 From 15285fbbf90964fb4e04727478207384390497ef Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 08:29:42 -0700 Subject: Add missing include --- src/core/lib/iomgr/timer_uv.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 3830d78ba8..8a4c01c4c5 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -38,6 +38,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/timer.h" #include -- cgit v1.2.3 From 6e18ef3880da523af4c131d34234aa1c4e4221c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 12:54:59 -0700 Subject: Fix portability --- src/core/lib/iomgr/timer_uv.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/core') diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 8a4c01c4c5..2952e44b58 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -104,4 +104,6 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, void grpc_timer_list_init(gpr_timespec now) {} void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {} +void grpc_timer_consume_kick(void) {} + #endif /* GRPC_UV */ -- cgit v1.2.3